[
  {
    "path": ".circleci/config.yml",
    "content": "version: 2.1\n\norbs:\n  slack: circleci/slack@4.2.0\n\ncommands:\n  install-go:\n    steps:\n      - run:\n          name: Install Go\n          command: |\n            sudo rm -rf /usr/local/go\n            wget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz\n            sudo tar -C /usr/local -xzf go1.17.3.linux-amd64.tar.gz\n            rm -rf go*.tar.gz\n            echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV\n            echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV\n            mkdir ~/go\n            echo 'export GOPATH=~/go' >> $BASH_ENV\n\n  quay-login:\n    description: Log Docker agent into Quay.io\n    steps:\n      - run:\n          name: Login to Quay\n          command: docker login -u=$QUAY_USERNAME -p=$QUAY_PASSWORD quay.io\n\njobs:\n  lint:\n    docker:\n      - image: cimg/python:3.7\n    resource_class: medium\n    steps:\n      - checkout\n      - install-go\n      - restore_cache:\n          keys:\n            - go-mod-v1-{{ checksum \"go.sum\" }}\n            - go-mod-v1-\n      - run:\n          name: Install Linting Tools\n          command: |\n            go get -u -v golang.org/x/lint/golint\n            go get -u -v github.com/kyoh86/looppointer/cmd/looppointer\n            pip3 install aiohttp black==20.8b1 click==8.0.4\n      - run:\n          name: Lint\n          command: make lint\n          no_output_timeout: 20m\n      - save_cache:\n          key: go-mod-v1-{{ checksum \"go.sum\" }}\n          paths:\n            - \"~/go/pkg/mod\"\n\n  test:\n    machine:\n      image: ubuntu-2004:202201-02 # machine executor necessary to run go integration tests\n    resource_class: medium\n    steps:\n      - checkout\n      - install-go\n      - restore_cache:\n          keys:\n            - go-mod-v1-{{ checksum \"go.sum\" }}\n            - go-mod-v1-\n      - run:\n          name: Initialize Credentials\n          command: |\n            echo 'export AWS_ACCESS_KEY_ID=${TEST_AWS_ACCESS_KEY_ID}' >> $BASH_ENV\n            echo 'export AWS_SECRET_ACCESS_KEY=${TEST_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV\n      - run:\n          name: Generate Cluster Config\n          command: |\n            mkdir -p dev/config\n            cat \\<< EOF > ./dev/config/cluster.yaml\n            cluster_name: cortex-nightly\n            region: us-east-1\n            node_groups:\n              - name: cpu\n                instance_type: m5.large\n                min_instances: 1\n                max_instances: 1\n            EOF\n      - run:\n          name: Go Tests\n          command: make test\n      - save_cache:\n          key: go-mod-v1-{{ checksum \"go.sum\" }}\n          paths:\n            - \"~/go/pkg/mod\"\n\n  build-and-upload-cli:\n    docker:\n      - image: cimg/python:3.7\n    resource_class: medium\n    steps:\n      - checkout\n      - install-go\n      - run: pip install awscli\n      - run: make ci-build-cli\n      - run: make ci-build-and-upload-cli\n\n  build-and-push-images-amd64:\n    machine:\n      image: ubuntu-2004:202101-01\n    resource_class: medium\n    steps:\n      - checkout\n      - run:\n          name: Build CI Images (amd64)\n          command: make ci-build-images-amd64\n          no_output_timeout: 20m\n      - quay-login\n      - run:\n          name: Push CI Images (amd64)\n          command: make ci-push-images-amd64\n          no_output_timeout: 20m\n\n  build-and-push-images-arm64:\n    machine:\n      image: ubuntu-2004:202201-02\n    resource_class: arm.medium\n    steps:\n      - checkout\n      - run:\n          name: Build CI Images (arm64)\n          command: make ci-build-images-arm64\n          no_output_timeout: 20m\n      - quay-login\n      - run:\n          name: Push CI Images (arm64)\n          command: make ci-push-images-arm64\n          no_output_timeout: 20m\n\n  amend-images:\n    docker:\n      - image: cimg/python:3.7\n    environment:\n      DOCKER_CLI_EXPERIMENTAL: enabled\n    resource_class: medium\n    steps:\n      - setup_remote_docker\n      - checkout\n      - quay-login\n      - run:\n          name: Amend CI Images\n          command: make ci-amend-images\n          no_output_timeout: 20m\n\n  cluster-up:\n    docker:\n      - image: cimg/python:3.7\n    steps:\n      - setup_remote_docker\n      - checkout\n      - run:\n          name: Install Dependencies\n          command: |\n            pip install boto3 pyyaml awscli\n            pip install https://s3-us-west-2.amazonaws.com/get-cortex/master/python/cortex-master.tar.gz\n      - run:\n          name: Initialize Credentials\n          command: |\n            echo 'export AWS_ACCESS_KEY_ID=${NIGHTLY_AWS_ACCESS_KEY_ID}' >> $BASH_ENV\n            echo 'export AWS_SECRET_ACCESS_KEY=${NIGHTLY_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV\n      - run:\n          name: Generate Cluster Config\n          # using a variety of node groups to test the multi-instance-type cluster functionality\n          command: |\n            cat \\<< EOF > ./cluster.yaml\n            cluster_name: cortex-nightly\n            region: us-east-1\n            node_groups:\n              - name: spot\n                instance_type: t3.medium\n                min_instances: 16\n                max_instances: 16\n                spot: true\n              - name: cpu\n                instance_type: c5.xlarge\n                min_instances: 1\n                max_instances: 2\n              - name: gpu\n                instance_type: g4dn.xlarge\n                min_instances: 1\n                max_instances: 2\n              - name: inferentia\n                instance_type: inf1.xlarge\n                min_instances: 1\n                max_instances: 2\n              - name: arm\n                instance_type: a1.large\n                min_instances: 1\n                max_instances: 2\n            EOF\n      - run:\n          name: Create/Update AWS User policy\n          command: python ./dev/create_user.py cortex-nightly $NIGHTLY_AWS_ACCOUNT_ID us-east-1 > $BASH_ENV\n      - run:\n          name: Wait for new keys to propagate in AWS\n          command: sleep 10\n      - run:\n          name: Verify configuration of credentials\n          command: aws sts get-caller-identity | jq \".Arn\" | grep \"dev-cortex-nightly-us-east-1\"\n      - run:\n          name: Create Cluster\n          command: cortex cluster up cluster.yaml --configure-env cortex -y\n      - slack/notify:\n          event: fail\n          channel: \"#builds\"\n          template: basic_fail_1\n\n  e2e-tests:\n    docker:\n      - image: cimg/python:3.7\n    steps:\n      - checkout\n      - run:\n          name: Install Dependencies\n          command: |\n            pip install boto3 pyyaml awscli\n            pip install -e ./test/e2e\n            pip install https://s3-us-west-2.amazonaws.com/get-cortex/master/python/cortex-master.tar.gz\n      - run:\n          name: Initialize Credentials\n          command: |\n            echo 'export AWS_ACCESS_KEY_ID=${NIGHTLY_AWS_ACCESS_KEY_ID}' >> $BASH_ENV\n            echo 'export AWS_SECRET_ACCESS_KEY=${NIGHTLY_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV\n      - run:\n          name: Configure Cortex CLI\n          command: cortex env configure cortex --operator-endpoint $(python dev/get_operator_url.py cortex-nightly us-east-1)\n      - run:\n          name: Run E2E Tests\n          no_output_timeout: 30m\n          command: |\n            pytest -v test/e2e/tests --env cortex --x86-nodegroups spot,cpu,gpu,inferentia --arm-nodegroups arm --skip-autoscaling --skip-load --skip-long-running\n            pytest -v test/e2e/tests --env cortex --x86-nodegroups spot,cpu,gpu,inferentia -k test_autoscaling\n            pytest -v test/e2e/tests --env cortex --x86-nodegroups spot,cpu,gpu,inferentia -k test_load\n      - slack/notify:\n          event: fail\n          channel: \"#builds\"\n          template: basic_fail_1\n\n  cluster-down:\n    docker:\n      - image: cimg/python:3.7\n    steps:\n      - setup_remote_docker\n      - checkout\n      - run:\n          name: Install Dependencies\n          command: |\n            pip install boto3 pyyaml awscli\n            pip install https://s3-us-west-2.amazonaws.com/get-cortex/master/python/cortex-master.tar.gz\n      - run:\n          name: Initialize Credentials\n          command: |\n            echo 'export AWS_ACCESS_KEY_ID=${NIGHTLY_AWS_ACCESS_KEY_ID}' >> $BASH_ENV\n            echo 'export AWS_SECRET_ACCESS_KEY=${NIGHTLY_AWS_SECRET_ACCESS_KEY}' >> $BASH_ENV\n      - run:\n          name: Create/Update AWS User policy\n          command: python ./dev/create_user.py cortex-nightly $NIGHTLY_AWS_ACCOUNT_ID us-east-1 > $BASH_ENV\n      - run:\n          name: Wait for new keys to propagate in AWS\n          command: sleep 10\n      - run:\n          name: Verify configuration of credentials\n          command: aws sts get-caller-identity | jq \".Arn\" | grep \"dev-cortex-nightly-us-east-1\"\n      - run:\n          name: Delete Cluster\n          command: cortex cluster down --name cortex-nightly --region us-east-1 -y\n          when: always\n      - slack/notify:\n          event: fail\n          channel: \"#builds\"\n          template: basic_fail_1\n\nworkflows:\n  build:\n    jobs:\n      - lint\n      - test\n      - build-and-deploy-approval:\n          type: approval\n          filters:\n            branches:\n              only:\n                - /^[0-9]+\\.[0-9]+$/\n      - build-and-upload-cli:\n          requires:\n            - lint\n            - test\n            - build-and-deploy-approval\n          filters:\n            branches:\n              only:\n                - master\n                - /^[0-9]+\\.[0-9]+$/\n      - build-and-push-images-amd64:\n          requires:\n            - lint\n            - test\n            - build-and-deploy-approval\n          filters:\n            branches:\n              only:\n                - master\n                - /^[0-9]+\\.[0-9]+$/\n      - build-and-push-images-arm64:\n          requires:\n            - lint\n            - test\n            - build-and-deploy-approval\n          filters:\n            branches:\n              only:\n                - master\n                - /^[0-9]+\\.[0-9]+$/\n      - amend-images:\n          requires:\n            - build-and-push-images-amd64\n            - build-and-push-images-arm64\n          filters:\n            branches:\n              only:\n                - master\n                - /^[0-9]+\\.[0-9]+$/\n\n  # nightly-cluster-up:\n  #   triggers:\n  #     - schedule:\n  #         cron: \"0 0 * * *\"\n  #         filters:\n  #           branches:\n  #             only:\n  #               - master\n  #   jobs:\n  #     - cluster-up\n\n  # nightly-e2e-tests:\n  #   triggers:\n  #     - schedule:\n  #         cron: \"0 1 * * *\"\n  #         filters:\n  #           branches:\n  #             only:\n  #               - master\n  #   jobs:\n  #     - e2e-tests\n\n  # nightly-cluster-down:\n  #   triggers:\n  #     - schedule:\n  #         cron: \"0 2 * * *\"\n  #         filters:\n  #           branches:\n  #             only:\n  #               - master\n  #   jobs:\n  #     - cluster-down\n"
  },
  {
    "path": ".dockerignore",
    "content": "/vendor/\n/bin/\n/testbin/\n/dev/\n/docs/\n/test/\n\n**/.*\n**/*.md\n**/*.zip\n\n**/*.pyc\n**/*.pyo\n**/*.pyd\n**/__pycache__/\n\n**/hack/\n**/PROJECT\n**/Makefile\n"
  },
  {
    "path": ".gitbook.yaml",
    "content": "root: ./docs/\n\nstructure:\n  readme: ./start.md\n  summary: summary.md\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/bug-report.md",
    "content": "---\nname: Bug report\nabout: Report a bug\ntitle: ''\nlabels: bug\nassignees: ''\n\n---\n\n### Version\n\n(use `cortex version` to determine your version)\n\n### Description\n\n(describe the bug)\n\n### Configuration\n\n(paste relevant `cortex.yaml` or `cluster.yaml` configuration)\n\n### Steps to reproduce\n\n1. ...\n2. ...\n3. ...\n\n### Expected behavior\n\n(describe the behavior you expected)\n\n### Actual behavior\n\n(describe the behavior you experienced)\n\n### Screenshots\n\n(optional)\n\n### Stack traces\n\n(error output from CloudWatch Insights or from a random pod `cortex logs <api name>`)\n\n```text\n<paste stack traces here>\n```\n\n### Additional context\n\n(optional)\n\n### Suggested solution\n\n(optional)\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/feature-request.md",
    "content": "---\nname: Feature request\nabout: Request a feature\ntitle: ''\nlabels: enhancement\nassignees: ''\n\n---\n\n### Description\n\n(describe the feature)\n\n### Motivation\n\n(how this will improve the product / what problem will this solve / what is the use case)\n\n### Additional context\n\n(optional)\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/question.md",
    "content": "---\nname: Question\nabout: Ask a question\ntitle: ''\nlabels: question\nassignees: ''\n\n---\n"
  },
  {
    "path": ".github/pull_request_template.md",
    "content": "closes #<issue ID>\n\n---\n\nchecklist:\n\n- [ ] run `make test` and `make lint`\n- [ ] test manually (i.e. build/push all images, restart operator, and re-deploy APIs)\n- [ ] update examples\n- [ ] update docs and add any new files to `summary.md` (view in [gitbook](https://cortex-labs.gitbook.io/staging/-MOmCGMADSRNQahK3Kox/) after merging)\n- [ ] cherry-pick into release branches if applicable\n- [ ] alert the dev team if the dev environment changed\n"
  },
  {
    "path": ".gitignore",
    "content": "/vendor/\n/bin/\n/testbin/\n/dev/config/\n\n# PYTHON\n__pycache__/\n*.py[cod]\n*$py.class\n.python-version\n.env\n.venv\n*.egg-info\n*.pytest_cache\n\n# OSX\n.DS_Store\n._*\n\n# MISC\n*.zip\n.unison*\n.vscode/\n\n# Binaries for programs and plugins\n*.exe\n*.exe~\n*.dll\n*.so\n*.dylib\n\n# Test binary, build with `go test -c`\n*.test\n\n# Kubernetes Generated files - skip generated files, except for vendored files\n\n!vendor/**/zz_generated.*\n\n# editor and IDE paraphernalia\n.idea\n*.swp\n*.swo\n*~\n"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "# Contributing\n\n## Remote development\n\nWe recommend that you run your development environment on an EC2 instance due to frequent docker registry pushing. We've had a good experience using [Mutagen](https://mutagen.io/documentation/introduction) to synchronize local / remote filesystems.\n\n## Prerequisites\n\n### System packages\n\nTo install the necessary system packages on Ubuntu, you can run these commands:\n\n```bash\nsudo apt-get update\nsudo apt install -y apt-transport-https ca-certificates software-properties-common gnupg-agent curl zip python3 python3-pip python3-dev build-essential jq tree\nsudo python3 -m pip install --upgrade pip setuptools boto3\n```\n\n### Go\n\nTo install Go on linux, run:\n\n```bash\nmkdir -p ~/bin && \\\nwget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz && \\\nsudo tar -xvf go1.17.3.linux-amd64.tar.gz && \\\nsudo mv go /usr/local && \\\nrm go1.17.3.linux-amd64.tar.gz && \\\necho 'export PATH=\"/usr/local/go/bin:$HOME/go/bin:$PATH\"' >> $HOME/.bashrc\n```\n\nAnd then log out and back in.\n\n### Docker\n\nTo install Docker on Ubuntu, run:\n\n```bash\ncurl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \\\nsudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable\" && \\\nsudo apt update && \\\nsudo apt install -y docker-ce docker-ce-cli containerd.io && \\\nsudo usermod -aG docker $USER\n```\n\nAnd then log out and back in. Then, bootstrap a buildx builder:\n\n```bash\ndocker buildx create --driver-opt image=moby/buildkit:master --name builder --platform linux/amd64,linux/arm64 --use\ndocker buildx inspect --bootstrap builder\n```\n\n### kubectl\n\nTo install kubectl on linux, run:\n\n```bash\ncurl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \\\nchmod +x ./kubectl && \\\nsudo mv ./kubectl /usr/local/bin/kubectl\n```\n\n### eksctl\n\nTo install eksctl run:\n\n```bash\ncurl --silent --location \"https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz\" | tar xz -C /tmp && \\\nsudo mv /tmp/eksctl /usr/local/bin\n```\n\n### aws-cli (v1)\n\nFollow [these instructions](https://github.com/aws/aws-cli#installation) to install aws-cli (v1).\n\nE.g. to install it globally, run:\n\n```bash\nsudo python3 -m pip install awscli\n\naws configure\n```\n\n## Cortex dev environment\n\n### Clone the repo\n\nClone the project:\n\n```bash\ngit clone https://github.com/cortexlabs/cortex.git\ncd cortex\n```\n\nRun the tests:\n\n```bash\nmake test\n```\n\n### Dev tools\n\nInstall development tools by running:\n\n```bash\nmake tools\n```\n\nAfter the dependencies are installed, there may be a diff in `go.mod` and `go.sum`, which you can revert.\n\nRun the linter:\n\n```bash\nmake lint\n```\n\nWe use `gofmt` for formatting Go files, `black` for Python files (line length = 100), and the VS Code [yaml extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) for YAML files. It is recommended to enable these in your code editor, but you can also run the Go and Python formatters from the terminal:\n\n```bash\nmake format\n\ngit diff  # there should be no diff\n```\n\n### Cluster configuration\n\nCreate a config directory in the repo's root directory:\n\n```bash\nmkdir dev/config\n```\n\nCreate `dev/config/env.sh` with the following information:\n\n```bash\n# dev/config/env.sh\n\nexport AWS_ACCOUNT_ID=\"***\"  # you can find your account ID in the AWS web console; here is an example: 764403040417\nexport AWS_REGION=\"***\"  # you can use any AWS region you'd like, e.g. \"us-west-2\"\nexport AWS_ACCESS_KEY_ID=\"***\"  # alternatively, you can remove this to use the default credentials chain on your machine\nexport AWS_SECRET_ACCESS_KEY=\"***\"  # alternatively, you can remove this to use the default credentials chain on your machine\nexport DEFAULT_USER_ARN=\"arn:aws:iam::<ACCOUNT_ID>:<AWS IAM ENTITY>\" # (e.g. arn:aws-us-gov:iam::123456789:user/foo)\n```\n\nCreate the ECR registries:\n\n```bash\nmake registry-create\n```\n\nCreate `dev/config/cluster.yaml`. Paste the following config, and update `region` and all registry URLs (replace `<account_id>` with your AWS account ID, and replace `<region>` with your region):\n\n```yaml\n# dev/config/cluster.yaml\n\ncluster_name: cortex\nregion: <region>  # e.g. us-west-2\n\nnode_groups:\n  - name: worker-ng\n    instance_type: m5.large\n    min_instances: 1\n    max_instances: 5\n```\n\n### Building\n\nAdd this to your bash profile (e.g. `~/.bash_profile`, `~/.profile` or `~/.bashrc`), replacing the placeholders accordingly:\n\n```bash\n# set the default image registry\nexport CORTEX_DEV_DEFAULT_IMAGE_REGISTRY=\"<account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs\"\n\n# enable api server monitoring in grafana\nexport CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD=\"true\"\n\n# redirect analytics and error reporting to our dev environment\nexport CORTEX_TELEMETRY_SENTRY_DSN=\"https://c334df915c014ffa93f2076769e5b334@sentry.io/1848098\"\nexport CORTEX_TELEMETRY_SEGMENT_WRITE_KEY=\"0WvoJyCey9z1W2EW7rYTPJUMRYat46dl\"\n\n# instruct the Python client to use your development CLI binary (update the path to point to your cortex repo)\nexport CORTEX_CLI_PATH=\"<cortex_repo_path>/bin/cortex\"\n\n# create a cortex alias which runs your development CLI\nalias cortex=\"$CORTEX_CLI_PATH\"\n```\n\nRefresh your bash profile:\n\n```bash\n. ~/.bash_profile  # or: `. ~/.bashrc`\n```\n\nBuild the Cortex CLI:\n\n```bash\nmake cli  # the binary will be placed in <path/to/cortex>/bin/cortex\ncortex version  # should show \"master\"\n```\n\nBuild and push all Cortex images:\n\n```bash\nmake images-all\n```\n\n## Dev workflow\n\nHere is the typical full dev workflow which covers most cases:\n\n1. `make cluster-up` (creates a cluster using `dev/config/cluster.yaml`)\n2. `make devstart` (deletes the in-cluster operator, builds the CLI, and starts the operator locally; file changes will trigger the CLI and operator to re-build)\n3. Make your changes\n4. `make images-dev` (only necessary if changes were made outside of the operator and CLI)\n5. Test your changes e.g. via `cortex deploy` (and repeat steps 3 and 4 as necessary)\n6. `make cluster-down` (deletes your cluster)\n\nIf you want to switch back to the in-cluster operator:\n\n1. `<ctrl+c>` to stop your local operator\n2. `make operator-start` to restart the operator in your cluster\n\n### Dev workflow optimizations\n\nIf you are only modifying the CLI, `make cli-watch` will build the CLI and re-build it when files are changed. When doing this, you can leave the operator running in the cluster instead of running it locally.\n\nIf you are only modifying the operator, `make operator-local` will build and start the operator locally, and build/restart it when files are changed.\n\nSee `Makefile` for additional dev commands.\n"
  },
  {
    "path": "LICENSE",
    "content": "                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n"
  },
  {
    "path": "Makefile",
    "content": "#!make\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nSHELL := /bin/bash\nexport BASH_ENV=./dev/config/env.sh\n\n# declare all targets as phony to avoid collisions with local files or folders\n.PHONY: $(MAKECMDGOALS)\n\n#######\n# Dev #\n#######\n\n# Cortex\n\n# build cli, start local operator, and watch for changes\ndevstart:\n\t@$(MAKE) operator-stop || true\n\t@./dev/operator_local.sh || true\n\ncli:\n\t@mkdir -p ./bin\n\t@go build -o ./bin/cortex ./cli\n\n# build cli and watch for changes\ncli-watch:\n\t@rerun -watch ./pkg ./cli -run sh -c \"clear && echo 'building cli...' && go build -o ./bin/cortex ./cli && clear && echo '\\033[1;32mCLI built\\033[0m'\" || true\n\n# start local operator and watch for changes\noperator-local:\n\t@$(MAKE) operator-stop || true\n\t@./dev/operator_local.sh --operator-only || true\n\n# start local operator and attach the delve debugger to it (in server mode)\noperator-local-dbg:\n\t@$(MAKE) operator-stop || true\n\t@./dev/operator_local.sh --debug || true\n\n# configure kubectl to point to the cluster specified in dev/config/cluster.yaml\nkubectl:\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && eksctl utils write-kubeconfig --cluster=\"$$CORTEX_CLUSTER_NAME\" --region=\"$$CORTEX_REGION\" --verbose=0 | (grep -v \"saved kubeconfig as\" || true); eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN\n\ncluster-up:\n\t@$(MAKE) images-all\n\t@$(MAKE) cli\n\t@kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster up ./dev/config/cluster.yaml --configure-env=\"$$CORTEX_CLUSTER_NAME\"; eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN\n\t@$(MAKE) kubectl\n\ncluster-up-y:\n\t@$(MAKE) images-all\n\t@$(MAKE) cli\n\t@kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster up ./dev/config/cluster.yaml --configure-env=\"$$CORTEX_CLUSTER_NAME\" --yes; eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN\n\t@$(MAKE) kubectl\n\ncluster-configure:\n\t@$(MAKE) images-manager-skip-push\n\t@$(MAKE) cli\n\t@kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster configure ./dev/config/cluster.yaml; eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN\n\t@$(MAKE) kubectl\n\ncluster-configure-y:\n\t@$(MAKE) images-manager-skip-push\n\t@$(MAKE) cli\n\t@kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster configure ./dev/config/cluster.yaml --yes; eksctl create iamidentitymapping --region $$CORTEX_REGION --cluster $$CORTEX_CLUSTER_NAME --arn $$DEFAULT_USER_ARN --group system:masters --username $$DEFAULT_USER_ARN\n\t@$(MAKE) kubectl\n\ncluster-down:\n\t@$(MAKE) images-manager-skip-push\n\t@$(MAKE) cli\n\t@kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster down --config=./dev/config/cluster.yaml\n\ncluster-down-y:\n\t@$(MAKE) images-manager-skip-push\n\t@$(MAKE) cli\n\t@kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster down --config=./dev/config/cluster.yaml --yes\n\ncluster-info:\n\t@$(MAKE) images-manager-skip-push\n\t@$(MAKE) cli\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eval $$(python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION) && sleep 10 && ./bin/cortex cluster info --config=./dev/config/cluster.yaml --configure-env=\"$$CORTEX_CLUSTER_NAME\" --yes\n\nupdate-credentials:\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && python3 ./dev/create_user.py $$CORTEX_CLUSTER_NAME $$AWS_ACCOUNT_ID $$CORTEX_REGION\n\n# stop the in-cluster operator\noperator-stop:\n\t@$(MAKE) kubectl\n\t@kubectl scale --namespace=default deployments/operator --replicas=0\n\n# start the in-cluster operator\noperator-start:\n\t@$(MAKE) kubectl\n\t@kubectl scale --namespace=default deployments/operator --replicas=1\n\t@operator_pod=$$(kubectl get pods -l workloadID=operator --namespace=default -o jsonpath='{.items[0].metadata.name}') && kubectl wait --for=condition=ready pod $$operator_pod --namespace=default\n\n# restart the in-cluster operator\noperator-restart:\n\t@$(MAKE) kubectl\n\t@kubectl delete pods -l workloadID=operator --namespace=default\n\t@operator_pod=$$(kubectl get pods -l workloadID=operator --namespace=default -o jsonpath='{.items[0].metadata.name}') && kubectl wait --for=condition=ready pod $$operator_pod --namespace=default\n\n# build and update the in-cluster operator\noperator-update:\n\t@$(MAKE) kubectl\n\t@kubectl scale --namespace=default deployments/operator --replicas=0\n\t@./dev/registry.sh update-single operator\n\t@kubectl scale --namespace=default deployments/operator --replicas=1\n\t@operator_pod=$$(kubectl get pods -l workloadID=operator --namespace=default -o jsonpath='{.items[0].metadata.name}') && kubectl wait --for=condition=ready pod $$operator_pod --namespace=default\n\n# restart all in-cluster async-gateways\nasync-gateway-restart:\n\t@$(MAKE) kubectl\n\t@kubectl delete pods -l cortex.dev/async=gateway --namespace=default\n\n# build and update all in-cluster async-gateways\nasync-gateway-update:\n\t@$(MAKE) kubectl\n\t@./dev/registry.sh update-single async-gateway\n\t@kubectl delete pods -l cortex.dev/async=gateway --namespace=default\n\n# docker images\nimages-all:\n\t@./dev/registry.sh update all\nimages-all-multi-arch:\n\t@./dev/registry.sh update all --include-arm64-arch\nimages-all-skip-push:\n\t@./dev/registry.sh update all --skip-push\n\nimages-dev:\n\t@./dev/registry.sh update dev\nimages-dev-multi-arch:\n\t@./dev/registry.sh update dev --include-arm64-arch\nimages-dev-skip-push:\n\t@./dev/registry.sh update dev --skip-push\n\nimages-manager-skip-push:\n\t@./dev/registry.sh update-single manager --skip-push\n\nimages-clean-cache:\n\t@./dev/registry.sh clean-cache\n\nregistry-create:\n\t@./dev/registry.sh create\n\nregistry-clean:\n\t@./dev/registry.sh clean\n\n# Misc\n\ntools:\n\t@go get -u -v golang.org/x/lint/golint\n\t@go get -u -v github.com/kyoh86/looppointer/cmd/looppointer\n\t@go get -u -v github.com/VojtechVitek/rerun/cmd/rerun\n\t@go get -u -v github.com/go-delve/delve/cmd/dlv\n\t@python3 -m pip install aiohttp boto3 pyyaml pydoc-markdown==3.* black==20.8b1 -U\n\t@python3 -m pip install -e test/e2e\n\nformat:\n\t@./dev/format.sh\n\n#########\n# Tests #\n#########\n\n# build test api images\n# make sure you login with your quay credentials\nbuild-test-api-images:\n\t@./test/utils/build-all.sh quay.io/cortexlabs-test\n\ntest:\n\t@./build/test.sh go\n\n# run e2e tests on an existing cluster\n# read test/e2e/README.md for instructions first\ntest-e2e:\n\t@$(MAKE) images-all\n\t@$(MAKE) operator-restart\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && CORTEX_CLI_PATH=\"$$(pwd)/bin/cortex\" ./build/test.sh e2e -e \"$$CORTEX_CLUSTER_NAME\"\n\n# run e2e tests with a new cluster\n# read test/e2e/README.md for instructions first\ntest-e2e-new:\n\t@$(MAKE) images-all\n\t@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && CORTEX_CLI_PATH=\"$$(pwd)/bin/cortex\" ./build/test.sh e2e \"$$(pwd)/dev/config/cluster.yaml\" --create-cluster\n\nlint:\n\t@./build/lint.sh\n\n# this is a subset of lint.sh, and is only meant to be run on master\nlint-docs:\n\t@./build/lint-docs.sh\n\n###############\n# CI Commands #\n###############\n\nci-build-images-amd64:\n\t@./build/build-images.sh amd64 quay.io docker.io\n\nci-build-images-arm64:\n\t@./build/build-images.sh arm64 quay.io docker.io\n\nci-push-images-amd64:\n\t@./build/push-images.sh amd64 quay.io docker.io\n\nci-push-images-arm64:\n\t@./build/push-images.sh arm64 quay.io docker.io\n\nci-amend-images:\n\t@./build/amend-images.sh quay.io docker.io\n\nci-build-cli:\n\t@./build/cli.sh\n\nci-build-and-upload-cli:\n\t@./build/cli.sh upload\n"
  },
  {
    "path": "README.md",
    "content": "**[Docs](https://docs.cortexlabs.com)** • **[Slack](https://community.cortexlabs.com)**\n\n<br>\n\n<img src='https://cortex-public.s3.us-west-2.amazonaws.com/logo.png' height='32'>\n\n<br>\n\nNote: This project is no longer actively maintained by its original authors.\n\n# Production infrastructure for machine learning at scale\n\nDeploy, manage, and scale machine learning models in production.\n\n<br>\n\n## Serverless workloads\n\n**Realtime** - respond to requests in real-time and autoscale based on in-flight request volumes.\n\n**Async** - process requests asynchronously and autoscale based on request queue length.\n\n**Batch** - run distributed and fault-tolerant batch processing jobs on-demand.\n\n<br>\n\n## Automated cluster management\n\n**Autoscaling** - elastically scale clusters with CPU and GPU instances.\n\n**Spot instances** - run workloads on spot instances with automated on-demand backups.\n\n**Environments** - create multiple clusters with different configurations.\n\n<br>\n\n## CI/CD and observability integrations\n\n**Provisioning** - provision clusters with declarative configuration or a Terraform provider.\n\n**Metrics** - send metrics to any monitoring tool or use pre-built Grafana dashboards.\n\n**Logs** - stream logs to any log management tool or use the pre-built CloudWatch integration.\n\n<br>\n\n## Built for AWS\n\n**EKS** - Cortex runs on top of EKS to scale workloads reliably and cost-effectively.\n\n**VPC** - deploy clusters into a VPC on your AWS account to keep your data private.\n\n**IAM** - integrate with IAM for authentication and authorization workflows.\n"
  },
  {
    "path": "build/amend-image.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nCORTEX_VERSION=master\n\nhost_primary=$1\nhost_backup=$2\nimage=$3\n\nhosts=(\n    \"$host_primary\"\n    \"$host_backup\"\n)\necho \"$DOCKER_PASSWORD\" | docker login -u \"$DOCKER_USERNAME\" --password-stdin\n\nfor host in \"${hosts[@]}\"; do\n    docker manifest create $host/cortexlabs/${image}:${CORTEX_VERSION} \\\n        -a $host/cortexlabs/${image}:manifest-${CORTEX_VERSION}-amd64 \\\n        -a $host/cortexlabs/${image}:manifest-${CORTEX_VERSION}-arm64\n    docker manifest push $host/cortexlabs/${image}:${CORTEX_VERSION}\ndone\n"
  },
  {
    "path": "build/amend-images.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nsource $ROOT/build/images.sh\nsource $ROOT/dev/util.sh\n\nhost_primary=$1\nhost_backup=$2\n\nfor image in \"${multi_arch_images[@]}\"; do\n    $ROOT/build/amend-image.sh $host_primary $host_backup $image\ndone\n"
  },
  {
    "path": "build/build-image.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nCORTEX_VERSION=master\n\nhost_primary=$1\nhost_backup=$2\nimage=$3\nis_multi_arch=$4\narch=$5\n\nif [ \"$is_multi_arch\" = \"true\" ]; then\n  tag=\"manifest-${CORTEX_VERSION}-$arch\"\nelse\n  tag=\"${CORTEX_VERSION}\"\nfi\n\ndocker build $ROOT \\\n  --build-arg TARGETOS=linux \\\n  --build-arg TARGETARCH=$arch \\\n  -f $ROOT/images/$image/Dockerfile \\\n  -t $host_primary/cortexlabs/${image}:${tag} \\\n  -t $host_backup/cortexlabs/${image}:${tag}\n"
  },
  {
    "path": "build/build-images.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nsource $ROOT/build/images.sh\nsource $ROOT/dev/util.sh\n\narch=$1\nhost_primary=$2\nhost_backup=$3\n\nfor image in \"${all_images[@]}\"; do\n  is_multi_arch=\"false\"\n  if [[ \" ${multi_arch_images[*]} \" =~ \" $image \" ]]; then\n    is_multi_arch=\"true\"\n    $ROOT/build/build-image.sh $host_primary $host_backup $image $is_multi_arch $arch\n  elif [ \"$arch\" = \"amd64\" ]; then\n    $ROOT/build/build-image.sh $host_primary $host_backup $image $is_multi_arch $arch\n  fi\ndone\n"
  },
  {
    "path": "build/cli.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nCORTEX_VERSION=master\n\narg1=${1:-\"\"}\nupload=\"false\"\nif [ \"$arg1\" == \"upload\" ]; then\n  upload=\"true\"\nfi\n\nfunction build_and_upload() {\n  set -euo pipefail\n\n  os=$1\n  echo -e \"\\nBuilding Cortex CLI for $os\"\n  GOOS=$os GOARCH=amd64 CGO_ENABLED=0 go build -o cortex \"$ROOT/cli\"\n  if [ \"$upload\" == \"true\" ]; then\n    echo \"Uploading Cortex CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex\"\n    aws s3 cp cortex s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex --only-show-errors\n\n    zip cortex.zip cortex\n    echo \"Uploading zipped Cortex CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex.zip\"\n    aws s3 cp cortex.zip s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex.zip --only-show-errors\n    rm cortex.zip\n  fi\n  echo \"Done ✓\"\n  rm cortex\n}\n\nfunction build_python {\n  pushd $ROOT/python/client\n  python setup.py sdist\n\n  if [ \"$upload\" == \"true\" ]; then\n    echo \"Uploading Cortex CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/python/cortex-$CORTEX_VERSION.tar.gz\"\n    aws s3 cp dist/cortex-$CORTEX_VERSION.tar.gz s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/python/cortex-$CORTEX_VERSION.tar.gz\n  fi\n\n  rm -rf dist/\n  rm -rf cortex.egg-info/\n\n  popd\n}\n\nbuild_and_upload darwin\n\nbuild_and_upload linux\n\nbuild_python\n"
  },
  {
    "path": "build/generate_ami_mapping.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"log\"\n\t\"os\"\n\t\"sort\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/ec2\"\n\t\"github.com/aws/aws-sdk-go/service/ec2/ec2iface\"\n)\n\n// run with `go run build/generate_ami_mapping.go manager/manifests/ami.json`\n// copied from https://github.com/weaveworks/eksctl/blob/c211e68d3c8cf3c7f800768bfa0251dda17e011c/pkg/apis/eksctl.io/v1alpha5/types.go\n// most of this code can be removed once eksctl can be imported: https://github.com/weaveworks/eksctl/issues/813\nconst (\n\teksResourceAccountStandard = \"602401143452\"\n\n\t// eksResourceAccountAPEast1 defines the AWS EKS account ID that provides node resources in ap-east-1 region\n\teksResourceAccountAPEast1 = \"800184023465\"\n\n\t// eksResourceAccountMESouth1 defines the AWS EKS account ID that provides node resources in me-south-1 region\n\teksResourceAccountMESouth1 = \"558608220178\"\n\n\t// eksResourceAccountCNNorthWest1 defines the AWS EKS account ID that provides node resources in cn-northwest-1 region\n\teksResourceAccountCNNorthWest1 = \"961992271922\"\n\n\t// eksResourceAccountCNNorth1 defines the AWS EKS account ID that provides node resources in cn-north-1\n\teksResourceAccountCNNorth1 = \"918309763551\"\n\n\t// eksResourceAccountAFSouth1 defines the AWS EKS account ID that provides node resources in af-south-1\n\teksResourceAccountAFSouth1 = \"877085696533\"\n\n\t// eksResourceAccountEUSouth1 defines the AWS EKS account ID that provides node resources in eu-south-1\n\teksResourceAccountEUSouth1 = \"590381155156\"\n\n\t// eksResourceAccountUSGovWest1 defines the AWS EKS account ID that provides node resources in us-gov-west-1\n\teksResourceAccountUSGovWest1 = \"013241004608\"\n\n\t// eksResourceAccountUSGovEast1 defines the AWS EKS account ID that provides node resources in us-gov-east-1\n\teksResourceAccountUSGovEast1 = \"151742754352\"\n)\n\n// Regions\nconst (\n\t// RegionUSWest1 represents the US West Region North California\n\tRegionUSWest1 = \"us-west-1\"\n\n\t// RegionUSWest2 represents the US West Region Oregon\n\tRegionUSWest2 = \"us-west-2\"\n\n\t// RegionUSEast1 represents the US East Region North Virginia\n\tRegionUSEast1 = \"us-east-1\"\n\n\t// RegionUSEast2 represents the US East Region Ohio\n\tRegionUSEast2 = \"us-east-2\"\n\n\t// RegionCACentral1 represents the Canada Central Region\n\tRegionCACentral1 = \"ca-central-1\"\n\n\t// RegionEUWest1 represents the EU West Region Ireland\n\tRegionEUWest1 = \"eu-west-1\"\n\n\t// RegionEUWest2 represents the EU West Region London\n\tRegionEUWest2 = \"eu-west-2\"\n\n\t// RegionEUWest3 represents the EU West Region Paris\n\tRegionEUWest3 = \"eu-west-3\"\n\n\t// RegionEUNorth1 represents the EU North Region Stockholm\n\tRegionEUNorth1 = \"eu-north-1\"\n\n\t// RegionEUCentral1 represents the EU Central Region Frankfurt\n\tRegionEUCentral1 = \"eu-central-1\"\n\n\t// RegionEUSouth1 represents te Eu South Region Milan\n\tRegionEUSouth1 = \"eu-south-1\"\n\n\t// RegionAPNorthEast1 represents the Asia-Pacific North East Region Tokyo\n\tRegionAPNorthEast1 = \"ap-northeast-1\"\n\n\t// RegionAPNorthEast2 represents the Asia-Pacific North East Region Seoul\n\tRegionAPNorthEast2 = \"ap-northeast-2\"\n\n\t// RegionAPNorthEast3 represents the Asia-Pacific North East region Osaka\n\tRegionAPNorthEast3 = \"ap-northeast-3\"\n\n\t// RegionAPSouthEast1 represents the Asia-Pacific South East Region Singapore\n\tRegionAPSouthEast1 = \"ap-southeast-1\"\n\n\t// RegionAPSouthEast2 represents the Asia-Pacific South East Region Sydney\n\tRegionAPSouthEast2 = \"ap-southeast-2\"\n\n\t// RegionAPSouth1 represents the Asia-Pacific South Region Mumbai\n\tRegionAPSouth1 = \"ap-south-1\"\n\n\t// RegionAPEast1 represents the Asia Pacific Region Hong Kong\n\tRegionAPEast1 = \"ap-east-1\"\n\n\t// RegionMESouth1 represents the Middle East Region Bahrain\n\tRegionMESouth1 = \"me-south-1\"\n\n\t// RegionSAEast1 represents the South America Region Sao Paulo\n\tRegionSAEast1 = \"sa-east-1\"\n\n\t// RegionAFSouth1 represents the Africa Region Cape Town\n\tRegionAFSouth1 = \"af-south-1\"\n\n\t// RegionCNNorthwest1 represents the China region Ningxia\n\tRegionCNNorthwest1 = \"cn-northwest-1\"\n\n\t// RegionCNNorth1 represents the China region Beijing\n\tRegionCNNorth1 = \"cn-north-1\"\n\n\t// RegionUSGovWest1 represents the region GovCloud (US-West)\n\tRegionUSGovWest1 = \"us-gov-west-1\"\n\n\t// RegionUSGovEast1 represents the region GovCloud (US-East)\n\tRegionUSGovEast1 = \"us-gov-east-1\"\n\n\t// DefaultRegion defines the default region, where to deploy the EKS cluster\n\tDefaultRegion = RegionUSWest2\n)\n\n// SupportedRegions are the regions where EKS is available\nfunc SupportedRegions() []string {\n\treturn []string{\n\t\tRegionUSWest1,\n\t\tRegionUSWest2,\n\t\tRegionUSEast1,\n\t\tRegionUSEast2,\n\t\tRegionCACentral1,\n\t\tRegionEUWest1,\n\t\tRegionEUWest2,\n\t\tRegionEUWest3,\n\t\tRegionEUNorth1,\n\t\tRegionEUCentral1,\n\t\tRegionEUSouth1,\n\t\tRegionAPNorthEast1,\n\t\tRegionAPNorthEast2,\n\t\tRegionAPNorthEast3,\n\t\tRegionAPSouthEast1,\n\t\tRegionAPSouthEast2,\n\t\tRegionAPSouth1,\n\t\tRegionAPEast1,\n\t\tRegionMESouth1,\n\t\tRegionSAEast1,\n\t\tRegionAFSouth1,\n\t\tRegionUSGovWest1,\n\t\tRegionUSGovEast1,\n\t\t// RegionCNNorthwest1,\n\t\t// RegionCNNorth1,\n\t}\n}\n\nfunc EKSResourceAccountID(region string) string {\n\tswitch region {\n\tcase RegionAPEast1:\n\t\treturn eksResourceAccountAPEast1\n\tcase RegionMESouth1:\n\t\treturn eksResourceAccountMESouth1\n\tcase RegionCNNorthwest1:\n\t\treturn eksResourceAccountCNNorthWest1\n\tcase RegionCNNorth1:\n\t\treturn eksResourceAccountCNNorth1\n\tcase RegionUSGovWest1:\n\t\treturn eksResourceAccountUSGovWest1\n\tcase RegionUSGovEast1:\n\t\treturn eksResourceAccountUSGovEast1\n\tcase RegionAFSouth1:\n\t\treturn eksResourceAccountAFSouth1\n\tcase RegionEUSouth1:\n\t\treturn eksResourceAccountEUSouth1\n\tdefault:\n\t\treturn eksResourceAccountStandard\n\t}\n}\n\nfunc main() {\n\tif len(os.Args) > 3 {\n\t\tfmt.Println(\"usage: go run generate_ami_mapping.go <abs_dest_path> public|govcloud\")\n\t\tos.Exit(1)\n\t}\n\n\tdestFile := os.Args[1]\n\tcloudType := os.Args[2]\n\n\tif cloudType != \"public\" && cloudType != \"govcloud\" {\n\t\tlog.Fatalf(\"%s is not a valid value; specify public or govcloud\", cloudType)\n\t}\n\n\tk8sVersionMap := map[string]map[string]map[string]string{}\n\n\tif _, err := os.Stat(destFile); !os.IsNotExist(err) {\n\t\tjsonBytes, err := ioutil.ReadFile(destFile)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err.Error())\n\t\t}\n\t\tjson.Unmarshal(jsonBytes, &k8sVersionMap)\n\t}\n\n\tk8sVersion := \"1.22\"\n\n\tif k8sVersionMap[k8sVersion] == nil {\n\t\tk8sVersionMap[k8sVersion] = map[string]map[string]string{}\n\t}\n\tfor _, region := range SupportedRegions() {\n\t\tif (cloudType == \"govcloud\") != (region == RegionUSGovEast1 || region == RegionUSGovWest1) {\n\t\t\t// cloudType == \"govcloud\" xor (region is us govclouds)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Print(region)\n\t\tsess := session.New(&aws.Config{Region: aws.String(region)})\n\t\tsvc := ec2.New(sess)\n\t\tcpuAmd64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf(\"amazon-eks-node-%s-v*\", k8sVersion))\n\t\tif err != nil {\n\t\t\tlog.Fatal(err.Error())\n\t\t}\n\t\tcpuArm64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf(\"amazon-eks-arm64-node-%s-v*\", k8sVersion))\n\t\tif err != nil {\n\t\t\tlog.Fatal(err.Error())\n\t\t}\n\t\tacceleratedAmd64AMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf(\"amazon-eks-gpu-node-%s-v*\", k8sVersion))\n\t\tif err != nil {\n\t\t\tlog.Fatal(err.Error())\n\t\t}\n\n\t\tif k8sVersionMap[k8sVersion][region] == nil {\n\t\t\tk8sVersionMap[k8sVersion][region] = map[string]string{}\n\t\t}\n\t\tk8sVersionMap[k8sVersion][region] = map[string]string{\n\t\t\t\"cpu_amd64\":         cpuAmd64AMI,\n\t\t\t\"cpu_arm64\":         cpuArm64AMI,\n\t\t\t\"accelerated_amd64\": acceleratedAmd64AMI,\n\t\t}\n\t\tfmt.Println(\" ✓\")\n\t}\n\n\tmarshalledBytes, err := json.MarshalIndent(k8sVersionMap, \"\", \"\\t\")\n\tif err != nil {\n\t\tlog.Fatal(err.Error())\n\t}\n\n\tmarshalledBytes = append(marshalledBytes, []byte(\"\\n\")...)\n\n\terr = ioutil.WriteFile(destFile, marshalledBytes, 0664)\n\tif err != nil {\n\t\tlog.Fatal(err.Error())\n\t}\n}\n\nfunc FindImage(ec2api ec2iface.EC2API, ownerAccount, namePattern string) (string, error) {\n\tinput := &ec2.DescribeImagesInput{\n\t\tOwners: []*string{&ownerAccount},\n\t\tFilters: []*ec2.Filter{\n\t\t\t{\n\t\t\t\tName:   aws.String(\"name\"),\n\t\t\t\tValues: []*string{&namePattern},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:   aws.String(\"virtualization-type\"),\n\t\t\t\tValues: []*string{aws.String(\"hvm\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:   aws.String(\"root-device-type\"),\n\t\t\t\tValues: []*string{aws.String(\"ebs\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:   aws.String(\"is-public\"),\n\t\t\t\tValues: []*string{aws.String(\"true\")},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:   aws.String(\"state\"),\n\t\t\t\tValues: []*string{aws.String(\"available\")},\n\t\t\t},\n\t\t},\n\t}\n\n\toutput, err := ec2api.DescribeImages(input)\n\tif err != nil {\n\t\treturn \"\", errors.Wrapf(err, \"error querying AWS for images\")\n\t}\n\n\tif len(output.Images) < 1 {\n\t\treturn \"\", nil\n\t}\n\n\tif len(output.Images) == 1 {\n\t\treturn *output.Images[0].ImageId, nil\n\t}\n\n\t// Sort images so newest is first\n\tsort.Slice(output.Images, func(i, j int) bool {\n\t\t//nolint:gosec\n\t\tcreationLeft, _ := time.Parse(time.RFC3339, *output.Images[i].CreationDate)\n\t\t//nolint:gosec\n\t\tcreationRight, _ := time.Parse(time.RFC3339, *output.Images[j].CreationDate)\n\t\treturn creationLeft.After(creationRight)\n\t})\n\n\treturn *output.Images[0].ImageId, nil\n}\n"
  },
  {
    "path": "build/images.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# images to build/push for development and CI commands\n# each image should appear exactly once on this page\n\nset -euo pipefail\n\ndev_images=(\n  \"manager\"\n  \"proxy\"\n  \"async-gateway\"\n  \"enqueuer\"\n  \"dequeuer\"\n  \"autoscaler\"\n  \"activator\"\n)\n\nnon_dev_images=(\n  \"cluster-autoscaler\"\n  \"operator\"\n  \"controller-manager\"\n  \"istio-proxy\"\n  \"istio-pilot\"\n  \"fluent-bit\"\n  \"prometheus\"\n  \"prometheus-config-reloader\"\n  \"prometheus-operator\"\n  \"prometheus-statsd-exporter\"\n  \"prometheus-dcgm-exporter\"\n  \"prometheus-kube-state-metrics\"\n  \"prometheus-node-exporter\"\n  \"kube-rbac-proxy\"\n  \"grafana\"\n  \"event-exporter\"\n  \"metrics-server\"\n  \"nvidia-device-plugin\"\n  \"neuron-device-plugin\"\n  \"neuron-scheduler\"\n  \"kubexit\"\n)\n\n# for linux/amd64 and linux/arm64\nmulti_arch_images=(\n  \"proxy\"\n  \"async-gateway\"\n  \"enqueuer\"\n  \"dequeuer\"\n  \"fluent-bit\"\n  \"prometheus-node-exporter\"\n  \"kube-rbac-proxy\"\n  \"kubexit\"\n)\n\nall_images=(\n  \"${dev_images[@]}\"\n  \"${non_dev_images[@]}\"\n)\n"
  },
  {
    "path": "build/lint-docs.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# This is a subset of lint.sh, and is only meant to be run on master\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\n# Check docs links\noutput=$(python3 $ROOT/dev/find_missing_docs_links.py)\nif [[ $output ]]; then\n  echo \"docs file(s) have broken links:\"\n  echo \"$output\"\n  exit 1\nfi\n\n# Check for trailing whitespace\noutput=$(cd \"$ROOT/docs\" && find . -type f \\\n-exec egrep -l \" +$\" {} \\;)\nif [[ $output ]]; then\n  echo \"File(s) have lines with trailing whitespace:\"\n  echo \"$output\"\n  exit 1\nfi\n\n# Check for missing new line at end of file\noutput=$(cd \"$ROOT/docs\" && find . -type f \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(tail -c 1 \"$0\")\" && echo \"No new line at end of $0\"' || true)\nif [[ $output ]]; then\n  echo \"$output\"\n  exit 1\nfi\n\n# Check for multiple new lines at end of file\noutput=$(cd \"$ROOT/docs\" && find . -type f \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(tail -c 2 \"$0\")\" || echo \"Multiple new lines at end of $0\"' || true)\nif [[ $output ]]; then\n  echo \"$output\"\n  exit 1\nfi\n\n# Check for new line(s) at beginning of file\noutput=$(cd \"$ROOT/docs\" && find . -type f \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(head -c 1 \"$0\")\" || echo \"New line at beginning of $0\"' || true)\nif [[ $output ]]; then\n  echo \"$output\"\n  exit 1\nfi\n"
  },
  {
    "path": "build/lint.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\ngit_branch=\"${CIRCLE_BRANCH:-\"\"}\"\nif [ \"$git_branch\" = \"\" ]; then\n  git_branch=$(cd \"$ROOT\" && git rev-parse --abbrev-ref HEAD)\nfi\n\nis_release_branch=\"false\"\nif echo \"$git_branch\" | grep -Eq ^[0-9]+.[0-9]+$; then\n  is_release_branch=\"true\"\nfi\n\nif ! command -v golint >/dev/null 2>&1; then\n  echo \"golint must be installed\"\n  exit 1\nfi\n\nif ! command -v looppointer >/dev/null 2>&1; then\n  echo \"looppointer must be installed\"\n  exit 1\nfi\n\nif ! command -v gofmt >/dev/null 2>&1; then\n  echo \"gofmt must be installed\"\n  exit 1\nfi\n\nif ! command -v black >/dev/null 2>&1; then\n  echo \"black must be installed\"\n  exit 1\nfi\n\ngo mod tidy\ngo vet \"$ROOT/...\"\n\noutput=$(golint \"$ROOT/...\" | grep -v \"comment\" || true)\nif [[ $output ]]; then\n  echo \"$output\"\n  exit 1\nfi\n\noutput=$(looppointer \"$ROOT/...\")\nif [[ $output ]]; then\n  echo \"$output\"\n  exit 1\nfi\n\noutput=$(gofmt -s -l \"$ROOT\")\nif [[ $output ]]; then\n  echo \"go files not properly formatted:\"\n  echo \"$output\"\n  exit 1\nfi\n\noutput=$(black --quiet --diff --line-length=100 \"$ROOT\")\nif [[ $output ]]; then\n  echo \"python files not properly formatted:\"\n  echo \"$output\"\n  black --version\n  exit 1\nfi\n\n# Check for missing license\noutput=$(cd \"$ROOT\" && find . -type f \\\n! -path \"./vendor/*\" \\\n! -path \"**/.vscode/*\" \\\n! -path \"**/.idea/*\" \\\n! -path \"**/.history/*\" \\\n! -path \"**/testbin/*\" \\\n! -path \"**/__pycache__/*\" \\\n! -path \"**/.pytest_cache/*\" \\\n! -path \"**/*.egg-info/*\" \\\n! -path \"./test/*\" \\\n! -path \"./dev/config/*\" \\\n! -path \"**/bin/*\" \\\n! -path \"./.circleci/*\" \\\n! -path \"./.git/*\" \\\n! -path \"./pkg/crds/config/*\" \\\n! -path \"**/tmp/*\" \\\n! -name LICENSE \\\n! -name \"*requirements.txt\" \\\n! -name \"go.*\" \\\n! -name \"*.md\" \\\n! -name \"*.json\" \\\n! -name \".*\" \\\n! -name \"*.bin\" \\\n! -name \"Dockerfile\" \\\n! -name \"PROJECT\" \\\n-exec grep -L \"Copyright 2022 Cortex Labs, Inc\" {} \\;)\nif [[ $output ]]; then\n  echo \"File(s) are missing Cortex license:\"\n  echo \"$output\"\n  exit 1\nfi\n\nif [ \"$is_release_branch\" = \"true\" ]; then\n  # Check for occurrences of \"master\" which should be changed to the version number\n  output=$(cd \"$ROOT\" && find . -type f \\\n  ! -path \"./build/lint.sh\" \\\n  ! -path \"./vendor/*\" \\\n  ! -path \"**/.vscode/*\" \\\n  ! -path \"**/.idea/*\" \\\n  ! -path \"**/.history/*\" \\\n  ! -path \"**/testbin/*\" \\\n  ! -path \"**/__pycache__/*\" \\\n  ! -path \"**/.pytest_cache/*\" \\\n  ! -path \"**/*.egg-info/*\" \\\n  ! -path \"./dev/config/*\" \\\n  ! -path \"**/bin/*\" \\\n  ! -path \"./.git/*\" \\\n  ! -name \".*\" \\\n  ! -name \"*.bin\" \\\n  -exec grep -R -A 5 -e \"CORTEX_VERSION\" {} \\;)\n  output=$(echo \"$output\" | grep -e \"master\" || true)\n  if [[ $output ]]; then\n    echo 'occurrences of \"master\" which should be changed to the version number:'\n    echo \"$output\"\n    exit 1\n  fi\nfi\n\n# Check for trailing whitespace\noutput=$(cd \"$ROOT\" && find . -type f \\\n! -path \"./vendor/*\" \\\n! -path \"**/.idea/*\" \\\n! -path \"**/.history/*\" \\\n! -path \"**/.vscode/*\" \\\n! -path \"**/testbin/*\" \\\n! -path \"**/__pycache__/*\" \\\n! -path \"**/.pytest_cache/*\" \\\n! -path \"**/*.egg-info/*\" \\\n! -path \"./dev/config/*\" \\\n! -path \"**/bin/*\" \\\n! -path \"./.git/*\" \\\n! -path \"./pkg/crds/config/*\" \\\n! -name \".*\" \\\n! -name \"*.bin\" \\\n! -name \"*.wav\" \\\n-exec egrep -l \" +$\" {} \\;)\nif [[ $output ]]; then\n  echo \"File(s) have lines with trailing whitespace:\"\n  echo \"$output\"\n  exit 1\nfi\n\n# Check for missing new line at end of file\noutput=$(cd \"$ROOT\" && find . -type f \\\n! -path \"./vendor/*\" \\\n! -path \"**/.idea/*\" \\\n! -path \"**/.history/*\" \\\n! -path \"**/.vscode/*\" \\\n! -path \"**/testbin/*\" \\\n! -path \"**/__pycache__/*\" \\\n! -path \"**/.pytest_cache/*\" \\\n! -path \"**/*.egg-info/*\" \\\n! -path \"./dev/config/*\" \\\n! -path \"./pkg/crds/config/*\" \\\n! -path \"**/bin/*\" \\\n! -path \"./.git/*\" \\\n! -name \".*\" \\\n! -name \"*.bin\" \\\n! -name \"*.wav\" \\\n! -name \"*.json\" \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(tail -c 1 \"$0\")\" && echo \"No new line at end of $0\"' || true)\nif [[ $output ]]; then\n  echo \"$output\"\n  exit 1\nfi\n\n# Check for multiple new lines at end of file\noutput=$(cd \"$ROOT\" && find . -type f \\\n! -path \"./vendor/*\" \\\n! -path \"**/.vscode/*\" \\\n! -path \"**/.idea/*\" \\\n! -path \"**/.history/*\" \\\n! -path \"**/testbin/*\" \\\n! -path \"**/__pycache__/*\" \\\n! -path \"**/.pytest_cache/*\" \\\n! -path \"**/*.egg-info/*\" \\\n! -path \"./dev/config/*\" \\\n! -path \"**/bin/*\" \\\n! -path \"./.git/*\" \\\n! -name \".*\" \\\n! -name \"*.bin\" \\\n! -name \"*.wav\" \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(tail -c 2 \"$0\")\" || [ ! -s \"$0\" ] || echo \"Multiple new lines at end of $0\"' || true)\nif [[ $output ]]; then\n  echo \"$output\"\n  exit 1\nfi\n\n# Check for new line(s) at beginning of file\noutput=$(cd \"$ROOT\" && find . -type f \\\n! -path \"./vendor/*\" \\\n! -path \"**/.idea/*\" \\\n! -path \"**/.history/*\" \\\n! -path \"**/.vscode/*\" \\\n! -path \"**/testbin/*\" \\\n! -path \"**/__pycache__/*\" \\\n! -path \"**/.pytest_cache/*\" \\\n! -path \"**/*.egg-info/*\" \\\n! -path \"./dev/config/*\" \\\n! -path \"./pkg/crds/config/*\" \\\n! -path \"./bin/*\" \\\n! -path \"./.git/*\" \\\n! -name \".*\" \\\n! -name \"*.bin\" \\\n! -name \"*.wav\" \\\n! -name \"boilerplate.go.txt\" \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(head -c 1 \"$0\")\" || [ ! -s \"$0\" ] || echo \"New line at beginning of $0\"' || true)\nif [[ $output ]]; then\n  echo \"$output\"\n  exit 1\nfi\n\n# Check that minimum_aws_policy.json is in-sync with docs\noutput=$(python3 -c \"\nimport sys\npolicy=open('./dev/minimum_aws_policy.json').read()\ndoc=open('./docs/clusters/management/auth.md').read()\nprint(policy in doc)\")\nif [[ \"$output\" != \"True\" ]]; then\n  echo \"./dev/minimum_aws_policy.json and the policy in ./docs/clusters/management/auth.md are out of sync\"\n  exit 1\nfi\n\n# Check docs links\noutput=$(python3 $ROOT/dev/find_missing_docs_links.py)\nif [[ $output ]]; then\n  echo \"docs file(s) have broken links:\"\n  echo \"$output\"\n  exit 1\nfi\n"
  },
  {
    "path": "build/push-image.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nCORTEX_VERSION=master\n\nhost_primary=$1\nhost_backup=$2\nimage=$3\nis_multi_arch=$4\narch=$5\n\necho \"$DOCKER_PASSWORD\" | docker login -u \"$DOCKER_USERNAME\" --password-stdin\nif [ \"$is_multi_arch\" = \"true\" ]; then\n  tag=\"manifest-${CORTEX_VERSION}-$arch\"\nelse\n  tag=\"${CORTEX_VERSION}\"\nfi\n\ndocker push $host_primary/cortexlabs/${image}:${tag}\ndocker push $host_backup/cortexlabs/${image}:${tag}\n"
  },
  {
    "path": "build/push-images.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nsource $ROOT/build/images.sh\nsource $ROOT/dev/util.sh\n\narch=$1\nhost_primary=$2\nhost_backup=$3\n\nfor image in \"${all_images[@]}\"; do\n  is_multi_arch=\"false\"\n  if [[ \" ${multi_arch_images[*]} \" =~ \" $image \" ]]; then\n    is_multi_arch=\"true\"\n    $ROOT/build/push-image.sh $host_primary $host_backup $image $is_multi_arch $arch\n  elif [ \"$arch\" = \"amd64\" ]; then\n    $ROOT/build/push-image.sh $host_primary $host_backup $image $is_multi_arch $arch\n  fi\ndone\n"
  },
  {
    "path": "build/test.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\nENVTEST_ASSETS_DIR=${ROOT}/testbin\n\ncluster_env=\"undefined\"\ncreate_cluster=\"no\"\npositional_args=()\nwhile [[ $# -gt 0 ]]; do\n  key=\"$1\"\n  case $key in\n    -e|--cluster-env)\n    cluster_env=\"$2\"\n    shift\n    ;;\n    -c|--create-cluster)\n    create_cluster=\"yes\"\n    shift\n    ;;\n    *)\n    positional_args+=(\"$1\")\n    shift\n    ;;\n  esac\ndone\nset -- \"${positional_args[@]}\"\npositional_args=()\nfor i in \"$@\"; do\n  case $i in\n    -e=*|--cluster-env=*)\n    cluster_env=\"${i#*=}\"\n    shift\n    ;;\n    -c|--create-cluster)\n    create_cluster=\"yes\"\n    shift\n    ;;\n    *)\n    positional_args+=(\"$1\")\n    shift\n    ;;\n  esac\ndone\nset -- \"${positional_args[@]}\"\nfor arg in \"$@\"; do\n  if [[ \"$arg\" == -* ]]; then\n    echo \"unknown flag: $arg\"\n    exit 1\n  fi\ndone\n\ncmd=${1:-\"\"}\nsub_cmd=${2:-\"\"}\n\nfunction run_go_tests() {\n  (\n    cd $ROOT\n    mkdir -p ${ENVTEST_ASSETS_DIR}\n\t  test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh\n\t  source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools ${ENVTEST_ASSETS_DIR}; setup_envtest_env ${ENVTEST_ASSETS_DIR}\n    go test -race ./... && echo \"go tests passed\"\n  )\n}\n\nfunction run_e2e_tests() {\n  if [ \"$create_cluster\" = \"yes\" ]; then\n    pytest $ROOT/test/e2e/tests --config \"$sub_cmd\"\n  else\n    pytest $ROOT/test/e2e/tests --env \"$cluster_env\"\n  fi\n}\n\nif [ \"$cmd\" = \"go\" ]; then\n  run_go_tests\nelif [ \"$cmd\" = \"e2e\" ]; then\n  run_e2e_tests\nfi\n"
  },
  {
    "path": "cli/cluster/delete.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cluster\n\nimport (\n\t\"fmt\"\n\t\"path\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\nfunc Delete(operatorConfig OperatorConfig, apiName string, keepCache bool, force bool) (schema.DeleteResponse, error) {\n\tif !force {\n\t\treadyReplicas := getReadyRealtimeAPIReplicasOrNil(operatorConfig, apiName)\n\t\tif readyReplicas != nil && *readyReplicas > 2 {\n\t\t\tprompt.YesOrExit(fmt.Sprintf(\"are you sure you want to delete %s (which has %d live replicas)?\", apiName, *readyReplicas), \"\", \"\")\n\t\t}\n\t}\n\n\tparams := map[string]string{\n\t\t\"apiName\":   apiName,\n\t\t\"keepCache\": s.Bool(keepCache),\n\t}\n\n\thttpRes, err := HTTPDelete(operatorConfig, \"/delete/\"+apiName, params)\n\tif err != nil {\n\t\treturn schema.DeleteResponse{}, err\n\t}\n\n\tvar deleteRes schema.DeleteResponse\n\terr = json.Unmarshal(httpRes, &deleteRes)\n\tif err != nil {\n\t\treturn schema.DeleteResponse{}, errors.Wrap(err, \"/delete\", string(httpRes))\n\t}\n\n\treturn deleteRes, nil\n}\n\nfunc getReadyRealtimeAPIReplicasOrNil(operatorConfig OperatorConfig, apiName string) *int32 {\n\thttpRes, err := HTTPGet(operatorConfig, \"/get/\"+apiName)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tvar apiRes schema.APIResponse\n\tif err = json.Unmarshal(httpRes, &apiRes); err != nil {\n\t\treturn nil\n\t}\n\n\tif apiRes.Status == nil {\n\t\treturn nil\n\t}\n\n\treturn pointer.Int32(apiRes.Status.Ready)\n}\n\nfunc StopJob(operatorConfig OperatorConfig, kind userconfig.Kind, apiName string, jobID string) (schema.DeleteResponse, error) {\n\tparams := map[string]string{\n\t\t\"apiName\": apiName,\n\t\t\"jobID\":   jobID,\n\t}\n\n\tvar endpointComponent string\n\tif kind == userconfig.BatchAPIKind {\n\t\tendpointComponent = \"batch\"\n\t} else {\n\t\tendpointComponent = \"tasks\"\n\t}\n\n\thttpRes, err := HTTPDelete(operatorConfig, path.Join(\"/\"+endpointComponent, apiName), params)\n\tif err != nil {\n\t\treturn schema.DeleteResponse{}, err\n\t}\n\n\tvar deleteRes schema.DeleteResponse\n\terr = json.Unmarshal(httpRes, &deleteRes)\n\tif err != nil {\n\t\treturn schema.DeleteResponse{}, errors.Wrap(err, string(httpRes))\n\t}\n\n\treturn deleteRes, nil\n}\n"
  },
  {
    "path": "cli/cluster/deploy.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cluster\n\nimport (\n\t\"path/filepath\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nfunc Deploy(operatorConfig OperatorConfig, configPath string, deploymentBytesMap map[string][]byte, force bool) ([]schema.DeployResult, error) {\n\tparams := map[string]string{\n\t\t\"force\":          s.Bool(force),\n\t\t\"configFileName\": filepath.Base(configPath),\n\t}\n\tuploadInput := &HTTPUploadInput{\n\t\tBytes: deploymentBytesMap,\n\t}\n\n\tresponse, err := HTTPUpload(operatorConfig, \"/deploy\", uploadInput, params)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar deployResults []schema.DeployResult\n\tif err := json.Unmarshal(response, &deployResults); err != nil {\n\t\treturn nil, errors.Wrap(err, \"/deploy\", string(response))\n\t}\n\n\treturn deployResults, nil\n}\n"
  },
  {
    "path": "cli/cluster/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cluster\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n)\n\nconst (\n\t_errStrCantMakeRequest = \"unable to make request\"\n\t_errStrRead            = \"unable to read\"\n)\n\nfunc errStrFailedToConnect(u url.URL) string {\n\treturn \"failed to connect to \" + urls.TrimQueryParamsURL(u)\n}\n\nconst (\n\tErrFailedToConnectOperator       = \"cli.failed_to_connect_operator\"\n\tErrOperatorSocketRead            = \"cli.operator_socket_read\"\n\tErrResponseUnknown               = \"cli.response_unknown\"\n\tErrOperatorResponseUnknown       = \"cli.operator_response_unknown\"\n\tErrOperatorStreamResponseUnknown = \"cli.operator_stream_response_unknown\"\n)\n\nfunc ErrorFailedToConnectOperator(originalError error, envName string, operatorURL string) error {\n\tmsg := \"\"\n\tif originalError != nil {\n\t\tmsg += urls.TrimQueryParamsStr(errors.Message(originalError)) + \"\\n\\n\"\n\t}\n\n\tif envName == \"\" {\n\t\tmsg += fmt.Sprintf(\"unable to connect to your cluster (operator endpoint: %s)\\n\\n\", operatorURL)\n\t\tmsg += \"if you don't have a cluster running:\\n\"\n\t\tmsg += \"    → to create a cluster, run `cortex cluster up`\\n\"\n\t\tmsg += \"\\nif you have a cluster running:\\n\"\n\t\tmsg += \"    → run `cortex cluster info --configure-env ENV_NAME` to update your environment (replace ENV_NAME with your desired environment name, and include `--config <cluster.yaml>` if you have a cluster configuration file)\\n\"\n\t} else {\n\t\tmsg += fmt.Sprintf(\"unable to connect to your cluster in the %s environment (operator endpoint: %s)\\n\\n\", envName, operatorURL)\n\t\tmsg += \"if you don't have a cluster running:\\n\"\n\t\tmsg += fmt.Sprintf(\"    → if you'd like to create a cluster, run `cortex cluster up --configure-env %s`\\n\", envName)\n\t\tmsg += fmt.Sprintf(\"    → otherwise you can ignore this message, and prevent it in the future with `cortex env delete %s`\\n\", envName)\n\t\tmsg += \"\\nif you have a cluster running:\\n\"\n\t\tmsg += fmt.Sprintf(\"    → run `cortex cluster info --configure-env %s` to update your environment (include `--config <cluster.yaml>` if you have a cluster configuration file)\\n\", envName)\n\t\tmsg += fmt.Sprintf(\"    → if you set `operator_load_balancer_scheme: internal` in your cluster configuration file, your CLI must run from within a VPC that has access to your cluster's VPC (see https://docs.cortexlabs.com/v/%s/)\\n\", consts.CortexVersionMinor)\n\t\tmsg += fmt.Sprintf(\"    → confirm that the ip address of this machine falls within the CIDR ranges specified in `operator_load_balancer_cidr_whitelist`\")\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFailedToConnectOperator,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorOperatorSocketRead(err error) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrOperatorSocketRead,\n\t\tMessage: err.Error(),\n\t\tNoPrint: true,\n\t})\n}\n\nfunc ErrorResponseUnknown(body string, statusCode int) error {\n\tmsg := body\n\tif strings.TrimSpace(body) == \"\" {\n\t\tmsg = fmt.Sprintf(\"empty response (status code %d)\", statusCode)\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrResponseUnknown,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorOperatorResponseUnknown(body string, statusCode int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrOperatorResponseUnknown,\n\t\tMessage: fmt.Sprintf(\"unexpected response from operator (status code %d): %s\", statusCode, body),\n\t})\n}\n\nfunc ErrorOperatorStreamResponseUnknown(body string, statusCode int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrOperatorStreamResponseUnknown,\n\t\tMessage: fmt.Sprintf(\"unexpected response from operator (status code %d): %s\", statusCode, body),\n\t})\n}\n"
  },
  {
    "path": "cli/cluster/get.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cluster\n\nimport (\n\t\"path\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nfunc GetAPIs(operatorConfig OperatorConfig) ([]schema.APIResponse, error) {\n\thttpRes, err := HTTPGet(operatorConfig, \"/get\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar apisRes []schema.APIResponse\n\tif err = json.Unmarshal(httpRes, &apisRes); err != nil {\n\t\treturn nil, errors.Wrap(err, \"/get\", string(httpRes))\n\t}\n\treturn apisRes, nil\n}\n\nfunc GetAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse, error) {\n\thttpRes, err := HTTPGet(operatorConfig, \"/get/\"+apiName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar apiRes []schema.APIResponse\n\tif err = json.Unmarshal(httpRes, &apiRes); err != nil {\n\t\treturn nil, errors.Wrap(err, \"/get/\"+apiName, string(httpRes))\n\t}\n\n\treturn apiRes, nil\n}\n\nfunc DescribeAPI(operatorConfig OperatorConfig, apiName string) ([]schema.APIResponse, error) {\n\thttpRes, err := HTTPGet(operatorConfig, \"/describe/\"+apiName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar apiRes []schema.APIResponse\n\tif err = json.Unmarshal(httpRes, &apiRes); err != nil {\n\t\treturn nil, errors.Wrap(err, \"/describe/\"+apiName, string(httpRes))\n\t}\n\n\treturn apiRes, nil\n}\n\nfunc GetAPIByID(operatorConfig OperatorConfig, apiName string, apiID string) ([]schema.APIResponse, error) {\n\thttpRes, err := HTTPGet(operatorConfig, \"/get/\"+apiName+\"/\"+apiID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar apiRes []schema.APIResponse\n\tif err = json.Unmarshal(httpRes, &apiRes); err != nil {\n\t\treturn nil, errors.Wrap(err, \"/get/\"+apiName+\"/\"+apiID, string(httpRes))\n\t}\n\n\treturn apiRes, nil\n}\n\nfunc GetBatchJob(operatorConfig OperatorConfig, apiName string, jobID string) (schema.BatchJobResponse, error) {\n\tendpoint := path.Join(\"/batch\", apiName)\n\thttpRes, err := HTTPGet(operatorConfig, endpoint, map[string]string{\"jobID\": jobID})\n\tif err != nil {\n\t\treturn schema.BatchJobResponse{}, err\n\t}\n\n\tvar jobRes schema.BatchJobResponse\n\tif err = json.Unmarshal(httpRes, &jobRes); err != nil {\n\t\treturn schema.BatchJobResponse{}, errors.Wrap(err, endpoint, string(httpRes))\n\t}\n\n\treturn jobRes, nil\n}\n\nfunc GetTaskJob(operatorConfig OperatorConfig, apiName string, jobID string) (schema.TaskJobResponse, error) {\n\tendpoint := path.Join(\"/tasks\", apiName)\n\thttpRes, err := HTTPGet(operatorConfig, endpoint, map[string]string{\"jobID\": jobID})\n\tif err != nil {\n\t\treturn schema.TaskJobResponse{}, err\n\t}\n\n\tvar jobRes schema.TaskJobResponse\n\tif err = json.Unmarshal(httpRes, &jobRes); err != nil {\n\t\treturn schema.TaskJobResponse{}, errors.Wrap(err, endpoint, string(httpRes))\n\t}\n\n\treturn jobRes, nil\n}\n"
  },
  {
    "path": "cli/cluster/info.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cluster\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nfunc Info(operatorConfig OperatorConfig) (*schema.InfoResponse, error) {\n\thttpResponse, err := HTTPGet(operatorConfig, \"/info\")\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"unable to connect to operator\", \"/info\")\n\t}\n\n\tvar infoResponse schema.InfoResponse\n\terr = json.Unmarshal(httpResponse, &infoResponse)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"/info\", string(httpResponse))\n\t}\n\n\treturn &infoResponse, nil\n}\n"
  },
  {
    "path": "cli/cluster/lib_http_client.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cluster\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"mime/multipart\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/archive\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\ntype OperatorClient struct {\n\t*http.Client\n}\n\ntype OperatorConfig struct {\n\tTelemetry        bool\n\tClientID         string\n\tEnvName          string\n\tOperatorEndpoint string\n}\n\nfunc HTTPGet(operatorConfig OperatorConfig, endpoint string, qParams ...map[string]string) ([]byte, error) {\n\treq, err := operatorRequest(operatorConfig, \"GET\", endpoint, nil, qParams...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn makeOperatorRequest(operatorConfig, req)\n}\n\nfunc HTTPPostObjAsJSON(operatorConfig OperatorConfig, endpoint string, requestData interface{}, qParams ...map[string]string) ([]byte, error) {\n\tjsonRequestData, err := json.Marshal(requestData)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn HTTPPostJSON(operatorConfig, endpoint, jsonRequestData, qParams...)\n}\n\nfunc HTTPPostJSON(operatorConfig OperatorConfig, endpoint string, jsonRequestData []byte, qParams ...map[string]string) ([]byte, error) {\n\tpayload := bytes.NewBuffer(jsonRequestData)\n\treq, err := operatorRequest(operatorConfig, http.MethodPost, endpoint, payload, qParams...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treq.Header.Set(\"Content-Type\", \"application/json\")\n\treturn makeOperatorRequest(operatorConfig, req)\n}\n\nfunc HTTPPostNoBody(operatorConfig OperatorConfig, endpoint string, qParams ...map[string]string) ([]byte, error) {\n\treq, err := operatorRequest(operatorConfig, http.MethodPost, endpoint, nil, qParams...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn makeOperatorRequest(operatorConfig, req)\n}\n\nfunc HTTPDelete(operatorConfig OperatorConfig, endpoint string, qParams ...map[string]string) ([]byte, error) {\n\treq, err := operatorRequest(operatorConfig, http.MethodDelete, endpoint, nil, qParams...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn makeOperatorRequest(operatorConfig, req)\n}\n\ntype HTTPUploadInput struct {\n\tFilePaths map[string]string\n\tBytes     map[string][]byte\n}\n\nfunc HTTPUpload(operatorConfig OperatorConfig, endpoint string, input *HTTPUploadInput, qParams ...map[string]string) ([]byte, error) {\n\tbody := new(bytes.Buffer)\n\twriter := multipart.NewWriter(body)\n\n\tfor fileName, filePath := range input.FilePaths {\n\t\tfile, err := files.Open(filePath)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdefer file.Close()\n\t\tif err := addFileToMultipart(fileName, writer, file); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tfor fileName, fileBytes := range input.Bytes {\n\t\tif err := addFileToMultipart(fileName, writer, bytes.NewReader(fileBytes)); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif err := writer.Close(); err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrCantMakeRequest)\n\t}\n\n\treq, err := operatorRequest(operatorConfig, http.MethodPost, endpoint, body, qParams...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treq.Header.Set(\"Content-Type\", writer.FormDataContentType())\n\treturn makeOperatorRequest(operatorConfig, req)\n}\n\nfunc addFileToMultipart(fileName string, writer *multipart.Writer, reader io.Reader) error {\n\tpart, err := writer.CreateFormFile(fileName, fileName)\n\tif err != nil {\n\t\treturn errors.Wrap(err, _errStrCantMakeRequest)\n\t}\n\n\tif _, err = io.Copy(part, reader); err != nil {\n\t\treturn errors.Wrap(err, _errStrCantMakeRequest)\n\t}\n\treturn nil\n}\n\nfunc HTTPUploadZip(operatorConfig OperatorConfig, endpoint string, zipInput *archive.Input, fileName string, qParams ...map[string]string) ([]byte, error) {\n\tzipBytes, _, err := archive.ZipToMem(zipInput)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"failed to zip configuration file\")\n\t}\n\n\tuploadInput := &HTTPUploadInput{\n\t\tBytes: map[string][]byte{\n\t\t\tfileName: zipBytes,\n\t\t},\n\t}\n\treturn HTTPUpload(operatorConfig, endpoint, uploadInput, qParams...)\n}\n\nfunc operatorRequest(operatorConfig OperatorConfig, method string, endpoint string, body io.Reader, qParams ...map[string]string) (*http.Request, error) {\n\treq, err := http.NewRequest(method, operatorConfig.OperatorEndpoint+endpoint, body)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrCantMakeRequest)\n\t}\n\n\tvalues := req.URL.Query()\n\tfor _, paramMap := range qParams {\n\t\tfor key, value := range paramMap {\n\t\t\tvalues.Set(key, value)\n\t\t}\n\t}\n\treq.URL.RawQuery = values.Encode()\n\n\treturn req, nil\n}\n\nfunc makeOperatorRequest(operatorConfig OperatorConfig, request *http.Request) ([]byte, error) {\n\tif operatorConfig.Telemetry {\n\t\tvalues := request.URL.Query()\n\t\tvalues.Set(\"clientID\", operatorConfig.ClientID)\n\t\trequest.URL.RawQuery = values.Encode()\n\t}\n\n\trequest.Header.Set(\"CortexAPIVersion\", consts.CortexVersion)\n\tawsClient, err := aws.New()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tauthHeader, err := awsClient.IdentityRequestAsHeader()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trequest.Header.Set(consts.AuthHeader, authHeader)\n\n\ttimeout := 600 * time.Second\n\tif request.URL.Path == \"/info\" {\n\t\ttimeout = 10 * time.Second\n\t}\n\n\tclient := &http.Client{\n\t\tTimeout: timeout,\n\t\tTransport: &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t\t},\n\t}\n\n\tresponse, err := client.Do(request)\n\tif err != nil {\n\t\treturn nil, ErrorFailedToConnectOperator(err, operatorConfig.EnvName, operatorConfig.OperatorEndpoint)\n\t}\n\tdefer response.Body.Close()\n\n\tif response.StatusCode != 200 {\n\t\tbodyBytes, err := ioutil.ReadAll(response.Body)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, _errStrRead)\n\t\t}\n\n\t\tvar output schema.ErrorResponse\n\t\terr = json.Unmarshal(bodyBytes, &output)\n\t\tif err != nil || output.Message == \"\" {\n\t\t\treturn nil, ErrorOperatorResponseUnknown(string(bodyBytes), response.StatusCode)\n\t\t}\n\n\t\treturn nil, errors.WithStack(&errors.Error{\n\t\t\tKind:        output.Kind,\n\t\t\tMessage:     output.Message,\n\t\t\tNoTelemetry: true,\n\t\t})\n\t}\n\n\tbodyBytes, err := ioutil.ReadAll(response.Body)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrRead)\n\t}\n\treturn bodyBytes, nil\n}\n"
  },
  {
    "path": "cli/cluster/logs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cluster\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/cli/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/gorilla/websocket\"\n)\n\nfunc GetLogs(operatorConfig OperatorConfig, apiName string) (schema.LogResponse, error) {\n\thttpRes, err := HTTPGet(operatorConfig, \"/logs/\"+apiName)\n\tif err != nil {\n\t\treturn schema.LogResponse{}, err\n\t}\n\n\tvar logResponse schema.LogResponse\n\tif err = json.Unmarshal(httpRes, &logResponse); err != nil {\n\t\treturn schema.LogResponse{}, errors.Wrap(err, \"/logs/\"+apiName, string(httpRes))\n\t}\n\n\treturn logResponse, nil\n}\n\nfunc GetJobLogs(operatorConfig OperatorConfig, apiName string, jobID string) (schema.LogResponse, error) {\n\thttpRes, err := HTTPGet(operatorConfig, \"/logs/\"+apiName, map[string]string{\"jobID\": jobID})\n\tif err != nil {\n\t\treturn schema.LogResponse{}, err\n\t}\n\n\tvar logResponse schema.LogResponse\n\tif err = json.Unmarshal(httpRes, &logResponse); err != nil {\n\t\treturn schema.LogResponse{}, errors.Wrap(err, \"/logs/\"+apiName, string(httpRes))\n\t}\n\n\treturn logResponse, nil\n}\n\nfunc StreamLogs(operatorConfig OperatorConfig, apiName string) error {\n\treturn streamLogs(operatorConfig, \"/streamlogs/\"+apiName)\n}\n\nfunc StreamJobLogs(operatorConfig OperatorConfig, apiName string, jobID string) error {\n\treturn streamLogs(operatorConfig, \"/streamlogs/\"+apiName, map[string]string{\"jobID\": jobID})\n}\n\nfunc streamLogs(operatorConfig OperatorConfig, path string, qParams ...map[string]string) error {\n\tinterrupt := make(chan os.Signal, 1)\n\tsignal.Notify(interrupt, os.Interrupt)\n\n\treq, err := operatorRequest(operatorConfig, \"GET\", path, nil, qParams...)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvalues := req.URL.Query()\n\tif operatorConfig.Telemetry {\n\t\tvalues.Set(\"clientID\", operatorConfig.ClientID)\n\t}\n\n\treq.URL.RawQuery = values.Encode()\n\twsURL := req.URL.String()\n\twsURL = strings.Replace(wsURL, \"http\", \"ws\", 1)\n\n\theader := http.Header{}\n\theader.Set(\"CortexAPIVersion\", consts.CortexVersion)\n\tawsClient, err := aws.New()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tauthHeader, err := awsClient.IdentityRequestAsHeader()\n\tif err != nil {\n\t\treturn err\n\t}\n\theader.Set(consts.AuthHeader, authHeader)\n\n\tvar dialer = websocket.Dialer{\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t}\n\n\tconnection, response, err := dialer.Dial(wsURL, header)\n\tif err != nil && response == nil {\n\t\treturn ErrorFailedToConnectOperator(err, operatorConfig.EnvName, strings.Replace(operatorConfig.OperatorEndpoint, \"http\", \"ws\", 1))\n\t}\n\tdefer response.Body.Close()\n\n\tif err != nil {\n\t\tbodyBytes, err := ioutil.ReadAll(response.Body)\n\t\tif err != nil || bodyBytes == nil || string(bodyBytes) == \"\" {\n\t\t\treturn ErrorFailedToConnectOperator(err, operatorConfig.EnvName, strings.Replace(operatorConfig.OperatorEndpoint, \"http\", \"ws\", 1))\n\t\t}\n\t\tvar output schema.ErrorResponse\n\t\terr = json.Unmarshal(bodyBytes, &output)\n\t\tif err != nil || output.Message == \"\" {\n\t\t\treturn ErrorOperatorStreamResponseUnknown(string(bodyBytes), response.StatusCode)\n\t\t}\n\t\treturn errors.WithStack(&errors.Error{\n\t\t\tKind:        output.Kind,\n\t\t\tMessage:     output.Message,\n\t\t\tNoTelemetry: true,\n\t\t})\n\t}\n\tdefer connection.Close()\n\n\tdone := make(chan struct{})\n\thandleConnection(connection, done)\n\tcloseConnection(connection, done, interrupt)\n\treturn nil\n}\n\nfunc handleConnection(connection *websocket.Conn, done chan struct{}) {\n\troutines.RunWithPanicHandler(func() {\n\t\tdefer close(done)\n\t\tfor {\n\t\t\t_, message, err := connection.ReadMessage()\n\t\t\tif err != nil {\n\t\t\t\texit.Error(ErrorOperatorSocketRead(err))\n\t\t\t}\n\t\t\tfmt.Print(string(message))\n\t\t}\n\t}, false)\n}\n\nfunc closeConnection(connection *websocket.Conn, done chan struct{}, interrupt chan os.Signal) {\n\tfor {\n\t\tselect {\n\t\tcase <-done:\n\t\t\treturn\n\t\tcase <-interrupt:\n\t\t\tconnection.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, \"\"))\n\t\t\treturn\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "cli/cluster/refresh.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cluster\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nfunc Refresh(operatorConfig OperatorConfig, apiName string, force bool) (schema.RefreshResponse, error) {\n\tparams := map[string]string{\n\t\t\"force\": s.Bool(force),\n\t}\n\n\thttpRes, err := HTTPPostNoBody(operatorConfig, \"/refresh/\"+apiName, params)\n\tif err != nil {\n\t\treturn schema.RefreshResponse{}, err\n\t}\n\n\tvar refreshRes schema.RefreshResponse\n\terr = json.Unmarshal(httpRes, &refreshRes)\n\tif err != nil {\n\t\treturn schema.RefreshResponse{}, errors.Wrap(err, \"/refresh\", string(httpRes))\n\t}\n\n\treturn refreshRes, nil\n}\n"
  },
  {
    "path": "cli/cmd/cluster.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"encoding/base64\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/autoscaling\"\n\t\"github.com/aws/aws-sdk-go/service/ec2\"\n\t\"github.com/aws/aws-sdk-go/service/eks\"\n\t\"github.com/aws/aws-sdk-go/service/elb\"\n\t\"github.com/aws/aws-sdk-go/service/elbv2\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/health\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/docker\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\tlibmath \"github.com/cortexlabs/cortex/pkg/lib/math\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterstate\"\n\t\"github.com/cortexlabs/yaml\"\n\t\"github.com/spf13/cobra\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\tclientgoscheme \"k8s.io/client-go/kubernetes/scheme\"\n\t\"k8s.io/client-go/rest\"\n\t\"sigs.k8s.io/aws-iam-authenticator/pkg/token\"\n)\n\nvar (\n\t_flagClusterUpEnv                string\n\t_flagClusterInfoEnv              string\n\t_flagClusterConfig               string\n\t_flagClusterName                 string\n\t_flagClusterRegion               string\n\t_flagClusterInfoDebug            bool\n\t_flagClusterInfoPrintConfig      bool\n\t_flagClusterDisallowPrompt       bool\n\t_flagClusterDownKeepAWSResources bool\n)\n\nvar _eksctlPrefixRegex = regexp.MustCompile(`^.*[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} \\[.+] {2}`)\n\nfunc clusterInit() {\n\t_clusterUpCmd.Flags().SortFlags = false\n\t_clusterUpCmd.Flags().StringVarP(&_flagClusterUpEnv, \"configure-env\", \"e\", \"\", \"name of environment to configure (default: the name of your cluster)\")\n\t_clusterUpCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, \"yes\", \"y\", false, \"skip prompts\")\n\t_clusterCmd.AddCommand(_clusterUpCmd)\n\n\t_clusterInfoCmd.Flags().SortFlags = false\n\taddClusterConfigFlag(_clusterInfoCmd)\n\taddClusterNameFlag(_clusterInfoCmd)\n\taddClusterRegionFlag(_clusterInfoCmd)\n\t_clusterInfoCmd.Flags().VarP(&_flagOutput, \"output\", \"o\", fmt.Sprintf(\"output format: one of %s\", strings.Join(flags.OutputTypeStrings(), \"|\")))\n\t_clusterInfoCmd.Flags().StringVarP(&_flagClusterInfoEnv, \"configure-env\", \"e\", \"\", \"name of environment to configure\")\n\t_clusterInfoCmd.Flags().BoolVarP(&_flagClusterInfoDebug, \"debug\", \"d\", false, \"save the current cluster state to a file\")\n\t_clusterInfoCmd.Flags().BoolVarP(&_flagClusterInfoPrintConfig, \"print-config\", \"\", false, \"print the cluster config\")\n\t_clusterInfoCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, \"yes\", \"y\", false, \"skip prompts\")\n\t_clusterCmd.AddCommand(_clusterInfoCmd)\n\n\t_clusterConfigureCmd.Flags().SortFlags = false\n\t_clusterConfigureCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, \"yes\", \"y\", false, \"skip prompts\")\n\t_clusterCmd.AddCommand(_clusterConfigureCmd)\n\n\t_clusterDownCmd.Flags().SortFlags = false\n\taddClusterConfigFlag(_clusterDownCmd)\n\taddClusterNameFlag(_clusterDownCmd)\n\taddClusterRegionFlag(_clusterDownCmd)\n\t_clusterDownCmd.Flags().BoolVarP(&_flagClusterDisallowPrompt, \"yes\", \"y\", false, \"skip prompts\")\n\t_clusterDownCmd.Flags().BoolVar(&_flagClusterDownKeepAWSResources, \"keep-aws-resources\", false, \"skip deletion of resources that cortex provisioned on aws (bucket contents, ebs volumes, log group)\")\n\t_clusterCmd.AddCommand(_clusterDownCmd)\n\n\t_clusterExportCmd.Flags().SortFlags = false\n\taddClusterConfigFlag(_clusterExportCmd)\n\taddClusterNameFlag(_clusterExportCmd)\n\taddClusterRegionFlag(_clusterExportCmd)\n\t_clusterCmd.AddCommand(_clusterExportCmd)\n\n\t_clusterHealthCmd.Flags().SortFlags = false\n\taddClusterConfigFlag(_clusterHealthCmd)\n\taddClusterNameFlag(_clusterHealthCmd)\n\taddClusterRegionFlag(_clusterHealthCmd)\n\t_clusterHealthCmd.Flags().VarP(&_flagOutput, \"output\", \"o\", fmt.Sprintf(\"output format: one of %s\", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), \"|\")))\n\t_clusterCmd.AddCommand(_clusterHealthCmd)\n}\n\nfunc addClusterConfigFlag(cmd *cobra.Command) {\n\tcmd.Flags().StringVarP(&_flagClusterConfig, \"config\", \"c\", \"\", \"path to a cluster configuration file\")\n\terr := cmd.Flags().SetAnnotation(\"config\", cobra.BashCompFilenameExt, _configFileExts)\n\tif err != nil {\n\t\texit.Error(err) // should never happen\n\t}\n}\n\nfunc addClusterNameFlag(cmd *cobra.Command) {\n\tcmd.Flags().StringVarP(&_flagClusterName, \"name\", \"n\", \"\", \"name of the cluster\")\n}\n\nfunc addClusterRegionFlag(cmd *cobra.Command) {\n\tcmd.Flags().StringVarP(&_flagClusterRegion, \"region\", \"r\", \"\", \"aws region of the cluster\")\n}\n\nvar _clusterCmd = &cobra.Command{\n\tUse:   \"cluster\",\n\tShort: \"manage cortex clusters (contains subcommands)\",\n}\n\nvar _clusterUpCmd = &cobra.Command{\n\tUse:   \"up CLUSTER_CONFIG_FILE\",\n\tShort: \"spin up a cluster on aws\",\n\tArgs:  cobra.ExactArgs(1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.EventNotify(\"cli.cluster.up\")\n\n\t\tclusterConfigFile := args[0]\n\n\t\tif _, err := docker.GetDockerClient(); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\taccessConfig, err := getNewClusterAccessConfig(clusterConfigFile)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tenvName := _flagClusterUpEnv\n\t\tif envName == \"\" {\n\t\t\tenvName = accessConfig.ClusterName\n\t\t}\n\n\t\tenvExists, err := isEnvConfigured(envName)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tif envExists {\n\t\t\tif _flagClusterDisallowPrompt {\n\t\t\t\tfmt.Printf(\"found an existing environment named \\\"%s\\\", which will be overwritten to connect to this cluster once it's created\\n\\n\", envName)\n\t\t\t} else {\n\t\t\t\tprompt.YesOrExit(fmt.Sprintf(\"found an existing environment named \\\"%s\\\"; would you like to overwrite it to connect to this cluster once it's created?\", envName), \"\", \"you can specify a different environment name to be configured to connect to this cluster by specifying the --configure-env flag (e.g. `cortex cluster up --configure-env prod`); or you can list your environments with `cortex env list` and delete an environment with `cortex env delete ENV_NAME`\")\n\t\t\t}\n\t\t}\n\n\t\tawsClient, err := newAWSClient(accessConfig.Region, true)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tstacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tstate := clusterstate.GetClusterState(stacks)\n\t\tif err := clusterstate.AssertClusterState(stacks, state, clusterstate.StateClusterDoesntExist); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tpromptIfNotAdmin(awsClient, _flagClusterDisallowPrompt)\n\n\t\tclusterConfig, err := getInstallClusterConfig(awsClient, clusterConfigFile)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tconfirmInstallClusterConfig(clusterConfig, awsClient, _flagClusterDisallowPrompt)\n\n\t\terr = createS3BucketIfNotFound(awsClient, clusterConfig.Bucket, clusterConfig.Tags)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\terr = setLifecycleRulesOnClusterUp(awsClient, clusterConfig.Bucket, clusterConfig.ClusterUID)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\terr = createLogGroupIfNotFound(awsClient, clusterConfig.ClusterName, clusterConfig.Tags)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\taccountID, _, err := awsClient.GetCachedAccountID()\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\terr = clusterconfig.CreateDefaultPolicy(awsClient, clusterconfig.CortexPolicyTemplateArgs{\n\t\t\tClusterName: clusterConfig.ClusterName,\n\t\t\tLogGroup:    clusterConfig.ClusterName,\n\t\t\tBucket:      clusterConfig.Bucket,\n\t\t\tRegion:      clusterConfig.Region,\n\t\t\tAccountID:   accountID,\n\t\t})\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tout, exitCode, err := runManagerWithClusterConfig(\"/root/install.sh\", clusterConfig, awsClient, nil, nil, nil)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tif exitCode == nil || *exitCode != 0 {\n\t\t\tout = s.LastNChars(filterEKSCTLOutput(out), 8192) // get the last 8192 characters because that is the sentry message limit\n\t\t\teksCluster, err := awsClient.EKSClusterOrNil(clusterConfig.ClusterName)\n\t\t\tif err != nil {\n\t\t\t\thelpStr := \"\\ndebugging tips (may or may not apply to this error):\"\n\t\t\t\thelpStr += fmt.Sprintf(\"\\n* if your cluster started spinning up but was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \\\"Activity\\\" or \\\"Activity History\\\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:\", clusterConfig.Region)\n\t\t\t\thelpStr += \"\\n* if your cluster started spinning up, please run `cortex cluster down` to delete the cluster before trying to create this cluster again\"\n\t\t\t\tfmt.Println(helpStr)\n\t\t\t\texit.Error(ErrorClusterUp(out))\n\t\t\t}\n\n\t\t\t// the cluster never started spinning up\n\t\t\tif eksCluster == nil {\n\t\t\t\texit.Error(ErrorClusterUp(out))\n\t\t\t}\n\n\t\t\tclusterTags := map[string]string{clusterconfig.ClusterNameTag: clusterConfig.ClusterName}\n\t\t\tasgs, err := awsClient.AutoscalingGroups(clusterTags)\n\t\t\tif err != nil {\n\t\t\t\thelpStr := \"\\ndebugging tips (may or may not apply to this error):\"\n\t\t\t\thelpStr += fmt.Sprintf(\"\\n* if your cluster was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \\\"Activity\\\" or \\\"Activity History\\\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:\", clusterConfig.Region)\n\t\t\t\thelpStr += \"\\n* please run `cortex cluster down` to delete the cluster before trying to create this cluster again\"\n\t\t\t\tfmt.Println(helpStr)\n\t\t\t\texit.Error(ErrorClusterUp(out + helpStr))\n\t\t\t}\n\n\t\t\t// no autoscaling groups were created\n\t\t\tif len(asgs) == 0 {\n\t\t\t\thelpStr := \"\\nplease run `cortex cluster down` to delete the cluster before trying to create this cluster again\"\n\t\t\t\tfmt.Println(helpStr)\n\t\t\t\texit.Error(ErrorClusterUp(out + helpStr))\n\t\t\t}\n\n\t\t\tfor _, asg := range asgs {\n\t\t\t\tactivity, err := awsClient.MostRecentASGActivity(*asg.AutoScalingGroupName)\n\t\t\t\tif err != nil {\n\t\t\t\t\thelpStr := \"\\ndebugging tips (may or may not apply to this error):\"\n\t\t\t\t\thelpStr += fmt.Sprintf(\"\\n* if your cluster was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \\\"Activity\\\" or \\\"Activity History\\\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:\", clusterConfig.Region)\n\t\t\t\t\thelpStr += \"\\n* please run `cortex cluster down` to delete the cluster before trying to create this cluster again\"\n\t\t\t\t\tfmt.Println(helpStr)\n\t\t\t\t\texit.Error(ErrorClusterUp(out + helpStr))\n\t\t\t\t}\n\n\t\t\t\tif activity != nil && (activity.StatusCode == nil || *activity.StatusCode != autoscaling.ScalingActivityStatusCodeSuccessful) {\n\t\t\t\t\tstatus := \"(none)\"\n\t\t\t\t\tif activity.StatusCode != nil {\n\t\t\t\t\t\tstatus = *activity.StatusCode\n\t\t\t\t\t}\n\t\t\t\t\tdescription := \"(none)\"\n\t\t\t\t\tif activity.Description != nil {\n\t\t\t\t\t\tdescription = *activity.Description\n\t\t\t\t\t}\n\n\t\t\t\t\thelpStr := \"\\nyour cluster was unable to provision EC2 instances; here is one of the encountered errors:\"\n\t\t\t\t\thelpStr += fmt.Sprintf(\"\\n\\n> status: %s\\n> description: %s\", status, description)\n\t\t\t\t\thelpStr += fmt.Sprintf(\"\\n\\nadditional error information might be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \\\"Activity\\\" or \\\"Activity History\\\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:\", clusterConfig.Region)\n\t\t\t\t\thelpStr += \"\\n\\nplease run `cortex cluster down` to delete the cluster before trying to create this cluster again\"\n\t\t\t\t\tfmt.Println(helpStr)\n\t\t\t\t\texit.Error(ErrorClusterUp(out + helpStr))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// No failed asg activities\n\t\t\thelpStr := \"\\nplease run `cortex cluster down` to delete the cluster before trying to create this cluster again\"\n\t\t\tfmt.Println(helpStr)\n\t\t\texit.Error(ErrorClusterUp(out + helpStr))\n\t\t}\n\n\t\tloadBalancer, err := getNLBLoadBalancer(clusterConfig.ClusterName, OperatorLoadBalancer, awsClient)\n\t\tif err != nil {\n\t\t\texit.Error(errors.Append(err, fmt.Sprintf(\"\\n\\nyou can attempt to resolve this issue and configure your cli environment by running `cortex cluster info --configure-env %s`\", envName)))\n\t\t}\n\n\t\tnewEnvironment := cliconfig.Environment{\n\t\t\tName:             envName,\n\t\t\tOperatorEndpoint: \"https://\" + *loadBalancer.DNSName,\n\t\t}\n\n\t\terr = addEnvToCLIConfig(newEnvironment, true)\n\t\tif err != nil {\n\t\t\texit.Error(errors.Append(err, fmt.Sprintf(\"\\n\\nyou can attempt to resolve this issue and configure your cli environment by running `cortex cluster info --configure-env %s`\", envName)))\n\t\t}\n\n\t\tif envExists {\n\t\t\tfmt.Printf(console.Bold(\"\\nthe environment named \\\"%s\\\" has been updated to point to this cluster (and was set as the default environment)\\n\"), envName)\n\t\t} else {\n\t\t\tfmt.Printf(console.Bold(\"\\nan environment named \\\"%s\\\" has been configured to point to this cluster (and was set as the default environment)\\n\"), envName)\n\t\t}\n\t},\n}\n\nvar _clusterConfigureCmd = &cobra.Command{\n\tUse:   \"configure CLUSTER_CONFIG_FILE\",\n\tShort: \"update the cluster's configuration\",\n\tArgs:  cobra.ExactArgs(1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.cluster.configure\")\n\n\t\tclusterConfigFile := args[0]\n\n\t\tif _, err := docker.GetDockerClient(); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\taccessConfig, err := getNewClusterAccessConfig(clusterConfigFile)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tawsClient, err := newAWSClient(accessConfig.Region, true)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\trestConfig, err := getClusterRESTConfig(awsClient, accessConfig.ClusterName)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tscheme := runtime.NewScheme()\n\t\tif err := clientgoscheme.AddToScheme(scheme); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tk8sClient, err := k8s.New(consts.DefaultNamespace, false, restConfig, scheme)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tstacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tstate := clusterstate.GetClusterState(stacks)\n\t\tif err := clusterstate.AssertClusterState(stacks, state, clusterstate.StateClusterExists); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\toldClusterConfig := refreshCachedClusterConfig(awsClient, accessConfig, true)\n\n\t\tpromptIfNotAdmin(awsClient, _flagClusterDisallowPrompt)\n\n\t\tnewClusterConfig, configureChanges, err := getConfigureClusterConfig(awsClient, k8sClient, stacks, oldClusterConfig, clusterConfigFile)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tif !configureChanges.HasChanges() {\n\t\t\tfmt.Println(\"your cluster is already up to date\")\n\t\t\texit.Ok()\n\t\t}\n\n\t\tconfirmConfigureClusterConfig(configureChanges, oldClusterConfig, *newClusterConfig, _flagClusterDisallowPrompt)\n\n\t\tout, exitCode, err := runManagerWithClusterConfig(\"/root/install.sh --configure\", newClusterConfig, awsClient, nil, nil, []string{\n\t\t\t\"CORTEX_NODEGROUP_NAMES_TO_UPDATE=\" + strings.Join(configureChanges.NodeGroupsToUpdate, \" \"),        // NodeGroupsToUpdate contain the cluster config node-group names\n\t\t\t\"CORTEX_NODEGROUP_NAMES_TO_ADD=\" + strings.Join(configureChanges.NodeGroupsToAdd, \" \"),              // NodeGroupsToAdd contain the cluster config node-group names\n\t\t\t\"CORTEX_EKS_NODEGROUP_NAMES_TO_REMOVE=\" + strings.Join(configureChanges.EKSNodeGroupsToRemove, \" \"), // EKSNodeGroupsToRemove contain the EKS node-group names\n\t\t})\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tif exitCode == nil || *exitCode != 0 {\n\t\t\tout = s.LastNChars(out, 8192) // get the last 8192 characters because that is the sentry message limit\n\n\t\t\thelpStr := \"\\ndebugging tips (may or may not apply to this error):\"\n\t\t\thelpStr += fmt.Sprintf(\n\t\t\t\t\"\\n* if your cluster was unable to provision/remove/scale some nodegroups, additional error information may be found in the description of your cloudformation stack (https://console.aws.amazon.com/cloudformation/home?region=%s#/stacks)\"+\n\t\t\t\t\t\" or in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the  \\\"Activity\\\" or \\\"Activity History\\\" tab) (https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups)\",\n\t\t\t\toldClusterConfig.Region,\n\t\t\t\toldClusterConfig.Region,\n\t\t\t)\n\t\t\tfmt.Println(helpStr)\n\t\t\texit.Error(ErrorClusterConfigure(out + helpStr))\n\t\t}\n\t},\n}\n\nvar _clusterInfoCmd = &cobra.Command{\n\tUse:   \"info\",\n\tShort: \"get information about a cluster\",\n\tArgs:  cobra.NoArgs,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.cluster.info\")\n\n\t\tif _, err := docker.GetDockerClient(); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\taccessConfig, err := getClusterAccessConfigWithCache(true)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tif _flagClusterInfoPrintConfig && _flagOutput == flags.PrettyOutputType {\n\t\t\t_flagOutput = flags.YAMLOutputType\n\t\t}\n\n\t\tawsClient, err := newAWSClient(accessConfig.Region, _flagOutput == flags.PrettyOutputType)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tif _flagClusterInfoPrintConfig && _flagClusterInfoDebug {\n\t\t\texit.Error(ErrorMutuallyExclusiveFlags(\"--print-config\", \"--debug\"))\n\t\t}\n\t\tif _flagClusterInfoDebug && _flagOutput != flags.PrettyOutputType {\n\t\t\texit.Error(ErrorMutuallyExclusiveFlags(\"--debug\", \"--output\"))\n\t\t}\n\n\t\tstacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tstate := clusterstate.GetClusterState(stacks)\n\t\tif err := clusterstate.AssertClusterState(stacks, state, clusterstate.StateClusterExists); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tif _flagClusterInfoDebug {\n\t\t\tcmdDebug(awsClient, accessConfig)\n\t\t} else if _flagClusterInfoPrintConfig {\n\t\t\tcmdPrintConfig(awsClient, accessConfig, _flagOutput)\n\t\t} else {\n\t\t\tcmdInfo(awsClient, accessConfig, stacks, _flagOutput, _flagClusterDisallowPrompt)\n\t\t}\n\t},\n}\n\nvar _clusterDownCmd = &cobra.Command{\n\tUse:   \"down\",\n\tShort: \"spin down a cluster\",\n\tArgs:  cobra.NoArgs,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.cluster.down\")\n\n\t\tif _, err := docker.GetDockerClient(); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\taccessConfig, err := getClusterAccessConfigWithCache(true)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\t// Check AWS access\n\t\tawsClient, err := newAWSClient(accessConfig.Region, true)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\taccountID, _, err := awsClient.GetCachedAccountID()\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tbucketName := clusterconfig.BucketName(accountID, accessConfig.ClusterName, accessConfig.Region)\n\n\t\twarnIfNotAdmin(awsClient)\n\n\t\tif _flagClusterDisallowPrompt {\n\t\t\tfmt.Printf(\"your cluster named \\\"%s\\\" in %s will be spun down and all apis will be deleted\\n\\n\", accessConfig.ClusterName, accessConfig.Region)\n\t\t} else {\n\t\t\tprompt.YesOrExit(fmt.Sprintf(\"your cluster named \\\"%s\\\" in %s will be spun down and all apis will be deleted, are you sure you want to continue?\", accessConfig.ClusterName, accessConfig.Region), \"\", \"\")\n\t\t}\n\n\t\tvar clusterExists bool\n\t\terrorsList := []error{}\n\n\t\tfmt.Print(\"￮ retrieving cluster ... \")\n\t\tstacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig)\n\t\tif err != nil {\n\t\t\terrorsList = append(errorsList, err)\n\t\t\tfmt.Print(\"failed ✗\")\n\t\t\tfmt.Printf(\"\\n\\ncouldn't retrieve cluster state; check the cluster stacks in the cloudformation console: https://%s.console.aws.amazon.com/cloudformation\\n\", accessConfig.Region)\n\t\t\terrors.PrintError(err)\n\t\t\tfmt.Println()\n\t\t} else {\n\t\t\tif clusterstate.GetClusterState(stacks) == clusterstate.StateClusterDoesntExist {\n\t\t\t\tfmt.Println(\"cluster doesn't exist ✓\")\n\t\t\t} else {\n\t\t\t\tfmt.Println(\"✓\")\n\t\t\t\tclusterExists = true\n\t\t\t}\n\t\t}\n\n\t\t// updating CLI env is best-effort, so ignore errors\n\t\tloadBalancer, _ := getNLBLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient)\n\n\t\tfmt.Print(\"￮ deleting sqs queues ... \")\n\t\tnumDeleted, err := awsClient.DeleteQueuesWithPrefix(clusterconfig.SQSNamePrefix(accessConfig.ClusterName))\n\t\tif err != nil {\n\t\t\terrorsList = append(errorsList, err)\n\t\t\tfmt.Print(\"failed ✗\")\n\t\t\tfmt.Printf(\"\\n\\nfailed to delete all sqs queues; please delete queues starting with the name %s via the cloudwatch console: https://%s.console.aws.amazon.com/sqs/v2/home\\n\", clusterconfig.SQSNamePrefix(accessConfig.ClusterName), accessConfig.Region)\n\t\t\terrors.PrintError(err)\n\t\t\tfmt.Println()\n\t\t} else if numDeleted == 0 {\n\t\t\tfmt.Println(\"no sqs queues exist ✓\")\n\t\t} else {\n\t\t\tfmt.Println(\"✓\")\n\t\t}\n\n\t\tclusterDoesntExist := !clusterExists\n\t\tif clusterExists {\n\t\t\tfmt.Print(\"￮ spinning down the cluster ...\")\n\t\t\tout, exitCode, err := runManagerAccessCommand(\"/root/uninstall.sh\", *accessConfig, awsClient, nil, nil)\n\t\t\tif err != nil {\n\t\t\t\terrorsList = append(errorsList, err)\n\t\t\t\tfmt.Println()\n\t\t\t\terrors.PrintError(err)\n\t\t\t} else if exitCode == nil || *exitCode != 0 {\n\t\t\t\ttemplate := \"\\nNote: if this error cannot be resolved, please ensure that all CloudFormation stacks for this cluster eventually become fully deleted (%s).\"\n\t\t\t\ttemplate += \" If the stack deletion process has failed, please delete the stacks directly from the AWS console (this may require manually deleting particular AWS resources that are blocking the stack deletion).\"\n\t\t\t\ttemplate += \" In addition to deleting the stacks manually from the AWS console, also make sure to empty and remove the %s bucket\"\n\t\t\t\thelpStr := fmt.Sprintf(template, clusterstate.CloudFormationURL(accessConfig.ClusterName, accessConfig.Region), bucketName)\n\t\t\t\tfmt.Println(helpStr)\n\t\t\t\terrorsList = append(errorsList, ErrorClusterDown(filterEKSCTLOutput(out)+helpStr))\n\t\t\t} else {\n\t\t\t\tclusterDoesntExist = true\n\t\t\t}\n\t\t\tfmt.Println()\n\t\t}\n\n\t\t// set lifecycle policy to clean the bucket\n\t\tvar bucketExists bool\n\t\tif !_flagClusterDownKeepAWSResources {\n\t\t\tfmt.Printf(\"￮ setting lifecycle policy to empty the %s bucket ... \", bucketName)\n\t\t\tbucketExists, err := awsClient.DoesBucketExist(bucketName)\n\t\t\tif err != nil {\n\t\t\t\terrorsList = append(errorsList, err)\n\t\t\t\tfmt.Print(\"failed ✗\")\n\t\t\t\tfmt.Printf(\"\\n\\nfailed to set lifecycle policy to empty the %s bucket; you can remove the bucket manually via the s3 console: https://s3.console.aws.amazon.com/s3/management/%s\\n\", bucketName, bucketName)\n\t\t\t\terrors.PrintError(err)\n\t\t\t\tfmt.Println()\n\t\t\t} else if !bucketExists {\n\t\t\t\tfmt.Println(\"bucket doesn't exist ✗\")\n\t\t\t} else {\n\t\t\t\terr = setLifecycleRulesOnClusterDown(awsClient, bucketName)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrorsList = append(errorsList, err)\n\t\t\t\t\tfmt.Print(\"failed ✗\")\n\t\t\t\t\tfmt.Printf(\"\\n\\nfailed to set lifecycle policy to empty the %s bucket; you can remove the bucket manually via the s3 console: https://s3.console.aws.amazon.com/s3/management/%s\\n\", bucketName, bucketName)\n\t\t\t\t\terrors.PrintError(err)\n\t\t\t\t\tfmt.Println()\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Println(\"✓\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// delete policy after spinning down the cluster (which deletes the roles) because policies can't be deleted if they are attached to roles\n\t\tif clusterDoesntExist {\n\t\t\tpolicyARN := clusterconfig.DefaultPolicyARN(accountID, accessConfig.ClusterName, accessConfig.Region)\n\t\t\tfmt.Printf(\"￮ deleting auto-generated iam policy %s ... \", policyARN)\n\t\t\tif policy, err := awsClient.GetPolicyOrNil(policyARN); err != nil {\n\t\t\t\terrorsList = append(errorsList, err)\n\t\t\t\tfmt.Print(\"failed ✗\")\n\t\t\t\tfmt.Printf(\"\\n\\nfailed to delete auto-generated cortex policy %s; please delete the policy via the iam console: https://console.aws.amazon.com/iam/home#/policies\\n\", policyARN)\n\t\t\t\terrors.PrintError(err)\n\t\t\t\tfmt.Println()\n\t\t\t} else if policy == nil {\n\t\t\t\tfmt.Println(\"policy doesn't exist ✓\")\n\t\t\t} else {\n\t\t\t\terr = awsClient.DeletePolicy(policyARN)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrorsList = append(errorsList, err)\n\t\t\t\t\tfmt.Print(\"failed ✗\")\n\t\t\t\t\tfmt.Printf(\"\\n\\nfailed to delete auto-generated cortex policy %s; please delete the policy via the iam console: https://console.aws.amazon.com/iam/home#/policies\\n\", policyARN)\n\t\t\t\t\terrors.PrintError(err)\n\t\t\t\t\tfmt.Println()\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Println(\"✓\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif !_flagClusterDownKeepAWSResources {\n\t\t\tfmt.Print(\"￮ deleting ebs volumes ... \")\n\t\t\tvolumes, err := listPVCVolumesForCluster(awsClient, accessConfig.ClusterName)\n\t\t\tif err != nil {\n\t\t\t\terrorsList = append(errorsList, err)\n\t\t\t\tfmt.Println(\"\\n\\nfailed to list volumes for deletion; please delete any volumes associated with your cluster via the ec2 console: https://console.aws.amazon.com/ec2/v2/home?#Volumes\")\n\t\t\t\terrors.PrintError(err)\n\t\t\t\tfmt.Println()\n\t\t\t} else {\n\t\t\t\tvar failedToDeleteVolumes []string\n\t\t\t\tvar lastErr error\n\t\t\t\tfor _, volume := range volumes {\n\t\t\t\t\terr := awsClient.DeleteVolume(*volume.VolumeId)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tfailedToDeleteVolumes = append(failedToDeleteVolumes, *volume.VolumeId)\n\t\t\t\t\t\tlastErr = err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif len(volumes) == 0 {\n\t\t\t\t\tfmt.Println(\"no ebs volumes exist ✓\")\n\t\t\t\t} else if lastErr != nil {\n\t\t\t\t\terrorsList = append(errorsList, lastErr)\n\t\t\t\t\tfmt.Printf(\"\\n\\nfailed to delete %s %s; please delete %s via the ec2 console: https://console.aws.amazon.com/ec2/v2/home?#Volumes\\n\", s.PluralS(\"volume\", len(failedToDeleteVolumes)), s.UserStrsAnd(failedToDeleteVolumes), s.PluralCustom(\"it\", \"them\", len(failedToDeleteVolumes)))\n\t\t\t\t\terrors.PrintError(lastErr)\n\t\t\t\t\tfmt.Println()\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Println(\"✓\")\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfmt.Printf(\"￮ deleting log group %s ... \", accessConfig.ClusterName)\n\t\t\tlogGroupExists, err := awsClient.DoesLogGroupExist(accessConfig.ClusterName)\n\t\t\tif err != nil {\n\t\t\t\terrorsList = append(errorsList, err)\n\t\t\t\tfmt.Print(\"failed ✗\")\n\t\t\t\tfmt.Printf(\"\\n\\nfailed to list log group for deletion; please delete the log group associated with your cluster via the ec2 console: https://%s.console.aws.amazon.com/cloudwatch/home?#logsV2:log-groups\\n\", accessConfig.Region)\n\t\t\t\terrors.PrintError(err)\n\t\t\t\tfmt.Println()\n\t\t\t} else {\n\t\t\t\tif !logGroupExists {\n\t\t\t\t\tfmt.Println(\"log group doesn't exist ✓\")\n\t\t\t\t} else {\n\t\t\t\t\terr = awsClient.DeleteLogGroup(accessConfig.ClusterName)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\terrorsList = append(errorsList, err)\n\t\t\t\t\t\tfmt.Print(\"failed ✗\")\n\t\t\t\t\t\tfmt.Printf(\"\\n\\nfailed to delete log group %s; please delete the log group associated with your cluster via the ec2 console: https://%s.console.aws.amazon.com/cloudwatch/home?#logsV2:log-groups\\n\", accessConfig.ClusterName, accessConfig.Region)\n\t\t\t\t\t\terrors.PrintError(err)\n\t\t\t\t\t\tfmt.Println()\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfmt.Println(\"✓\")\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// best-effort deletion of cached config\n\t\tcachedClusterConfigPath := getCachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region)\n\t\t_ = os.Remove(cachedClusterConfigPath)\n\n\t\tif len(errorsList) > 0 {\n\t\t\texit.Error(errors.ListOfErrors(ErrClusterDown, false, errorsList...))\n\t\t}\n\t\tfmt.Printf(\"\\nplease check CloudFormation to ensure that all resources for the %s cluster eventually become successfully deleted: %s\\n\", accessConfig.ClusterName, clusterstate.CloudFormationURL(accessConfig.ClusterName, accessConfig.Region))\n\t\tif !_flagClusterDownKeepAWSResources && bucketExists {\n\t\t\tfmt.Printf(\"\\na lifecycle rule has been applied to the cluster's %s bucket to empty its contents within the next 24 hours; you can delete the %s bucket via the s3 console once it has been emptied (or you can empty and delete it now): https://s3.console.aws.amazon.com/s3/management/%s\\n\", bucketName, bucketName, bucketName)\n\t\t}\n\t\tfmt.Println()\n\n\t\t// best-effort deletion of cli environment(s)\n\t\tif loadBalancer != nil {\n\t\t\tenvNames, isDefaultEnv, _ := getEnvNamesByOperatorEndpoint(*loadBalancer.DNSName)\n\t\t\tif len(envNames) > 0 {\n\t\t\t\tfor _, envName := range envNames {\n\t\t\t\t\terr := removeEnvFromCLIConfig(envName)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\texit.Error(err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfmt.Printf(\"deleted the %s environment configuration%s\\n\", s.StrsAnd(envNames), s.SIfPlural(len(envNames)))\n\t\t\t\tif isDefaultEnv {\n\t\t\t\t\tnewDefaultEnv, err := getDefaultEnv()\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\texit.Error(err)\n\t\t\t\t\t}\n\t\t\t\t\tif newDefaultEnv != nil {\n\t\t\t\t\t\tfmt.Println(fmt.Sprintf(\"set the default environment to %s\", *newDefaultEnv))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t},\n}\n\nvar _clusterExportCmd = &cobra.Command{\n\tUse:   \"export\",\n\tShort: \"download the configurations for all APIs\",\n\tArgs:  cobra.NoArgs,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.cluster.export\")\n\n\t\taccessConfig, err := getClusterAccessConfigWithCache(true)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\t// Check AWS access\n\t\tawsClient, err := newAWSClient(accessConfig.Region, true)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\twarnIfNotAdmin(awsClient)\n\n\t\tstacks, err := clusterstate.GetClusterStacks(awsClient, accessConfig)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tstate := clusterstate.GetClusterState(stacks)\n\t\tif err := clusterstate.AssertClusterState(stacks, state, clusterstate.StateClusterExists); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tloadBalancer, err := getNLBLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\toperatorConfig := cluster.OperatorConfig{\n\t\t\tTelemetry:        isTelemetryEnabled(),\n\t\t\tClientID:         clientID(),\n\t\t\tOperatorEndpoint: \"https://\" + *loadBalancer.DNSName,\n\t\t}\n\n\t\tvar apisResponse []schema.APIResponse\n\t\tapisResponse, err = cluster.GetAPIs(operatorConfig)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tif len(apisResponse) == 0 {\n\t\t\tfmt.Println(fmt.Sprintf(\"no apis found in your cluster named %s in %s\", accessConfig.ClusterName, accessConfig.Region))\n\t\t\texit.Ok()\n\t\t}\n\n\t\texportPath := fmt.Sprintf(\"export-%s-%s\", accessConfig.Region, accessConfig.ClusterName)\n\n\t\terr = files.CreateDir(exportPath)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tfor _, api := range apisResponse {\n\t\t\tapisWithSpec, err := cluster.GetAPI(operatorConfig, api.Metadata.Name)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\n\t\t\tspecFilePath := filepath.Join(exportPath, api.Metadata.Name+\".yaml\")\n\t\t\tfmt.Println(fmt.Sprintf(\"exporting %s to %s\", api.Metadata.Name, specFilePath))\n\n\t\t\tyamlBytes, err := yaml.Marshal(apisWithSpec[0].Spec.API.SubmittedAPISpec)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\n\t\t\terr = files.WriteFile(yamlBytes, specFilePath)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t}\n\t},\n}\n\nvar _clusterHealthCmd = &cobra.Command{\n\tUse:   \"health\",\n\tShort: \"inspect the health of components in the cluster\",\n\tArgs:  cobra.NoArgs,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\taccessConfig, err := getClusterAccessConfigWithCache(true)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tawsClient, err := awslib.NewForRegion(accessConfig.Region)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\trestConfig, err := getClusterRESTConfig(awsClient, accessConfig.ClusterName)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tscheme := runtime.NewScheme()\n\t\tif err := clientgoscheme.AddToScheme(scheme); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tk8sClient, err := k8s.New(consts.DefaultNamespace, false, restConfig, scheme)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tclusterHealth, err := health.Check(awsClient, k8sClient, accessConfig.ClusterName)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tclusterWarnings, err := health.GetWarnings(k8sClient)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tif _flagOutput == flags.JSONOutputType {\n\t\t\tfmt.Println(clusterHealth)\n\t\t\treturn\n\t\t}\n\n\t\thealthTable := table.Table{\n\t\t\tHeaders: []table.Header{\n\t\t\t\t{Title: \"\"},\n\t\t\t\t{Title: \"live\"},\n\t\t\t\t{Title: \"warning\", Hidden: !clusterWarnings.HasWarnings()},\n\t\t\t},\n\t\t\tRows: [][]interface{}{\n\t\t\t\t{\"operator\", console.BoolColor(clusterHealth.Operator), \"\"},\n\t\t\t\t{\"prometheus\", console.BoolColor(clusterHealth.Prometheus), clusterWarnings.Prometheus},\n\t\t\t\t{\"autoscaler\", console.BoolColor(clusterHealth.Autoscaler), \"\"},\n\t\t\t\t{\"activator\", console.BoolColor(clusterHealth.Activator), \"\"},\n\t\t\t\t{\"async gateway\", console.BoolColor(clusterHealth.AsyncGateway), \"\"},\n\t\t\t\t{\"grafana\", console.BoolColor(clusterHealth.Grafana), \"\"},\n\t\t\t\t{\"controller manager\", console.BoolColor(clusterHealth.ControllerManager), \"\"},\n\t\t\t\t{\"apis gateway\", console.BoolColor(clusterHealth.APIsGateway), \"\"},\n\t\t\t\t{\"operator gateway\", console.BoolColor(clusterHealth.APIsGateway), \"\"},\n\t\t\t\t{\"cluster autoscaler\", console.BoolColor(clusterHealth.ClusterAutoscaler), \"\"},\n\t\t\t\t{\"operator load balancer\", console.BoolColor(clusterHealth.OperatorLoadBalancer), \"\"},\n\t\t\t\t{\"apis load balancer\", console.BoolColor(clusterHealth.APIsLoadBalancer), \"\"},\n\t\t\t\t{\"fluent bit\", console.BoolColor(clusterHealth.FluentBit), \"\"},\n\t\t\t\t{\"node exporter\", console.BoolColor(clusterHealth.NodeExporter), \"\"},\n\t\t\t\t{\"dcgm exporter\", console.BoolColor(clusterHealth.DCGMExporter), \"\"},\n\t\t\t\t{\"statsd exporter\", console.BoolColor(clusterHealth.StatsDExporter), \"\"},\n\t\t\t\t{\"event exporter\", console.BoolColor(clusterHealth.EventExporter), \"\"},\n\t\t\t\t{\"kube state metrics\", console.BoolColor(clusterHealth.KubeStateMetrics), \"\"},\n\t\t\t},\n\t\t}\n\n\t\tfmt.Println(healthTable.MustFormat())\n\t},\n}\n\nfunc cmdPrintConfig(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig, outputType flags.OutputType) {\n\tclusterConfig := refreshCachedClusterConfig(awsClient, accessConfig, outputType == flags.PrettyOutputType)\n\n\tinfoInterface := clusterConfig.CoreConfig\n\n\tif outputType == flags.JSONOutputType {\n\t\toutputBytes, err := libjson.Marshal(infoInterface)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tfmt.Println(string(outputBytes))\n\t} else {\n\t\toutputBytes, err := yaml.Marshal(infoInterface)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tfmt.Println(string(outputBytes))\n\t}\n}\n\nfunc cmdInfo(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig, stacks clusterstate.ClusterStacks, outputType flags.OutputType, disallowPrompt bool) {\n\tclusterConfig := refreshCachedClusterConfig(awsClient, accessConfig, outputType == flags.PrettyOutputType)\n\n\toperatorLoadBalancer, err := getNLBLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient)\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\toperatorEndpoint := s.EnsurePrefix(*operatorLoadBalancer.DNSName, \"https://\")\n\n\tvar apiEndpoint string\n\tif clusterConfig.APILoadBalancerType == clusterconfig.NLBLoadBalancerType {\n\t\tapiLoadBalancer, err := getNLBLoadBalancer(accessConfig.ClusterName, APILoadBalancer, awsClient)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tapiEndpoint = *apiLoadBalancer.DNSName\n\t}\n\tif clusterConfig.APILoadBalancerType == clusterconfig.ELBLoadBalancerType {\n\t\tapiLoadBalancer, err := getELBLoadBalancer(accessConfig.ClusterName, APILoadBalancer, awsClient)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tapiEndpoint = *apiLoadBalancer.DNSName\n\t}\n\n\tif outputType == flags.JSONOutputType || outputType == flags.YAMLOutputType {\n\t\tinfoResponse, err := getInfoOperatorResponse(operatorEndpoint)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tinfoResponse.ClusterConfig.Config = clusterConfig\n\n\t\tinfoInterface := map[string]interface{}{\n\t\t\t\"cluster_config\":      infoResponse.ClusterConfig.Config,\n\t\t\t\"cluster_metadata\":    infoResponse.ClusterConfig.OperatorMetadata,\n\t\t\t\"worker_node_infos\":   infoResponse.WorkerNodeInfos,\n\t\t\t\"operator_node_infos\": infoResponse.OperatorNodeInfos,\n\t\t\t\"endpoint_operator\":   operatorEndpoint,\n\t\t\t\"endpoint_api\":        apiEndpoint,\n\t\t}\n\n\t\tvar outputBytes []byte\n\t\tif outputType == flags.JSONOutputType {\n\t\t\toutputBytes, err = libjson.Marshal(infoInterface)\n\t\t} else {\n\t\t\toutputBytes, err = yaml.Marshal(infoInterface)\n\t\t}\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tfmt.Println(string(outputBytes))\n\t}\n\tif outputType == flags.PrettyOutputType {\n\t\tfmt.Println(console.Bold(\"endpoints:\"))\n\t\tfmt.Println(\"operator:         \", operatorEndpoint)\n\t\tfmt.Println(\"api load balancer:\", apiEndpoint)\n\t\tfmt.Println()\n\n\t\tif err := printInfoOperatorResponse(clusterConfig, stacks, operatorEndpoint); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t}\n\n\tif _flagClusterInfoEnv != \"\" {\n\t\tif err := updateCLIEnv(_flagClusterInfoEnv, operatorEndpoint, disallowPrompt, outputType == flags.PrettyOutputType); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t}\n}\n\nfunc printInfoOperatorResponse(clusterConfig clusterconfig.Config, stacks clusterstate.ClusterStacks, operatorEndpoint string) error {\n\tfmt.Print(\"fetching cluster status ...\\n\\n\")\n\n\tfmt.Println(stacks.TableString())\n\n\tyamlBytes, err := yaml.Marshal(clusterConfig)\n\tif err != nil {\n\t\treturn err\n\t}\n\tyamlString := string(yamlBytes)\n\n\tinfoResponse, err := getInfoOperatorResponse(operatorEndpoint)\n\tif err != nil {\n\t\tfmt.Println(yamlString)\n\t\treturn err\n\t}\n\tinfoResponse.ClusterConfig.Config = clusterConfig\n\n\tfmt.Println(console.Bold(\"cluster config:\"))\n\tfmt.Println(fmt.Sprintf(\"cluster version: %s\", infoResponse.ClusterConfig.APIVersion))\n\tfmt.Print(yamlString)\n\n\tprintInfoPricing(infoResponse, clusterConfig)\n\tprintInfoNodes(infoResponse)\n\n\treturn nil\n}\n\nfunc getInfoOperatorResponse(operatorEndpoint string) (*schema.InfoResponse, error) {\n\toperatorConfig := cluster.OperatorConfig{\n\t\tTelemetry:        isTelemetryEnabled(),\n\t\tClientID:         clientID(),\n\t\tOperatorEndpoint: operatorEndpoint,\n\t}\n\treturn cluster.Info(operatorConfig)\n}\n\nfunc printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterconfig.Config) {\n\teksPrice := awslib.EKSPrices[clusterConfig.Region]\n\toperatorInstancePrice := awslib.InstanceMetadatas[clusterConfig.Region][\"t3.medium\"].Price\n\toperatorEBSPrice := awslib.EBSMetadatas[clusterConfig.Region][\"gp3\"].PriceGB * 20 / 30 / 24\n\tprometheusInstancePrice := awslib.InstanceMetadatas[clusterConfig.Region][clusterConfig.PrometheusInstanceType].Price\n\tprometheusEBSPrice := awslib.EBSMetadatas[clusterConfig.Region][\"gp3\"].PriceGB * 20 / 30 / 24\n\tmetricsEBSPrice := awslib.EBSMetadatas[clusterConfig.Region][\"gp2\"].PriceGB * (40 + 2) / 30 / 24\n\tnlbPrice := awslib.NLBMetadatas[clusterConfig.Region].Price\n\telbPrice := awslib.ELBMetadatas[clusterConfig.Region].Price\n\tnatUnitPrice := awslib.NATMetadatas[clusterConfig.Region].Price\n\n\tvar loadBalancersPrice float64\n\tusesELBForAPILoadBalancer := clusterConfig.APILoadBalancerType == clusterconfig.ELBLoadBalancerType\n\tif usesELBForAPILoadBalancer {\n\t\tloadBalancersPrice = nlbPrice + elbPrice\n\t} else {\n\t\tloadBalancersPrice = 2 * nlbPrice\n\t}\n\n\theaders := []table.Header{\n\t\t{Title: \"aws resource\"},\n\t\t{Title: \"cost per hour\"},\n\t}\n\n\tvar rows [][]interface{}\n\trows = append(rows, []interface{}{\"1 eks cluster\", s.DollarsMaxPrecision(eksPrice)})\n\n\tvar totalNodeGroupsPrice float64\n\tfor _, ng := range clusterConfig.NodeGroups {\n\t\tvar ngNamePrefix string\n\t\tif ng.Spot {\n\t\t\tngNamePrefix = \"cx-ws-\"\n\t\t} else {\n\t\t\tngNamePrefix = \"cx-wd-\"\n\t\t}\n\t\tnodesInfo := infoResponse.GetNodesWithNodeGroupName(ngNamePrefix + ng.Name)\n\t\tnumInstances := len(nodesInfo)\n\n\t\tebsPrice := awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24\n\t\tif ng.InstanceVolumeType == clusterconfig.IO1VolumeType && ng.InstanceVolumeIOPS != nil {\n\t\t\tebsPrice += awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24\n\t\t}\n\t\tif ng.InstanceVolumeType == clusterconfig.GP3VolumeType && ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput != nil {\n\t\t\tebsPrice += libmath.MaxFloat64(0, (awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS-3000)*float64(*ng.InstanceVolumeIOPS)/30/24)\n\t\t\tebsPrice += libmath.MaxFloat64(0, (awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceThroughput-125)*float64(*ng.InstanceVolumeThroughput)/30/24)\n\t\t}\n\t\ttotalEBSPrice := ebsPrice * float64(numInstances)\n\n\t\ttotalInstancePrice := float64(0)\n\t\tfor _, nodeInfo := range nodesInfo {\n\t\t\ttotalInstancePrice += nodeInfo.Price\n\t\t}\n\n\t\trows = append(rows, []interface{}{fmt.Sprintf(\"nodegroup %s: %d (out of %d) %s\", ng.Name, numInstances, ng.MaxInstances, s.PluralS(\"instance\", numInstances)), s.DollarsAndTenthsOfCents(totalInstancePrice+totalEBSPrice) + \" total\"})\n\n\t\ttotalNodeGroupsPrice += totalEBSPrice + totalInstancePrice\n\t}\n\n\toperatorNodeGroupPrice := float64(len(infoResponse.OperatorNodeInfos)) * (operatorInstancePrice + operatorEBSPrice)\n\tprometheusNodeGroupPrice := prometheusInstancePrice + prometheusEBSPrice + metricsEBSPrice\n\n\tvar natTotalPrice float64\n\tif clusterConfig.NATGateway == clusterconfig.SingleNATGateway {\n\t\tnatTotalPrice = natUnitPrice\n\t} else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway {\n\t\tnatTotalPrice = natUnitPrice * float64(len(clusterConfig.AvailabilityZones))\n\t}\n\ttotalPrice := eksPrice + totalNodeGroupsPrice + operatorNodeGroupPrice + prometheusNodeGroupPrice + loadBalancersPrice + natTotalPrice\n\tfmt.Printf(console.Bold(\"\\nyour cluster currently costs %s per hour\\n\\n\"), s.DollarsAndCents(totalPrice))\n\n\trows = append(rows, []interface{}{fmt.Sprintf(\"%d t3.medium %s (cortex system)\", len(infoResponse.OperatorNodeInfos), s.PluralS(\"instance\", len(infoResponse.OperatorNodeInfos))), s.DollarsAndTenthsOfCents(operatorNodeGroupPrice) + \" total\"})\n\trows = append(rows, []interface{}{fmt.Sprintf(\"1 %s instance (prometheus)\", clusterConfig.PrometheusInstanceType), s.DollarsAndTenthsOfCents(prometheusNodeGroupPrice)})\n\tif usesELBForAPILoadBalancer {\n\t\trows = append(rows, []interface{}{\"1 network load balancer\", s.DollarsMaxPrecision(nlbPrice)})\n\t\trows = append(rows, []interface{}{\"1 classic load balancer\", s.DollarsMaxPrecision(elbPrice)})\n\t} else {\n\t\trows = append(rows, []interface{}{\"2 network load balancers\", s.DollarsMaxPrecision(loadBalancersPrice) + \" total\"})\n\t}\n\n\tif clusterConfig.NATGateway == clusterconfig.SingleNATGateway {\n\t\trows = append(rows, []interface{}{\"1 nat gateway\", s.DollarsMaxPrecision(natUnitPrice)})\n\t} else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway {\n\t\tnumNATs := len(clusterConfig.AvailabilityZones)\n\t\trows = append(rows, []interface{}{fmt.Sprintf(\"%d nat gateways\", numNATs), s.DollarsMaxPrecision(natUnitPrice*float64(numNATs)) + \" total\"})\n\t}\n\n\tt := table.Table{\n\t\tHeaders: headers,\n\t\tRows:    rows,\n\t}\n\tt.MustPrint(&table.Opts{Sort: pointer.Bool(false)})\n}\n\nfunc printInfoNodes(infoResponse *schema.InfoResponse) {\n\tnumAPIInstances := len(infoResponse.WorkerNodeInfos)\n\n\tvar totalReplicas int\n\tvar doesClusterHaveGPUs, doesClusterHaveInfs, doesClusterHaveEnqueuers bool\n\tfor _, nodeInfo := range infoResponse.WorkerNodeInfos {\n\t\ttotalReplicas += nodeInfo.NumReplicas\n\t\tif nodeInfo.ComputeUserCapacity.GPU > 0 {\n\t\t\tdoesClusterHaveGPUs = true\n\t\t}\n\t\tif nodeInfo.ComputeUserCapacity.Inf > 0 {\n\t\t\tdoesClusterHaveInfs = true\n\t\t}\n\t\tif nodeInfo.NumEnqueuerReplicas > 0 {\n\t\t\tdoesClusterHaveEnqueuers = true\n\t\t}\n\t}\n\n\tvar pendingReplicasStr string\n\tif infoResponse.NumPendingReplicas > 0 {\n\t\tpendingReplicasStr = fmt.Sprintf(\", and %d unscheduled %s\", infoResponse.NumPendingReplicas, s.PluralS(\"replica\", infoResponse.NumPendingReplicas))\n\t}\n\n\tfmt.Printf(console.Bold(\"\\nyour cluster has %d API %s running across %d %s%s\\n\"), totalReplicas, s.PluralS(\"replica\", totalReplicas), numAPIInstances, s.PluralS(\"instance\", numAPIInstances), pendingReplicasStr)\n\n\tif len(infoResponse.WorkerNodeInfos) == 0 {\n\t\treturn\n\t}\n\n\theaders := []table.Header{\n\t\t{Title: \"instance type\"},\n\t\t{Title: \"lifecycle\"},\n\t\t{Title: \"replicas\"},\n\t\t{Title: \"batch enqueuer replicas\", Hidden: !doesClusterHaveEnqueuers},\n\t\t{Title: \"CPU (requested / total allocatable)\"},\n\t\t{Title: \"memory (requested / total allocatable)\"},\n\t\t{Title: \"GPU (requested / total allocatable)\", Hidden: !doesClusterHaveGPUs},\n\t\t{Title: \"Inf (requested / total allocatable)\", Hidden: !doesClusterHaveInfs},\n\t}\n\n\tvar rows [][]interface{}\n\tfor _, nodeInfo := range infoResponse.WorkerNodeInfos {\n\t\tlifecycle := \"on-demand\"\n\t\tif nodeInfo.IsSpot {\n\t\t\tlifecycle = \"spot\"\n\t\t}\n\n\t\tcpuStr := nodeInfo.ComputeUserRequested.CPU.MilliString() + \" / \" + nodeInfo.ComputeUserCapacity.CPU.MilliString()\n\t\tmemStr := nodeInfo.ComputeUserRequested.Mem.String() + \" / \" + nodeInfo.ComputeUserCapacity.Mem.String()\n\t\tgpuStr := s.Int64(nodeInfo.ComputeUserRequested.GPU) + \" / \" + s.Int64(nodeInfo.ComputeUserCapacity.GPU)\n\t\tinfStr := s.Int64(nodeInfo.ComputeUserRequested.Inf) + \" / \" + s.Int64(nodeInfo.ComputeUserCapacity.Inf)\n\t\trows = append(rows, []interface{}{nodeInfo.InstanceType, lifecycle, nodeInfo.NumReplicas, nodeInfo.NumEnqueuerReplicas, cpuStr, memStr, gpuStr, infStr})\n\t}\n\n\tt := table.Table{\n\t\tHeaders: headers,\n\t\tRows:    rows,\n\t}\n\tfmt.Println()\n\tt.MustPrint(&table.Opts{Sort: pointer.Bool(false)})\n}\n\nfunc updateCLIEnv(envName string, operatorEndpoint string, disallowPrompt bool, printToStdout bool) error {\n\tprevEnv, err := readEnv(envName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tnewEnvironment := cliconfig.Environment{\n\t\tName:             envName,\n\t\tOperatorEndpoint: operatorEndpoint,\n\t}\n\n\tshouldWriteEnv := false\n\tenvWasUpdated := false\n\tif prevEnv == nil {\n\t\tshouldWriteEnv = true\n\t\tif printToStdout {\n\t\t\tfmt.Println()\n\t\t}\n\t} else if prevEnv.OperatorEndpoint != operatorEndpoint {\n\t\tenvWasUpdated = true\n\t\tif printToStdout {\n\t\t\tif disallowPrompt {\n\t\t\t\tshouldWriteEnv = true\n\t\t\t\tfmt.Println()\n\t\t\t} else {\n\t\t\t\tshouldWriteEnv = prompt.YesOrNo(fmt.Sprintf(\"\\nfound an existing environment named \\\"%s\\\"; would you like to overwrite it to connect to this cluster?\", envName), \"\", \"\")\n\t\t\t}\n\t\t} else {\n\t\t\tshouldWriteEnv = true\n\t\t}\n\t}\n\n\tif shouldWriteEnv {\n\t\terr := addEnvToCLIConfig(newEnvironment, true)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif printToStdout {\n\t\t\tif envWasUpdated {\n\t\t\t\tfmt.Printf(console.Bold(\"the environment named \\\"%s\\\" has been updated to point to this cluster (and was set as the default environment)\\n\"), envName)\n\t\t\t} else {\n\t\t\t\tfmt.Printf(console.Bold(\"an environment named \\\"%s\\\" has been configured to point to this cluster (and was set as the default environment)\\n\"), envName)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc cmdDebug(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig) {\n\t// note: if modifying this string, also change it in files.IgnoreCortexDebug()\n\tdebugFileName := fmt.Sprintf(\"cortex-debug-%s.tgz\", time.Now().UTC().Format(\"2006-01-02-15-04-05\"))\n\n\tcontainerDebugPath := \"/out/\" + debugFileName\n\tcopyFromPaths := []dockerCopyFromPath{\n\t\t{\n\t\t\tcontainerPath: containerDebugPath,\n\t\t\tlocalDir:      _cwd,\n\t\t},\n\t}\n\n\tout, exitCode, err := runManagerAccessCommand(\"/root/debug.sh \"+containerDebugPath, *accessConfig, awsClient, nil, copyFromPaths)\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\tif exitCode == nil || *exitCode != 0 {\n\t\texit.Error(ErrorClusterDebug(out))\n\t}\n\n\tfmt.Println(\"saved cluster info to ./\" + debugFileName)\n\treturn\n}\n\nfunc refreshCachedClusterConfig(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig, printToStdout bool) clusterconfig.Config {\n\t// add empty file if cached cluster doesn't exist so that the file output by manager container maintains current user permissions\n\tcachedClusterConfigPath := getCachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region)\n\tcontainerConfigPath := fmt.Sprintf(\"/out/%s\", filepath.Base(cachedClusterConfigPath))\n\n\tcopyFromPaths := []dockerCopyFromPath{\n\t\t{\n\t\t\tcontainerPath: containerConfigPath,\n\t\t\tlocalDir:      files.Dir(cachedClusterConfigPath),\n\t\t},\n\t}\n\n\tif printToStdout {\n\t\tfmt.Print(\"syncing cluster configuration ...\\n\\n\")\n\t}\n\tout, exitCode, err := runManagerAccessCommand(\"/root/refresh.sh \"+containerConfigPath, *accessConfig, awsClient, nil, copyFromPaths)\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\tif exitCode == nil || *exitCode != 0 {\n\t\texit.Error(ErrorClusterRefresh(out))\n\t}\n\n\trefreshedClusterConfig := &clusterconfig.Config{}\n\terr = readCachedClusterConfigFile(refreshedClusterConfig, cachedClusterConfigPath)\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\treturn *refreshedClusterConfig\n}\n\nfunc createS3BucketIfNotFound(awsClient *awslib.Client, bucket string, tags map[string]string) error {\n\tbucketFound, err := awsClient.DoesBucketExist(bucket)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !bucketFound {\n\t\tfmt.Print(\"￮ creating a new s3 bucket: \", bucket)\n\t\terr = awsClient.CreateBucket(bucket)\n\t\tif err != nil {\n\t\t\tfmt.Print(\"\\n\\n\")\n\t\t\treturn err\n\t\t}\n\t\terr = awsClient.EnableBucketEncryption(bucket)\n\t\tif err != nil {\n\t\t\tfmt.Print(\"\\n\\n\")\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tfmt.Print(\"￮ using existing s3 bucket: \", bucket)\n\t}\n\n\t// retry since it's possible that it takes some time for the new bucket to be registered by AWS\n\tfor i := 0; i < 10; i++ {\n\t\terr = awsClient.TagBucket(bucket, tags)\n\t\tif err == nil {\n\t\t\tfmt.Println(\" ✓\")\n\t\t\treturn nil\n\t\t}\n\t\tif !awslib.IsNoSuchBucketErr(err) {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(1 * time.Second)\n\t}\n\n\tfmt.Print(\"\\n\\n\")\n\treturn err\n}\n\nfunc setLifecycleRulesOnClusterUp(awsClient *awslib.Client, bucket, newClusterUID string) error {\n\terr := awsClient.DeleteLifecycleRules(bucket)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tclusterUIDs, err := awsClient.ListS3TopLevelDirs(bucket)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif len(clusterUIDs)+1 > consts.MaxBucketLifecycleRules {\n\t\treturn ErrorClusterUIDsLimitInBucket(bucket)\n\t}\n\n\texpirationDate := libtime.GetCurrentUTCDate().Add(-24 * time.Hour)\n\trules := []s3.LifecycleRule{}\n\tfor _, clusterUID := range clusterUIDs {\n\t\trules = append(rules, s3.LifecycleRule{\n\t\t\tExpiration: &s3.LifecycleExpiration{\n\t\t\t\tDate: &expirationDate,\n\t\t\t},\n\t\t\tID: pointer.String(\"cluster-remove-\" + clusterUID),\n\t\t\tFilter: &s3.LifecycleRuleFilter{\n\t\t\t\tPrefix: pointer.String(s.EnsureSuffix(clusterUID, \"/\")),\n\t\t\t},\n\t\t\tStatus: pointer.String(\"Enabled\"),\n\t\t})\n\t}\n\n\trules = append(rules, s3.LifecycleRule{\n\t\tExpiration: &s3.LifecycleExpiration{\n\t\t\tDays: pointer.Int64(consts.AsyncWorkloadsExpirationDays),\n\t\t},\n\t\tID: pointer.String(\"async-workloads-expiry-policy\"),\n\t\tFilter: &s3.LifecycleRuleFilter{\n\t\t\tPrefix: pointer.String(s.EnsureSuffix(filepath.Join(newClusterUID, \"workloads\"), \"/\")),\n\t\t},\n\t\tStatus: pointer.String(\"Enabled\"),\n\t})\n\n\treturn awsClient.SetLifecycleRules(bucket, rules)\n}\n\nfunc setLifecycleRulesOnClusterDown(awsClient *awslib.Client, bucket string) error {\n\terr := awsClient.DeleteLifecycleRules(bucket)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\texpirationDate := libtime.GetCurrentUTCDate().Add(-24 * time.Hour)\n\treturn awsClient.SetLifecycleRules(bucket, []s3.LifecycleRule{\n\t\t{\n\t\t\tExpiration: &s3.LifecycleExpiration{\n\t\t\t\tDate: &expirationDate,\n\t\t\t},\n\t\t\tID: pointer.String(\"bucket-cleaner\"),\n\t\t\tFilter: &s3.LifecycleRuleFilter{\n\t\t\t\tPrefix: pointer.String(\"\"),\n\t\t\t},\n\t\t\tStatus: pointer.String(\"Enabled\"),\n\t\t},\n\t})\n}\n\nfunc createLogGroupIfNotFound(awsClient *awslib.Client, logGroup string, tags map[string]string) error {\n\tlogGroupFound, err := awsClient.DoesLogGroupExist(logGroup)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !logGroupFound {\n\t\tfmt.Print(\"￮ creating a new cloudwatch log group: \", logGroup)\n\t\terr = awsClient.CreateLogGroup(logGroup, tags)\n\t\tif err != nil {\n\t\t\tfmt.Print(\"\\n\\n\")\n\t\t\treturn err\n\t\t}\n\t\tfmt.Println(\" ✓\")\n\t\treturn nil\n\t}\n\n\tfmt.Print(\"￮ using existing cloudwatch log group: \", logGroup)\n\n\t// retry since it's possible that it takes some time for the new log group to be registered by AWS\n\terr = awsClient.TagLogGroup(logGroup, tags)\n\tif err != nil {\n\t\tfmt.Print(\"\\n\\n\")\n\t\treturn err\n\t}\n\n\tfmt.Println(\" ✓\")\n\n\treturn nil\n}\n\ntype LoadBalancer string\n\nvar (\n\tOperatorLoadBalancer LoadBalancer = \"operator\"\n\tAPILoadBalancer      LoadBalancer = \"api\"\n)\n\nfunc (lb LoadBalancer) String() string {\n\treturn string(lb)\n}\n\n// Will return error if the load balancer can't be found\nfunc getNLBLoadBalancer(clusterName string, whichLB LoadBalancer, awsClient *awslib.Client) (*elbv2.LoadBalancer, error) {\n\tloadBalancer, err := awsClient.FindLoadBalancerV2(map[string]string{\n\t\tclusterconfig.ClusterNameTag: clusterName,\n\t\t\"cortex.dev/load-balancer\":   whichLB.String(),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"unable to locate %s load balancer\", whichLB.String()))\n\t}\n\n\tif loadBalancer == nil {\n\t\treturn nil, ErrorNoOperatorLoadBalancer(whichLB.String())\n\t}\n\n\treturn loadBalancer, nil\n}\n\n// Will return error if the load balancer can't be found\nfunc getELBLoadBalancer(clusterName string, whichLB LoadBalancer, awsClient *awslib.Client) (*elb.LoadBalancerDescription, error) {\n\tloadBalancer, err := awsClient.FindLoadBalancer(map[string]string{\n\t\tclusterconfig.ClusterNameTag: clusterName,\n\t\t\"cortex.dev/load-balancer\":   whichLB.String(),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"unable to locate %s load balancer\", whichLB.String()))\n\t}\n\n\tif loadBalancer == nil {\n\t\treturn nil, ErrorNoOperatorLoadBalancer(whichLB.String())\n\t}\n\n\treturn loadBalancer, nil\n}\n\nfunc listPVCVolumesForCluster(awsClient *awslib.Client, clusterName string) ([]ec2.Volume, error) {\n\treturn awsClient.ListVolumes(ec2.Tag{\n\t\tKey:   pointer.String(fmt.Sprintf(\"kubernetes.io/cluster/%s\", clusterName)),\n\t\tValue: nil, // any value should be ok as long as the key is present\n\t})\n}\n\nfunc filterEKSCTLOutput(out string) string {\n\treturn strings.Join(s.RemoveDuplicates(strings.Split(out, \"\\n\"), _eksctlPrefixRegex), \"\\n\")\n}\n\nfunc getClusterRESTConfig(awsClient *awslib.Client, clusterName string) (*rest.Config, error) {\n\tclusterOutput, err := awsClient.EKS().DescribeCluster(\n\t\t&eks.DescribeClusterInput{\n\t\t\tName: aws.String(clusterName),\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tgen, err := token.NewGenerator(true, false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\topts := &token.GetTokenOptions{\n\t\tClusterID: aws.StringValue(clusterOutput.Cluster.Name),\n\t}\n\n\ttok, err := gen.GetWithOptions(opts)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tca, err := base64.StdEncoding.DecodeString(aws.StringValue(clusterOutput.Cluster.CertificateAuthority.Data))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &rest.Config{\n\t\tHost:        aws.StringValue(clusterOutput.Cluster.Endpoint),\n\t\tBearerToken: tok.Token,\n\t\tTLSClientConfig: rest.TLSClientConfig{\n\t\t\tCAData: ca,\n\t\t},\n\t}, nil\n}\n"
  },
  {
    "path": "cli/cmd/completion.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc completionInit() {\n\t_completionCmd.Flags().SortFlags = false\n}\n\nvar _bashAliasText = `\n# alias\n\nalias cx='cortex'\nif [[ $(type -t compopt) = \"builtin\" ]]; then\n    complete -o default -F __start_cortex cx\nelse\n    complete -o default -o nospace -F __start_cortex cx\nfi\n`\n\nvar _completionCmd = &cobra.Command{\n\tUse:   \"completion SHELL\",\n\tShort: \"generate shell completion scripts\",\n\tLong: `generate shell completion scripts\n\nto enable cortex shell completion:\n    bash:\n        add this to ~/.bash_profile (mac) or ~/.bashrc (linux):\n            source <(cortex completion bash)\n\n        note: bash-completion must be installed on your system; example installation instructions:\n            mac:\n                1) install bash completion:\n                   brew install bash-completion\n                2) add this to your ~/.bash_profile:\n                   source $(brew --prefix)/etc/bash_completion\n                3) log out and back in, or close your terminal window and reopen it\n            ubuntu:\n                1) install bash completion:\n                   apt update && apt install -y bash-completion  # you may need sudo\n                2) open ~/.bashrc and uncomment the bash completion section, or add this:\n                   if [ -f /etc/bash_completion ] && ! shopt -oq posix; then . /etc/bash_completion; fi\n                3) log out and back in, or close your terminal window and reopen it\n\n    zsh:\n        option 1:\n            add this to ~/.zshrc:\n                source <(cortex completion zsh)\n            if that failed, you can try adding this line (above the source command you just added):\n                autoload -Uz compinit && compinit\n        option 2:\n            create a _cortex file in your fpath, for example:\n                cortex completion zsh > /usr/local/share/zsh/site-functions/_cortex\n\nNote: this will also add the \"cx\" alias for cortex for convenience\n`,\n\tArgs:      cobra.ExactArgs(1),\n\tValidArgs: []string{\"bash\", \"zsh\"},\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tswitch args[0] {\n\t\tcase \"bash\":\n\t\t\t_rootCmd.GenBashCompletion(os.Stdout)\n\t\t\tfmt.Print(_bashAliasText)\n\n\t\tcase \"zsh\":\n\t\t\t_rootCmd.GenZshCompletion(os.Stdout)\n\t\t\tfmt.Print(\"alias cx='cortex'\\n\\n\")\n\n\t\t\t// https://github.com/spf13/cobra/pull/887\n\t\t\t// https://github.com/corneliusweig/rakkess/blob/master/cmd/completion.go\n\t\t\t// https://github.com/GoogleContainerTools/skaffold/blob/master/cmd/skaffold/app/cmd/completion.go\n\t\t\t// https://github.com/spf13/cobra/issues/881\n\t\t\t// https://github.com/asdf-vm/asdf/issues/266\n\t\t\tfmt.Println(\"if compquote '' 2>/dev/null; then _cortex; else compdef _cortex cortex; fi\")\n\n\t\tdefault:\n\t\t\tfmt.Println()\n\t\t\texit.Error(ErrorShellCompletionNotSupported(args[0]))\n\t\t}\n\t},\n}\n"
  },
  {
    "path": "cli/cmd/const.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nconst (\n\t_timeFormat = \"02 Jan 06 15:04:05 MST\"\n)\n"
  },
  {
    "path": "cli/cmd/delete.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/print\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar (\n\t_flagDeleteEnv       string\n\t_flagDeleteKeepCache bool\n\t_flagDeleteForce     bool\n)\n\nfunc deleteInit() {\n\t_deleteCmd.Flags().SortFlags = false\n\t_deleteCmd.Flags().StringVarP(&_flagDeleteEnv, \"env\", \"e\", \"\", \"environment to use\")\n\n\t_deleteCmd.Flags().BoolVarP(&_flagDeleteForce, \"force\", \"f\", false, \"delete the api without confirmation\")\n\t_deleteCmd.Flags().BoolVarP(&_flagDeleteKeepCache, \"keep-cache\", \"c\", false, \"keep cached data for the api\")\n\t_deleteCmd.Flags().VarP(&_flagOutput, \"output\", \"o\", fmt.Sprintf(\"output format: one of %s\", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), \"|\")))\n}\n\nvar _deleteCmd = &cobra.Command{\n\tUse:   \"delete API_NAME [JOB_ID]\",\n\tShort: \"delete an api or stop a job\",\n\tArgs:  cobra.RangeArgs(1, 2),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tenvName, err := getEnvFromFlag(_flagDeleteEnv)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.delete\")\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.delete\")\n\t\t\texit.Error(err)\n\t\t}\n\t\ttelemetry.Event(\"cli.delete\", map[string]interface{}{\"env_name\": env.Name})\n\n\t\terr = printEnvIfNotSpecified(env.Name, cmd)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tvar deleteResponse schema.DeleteResponse\n\t\tif len(args) == 2 {\n\t\t\tapisRes, err := cluster.GetAPI(MustGetOperatorConfig(env.Name), args[0])\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\n\t\t\tdeleteResponse, err = cluster.StopJob(MustGetOperatorConfig(env.Name), apisRes[0].Spec.Kind, args[0], args[1])\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t} else {\n\t\t\tdeleteResponse, err = cluster.Delete(MustGetOperatorConfig(env.Name), args[0], _flagDeleteKeepCache, _flagDeleteForce)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t}\n\n\t\tif _flagOutput == flags.JSONOutputType {\n\t\t\tbytes, err := libjson.Marshal(deleteResponse)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t\tfmt.Print(string(bytes))\n\t\t\treturn\n\t\t}\n\n\t\tprint.BoldFirstLine(deleteResponse.Message)\n\t},\n}\n"
  },
  {
    "path": "cli/cmd/deploy.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/print\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar (\n\t_warningFileBytes    = 1024 * 1024 * 10\n\t_warningProjectBytes = 1024 * 1024 * 10\n\t_warningFileCount    = 1000\n\n\t_maxFileSizeBytes    int64 = 1024 * 1024 * 32 // 32mb\n\t_maxProjectSizeBytes int64 = 1024 * 1024 * 32 // 32mb\n\n\t_flagDeployEnv            string\n\t_flagDeployForce          bool\n\t_flagDeployDisallowPrompt bool\n)\n\nfunc deployInit() {\n\t_deployCmd.Flags().SortFlags = false\n\t_deployCmd.Flags().StringVarP(&_flagDeployEnv, \"env\", \"e\", \"\", \"environment to use\")\n\t_deployCmd.Flags().BoolVarP(&_flagDeployForce, \"force\", \"f\", false, \"override the in-progress api update\")\n\t_deployCmd.Flags().BoolVarP(&_flagDeployDisallowPrompt, \"yes\", \"y\", false, \"skip prompts\")\n\t_deployCmd.Flags().VarP(&_flagOutput, \"output\", \"o\", fmt.Sprintf(\"output format: one of %s\", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), \"|\")))\n}\n\nvar _deployCmd = &cobra.Command{\n\tUse:   \"deploy [CONFIG_FILE]\",\n\tShort: \"create or update apis\",\n\tArgs:  cobra.RangeArgs(0, 1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tenvName, err := getEnvFromFlag(_flagDeployEnv)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.deploy\")\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.deploy\")\n\t\t\texit.Error(err)\n\t\t}\n\t\ttelemetry.Event(\"cli.deploy\", map[string]interface{}{\"env_name\": env.Name})\n\n\t\terr = printEnvIfNotSpecified(env.Name, cmd)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tconfigPath := getConfigPath(args)\n\n\t\tprojectRoot := files.Dir(configPath)\n\t\tif projectRoot == _homeDir {\n\t\t\texit.Error(ErrorDeployFromTopLevelDir(\"home\"))\n\t\t}\n\t\tif projectRoot == \"/\" {\n\t\t\texit.Error(ErrorDeployFromTopLevelDir(\"root\"))\n\t\t}\n\n\t\tdeploymentBytes, err := getDeploymentBytes(configPath)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tdeployResults, err := cluster.Deploy(MustGetOperatorConfig(env.Name), configPath, deploymentBytes, _flagDeployForce)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tswitch _flagOutput {\n\t\tcase flags.JSONOutputType:\n\t\t\tbytes, err := libjson.Marshal(deployResults)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t\tfmt.Print(string(bytes))\n\t\tcase flags.PrettyOutputType:\n\t\t\tmessage := mergeResultMessages(deployResults)\n\t\t\tif didAnyResultsError(deployResults) {\n\t\t\t\tprint.StderrBoldFirstBlock(message)\n\t\t\t} else {\n\t\t\t\tprint.BoldFirstBlock(message)\n\t\t\t}\n\t\t}\n\n\t\tif didAnyResultsError(deployResults) {\n\t\t\texit.Error(nil)\n\t\t}\n\t},\n}\n\n// Returns absolute path\nfunc getConfigPath(args []string) string {\n\tvar configPath string\n\n\tif len(args) == 0 {\n\t\tconfigPath = \"cortex.yaml\"\n\t\tif !files.IsFile(configPath) {\n\t\t\texit.Error(ErrorCortexYAMLNotFound())\n\t\t}\n\t} else {\n\t\tconfigPath = args[0]\n\t\tif err := files.CheckFile(configPath); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t}\n\n\treturn files.RelToAbsPath(configPath, _cwd)\n}\n\nfunc getDeploymentBytes(configPath string) (map[string][]byte, error) {\n\tconfigBytes, err := files.ReadFileBytes(configPath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tuploadBytes := map[string][]byte{\n\t\t\"config\": configBytes,\n\t}\n\n\treturn uploadBytes, nil\n}\n\nfunc mergeResultMessages(results []schema.DeployResult) string {\n\tvar okMessages []string\n\tvar errMessages []string\n\n\tfor _, result := range results {\n\t\tif result.Error != \"\" {\n\t\t\terrMessages = append(errMessages, result.Error)\n\t\t} else {\n\t\t\tokMessages = append(okMessages, result.Message)\n\t\t}\n\t}\n\n\toutput := \"\"\n\n\tif len(okMessages) > 0 {\n\t\toutput += strings.Join(okMessages, \"\\n\")\n\t\tif len(errMessages) > 0 {\n\t\t\toutput += \"\\n\\n\"\n\t\t}\n\t}\n\n\tif len(errMessages) > 0 {\n\t\toutput += strings.Join(errMessages, \"\\n\")\n\t}\n\n\treturn output\n}\n\nfunc didAllResultsError(results []schema.DeployResult) bool {\n\tfor _, result := range results {\n\t\tif result.Error == \"\" {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc didAnyResultsError(results []schema.DeployResult) bool {\n\tfor _, result := range results {\n\t\tif result.Error != \"\" {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n"
  },
  {
    "path": "cli/cmd/describe.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst (\n\t_titleReplicaStatus = \"replica status\"\n\t_titleReplicaCount  = \"replica count\"\n)\n\nvar (\n\t_flagDescribeEnv   string\n\t_flagDescribeWatch bool\n)\n\nfunc describeInit() {\n\t_describeCmd.Flags().SortFlags = false\n\t_describeCmd.Flags().StringVarP(&_flagDescribeEnv, \"env\", \"e\", \"\", \"environment to use\")\n\t_describeCmd.Flags().BoolVarP(&_flagDescribeWatch, \"watch\", \"w\", false, \"re-run the command every 2 seconds\")\n}\n\nvar _describeCmd = &cobra.Command{\n\tUse:   \"describe [API_NAME]\",\n\tShort: \"describe an api\",\n\tArgs:  cobra.ExactArgs(1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tapiName := args[0]\n\n\t\tvar envName string\n\t\tif wasFlagProvided(cmd, \"env\") {\n\t\t\tenvName = _flagDescribeEnv\n\t\t} else {\n\t\t\tvar err error\n\t\t\tenvName, err = getEnvFromFlag(\"\")\n\t\t\tif err != nil {\n\t\t\t\ttelemetry.Event(\"cli.describe\")\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t}\n\n\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.describe\")\n\t\t\texit.Error(err)\n\t\t}\n\t\ttelemetry.Event(\"cli.describe\", map[string]interface{}{\"env_name\": env.Name})\n\n\t\trerun(_flagDescribeWatch, func() (string, error) {\n\t\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\n\t\t\tout, err := envStringIfNotSpecified(envName, cmd)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", err\n\t\t\t}\n\t\t\tapiTable, err := describeAPI(env, apiName)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", err\n\t\t\t}\n\n\t\t\treturn out + apiTable, nil\n\t\t})\n\t},\n}\n\nfunc describeAPI(env cliconfig.Environment, apiName string) (string, error) {\n\tapisRes, err := cluster.DescribeAPI(MustGetOperatorConfig(env.Name), apiName)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif len(apisRes) == 0 {\n\t\texit.Error(errors.ErrorUnexpected(fmt.Sprintf(\"unable to find api %s\", apiName)))\n\t}\n\n\tapiRes := apisRes[0]\n\n\tswitch apiRes.Metadata.Kind {\n\tcase userconfig.RealtimeAPIKind:\n\t\treturn realtimeDescribeAPITable(apiRes, env)\n\tcase userconfig.AsyncAPIKind:\n\t\treturn asyncDescribeAPITable(apiRes, env)\n\tdefault:\n\t\treturn \"\", errors.ErrorUnexpected(fmt.Sprintf(\"encountered unexpected kind %s for api %s\", apiRes.Metadata.Kind, apiRes.Metadata.Name))\n\t}\n}\n"
  },
  {
    "path": "cli/cmd/env.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/print\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar (\n\t_flagEnvOperatorEndpoint string\n)\n\nfunc envInit() {\n\t_envConfigureCmd.Flags().SortFlags = false\n\t_envConfigureCmd.Flags().StringVarP(&_flagEnvOperatorEndpoint, \"operator-endpoint\", \"o\", \"\", \"set the operator endpoint without prompting\")\n\t_envCmd.AddCommand(_envConfigureCmd)\n\n\t_envListCmd.Flags().SortFlags = false\n\t_envListCmd.Flags().VarP(&_flagOutput, \"output\", \"o\", fmt.Sprintf(\"output format: one of %s\", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), \"|\")))\n\t_envCmd.AddCommand(_envListCmd)\n\n\t_envDefaultCmd.Flags().SortFlags = false\n\t_envCmd.AddCommand(_envDefaultCmd)\n\n\t_envRenameCmd.Flags().SortFlags = false\n\t_envCmd.AddCommand(_envRenameCmd)\n\n\t_envDeleteCmd.Flags().SortFlags = false\n\t_envCmd.AddCommand(_envDeleteCmd)\n}\n\nvar _envCmd = &cobra.Command{\n\tUse:   \"env\",\n\tShort: \"manage cli environments (contains subcommands)\",\n}\n\nvar _envConfigureCmd = &cobra.Command{\n\tUse:   \"configure [ENVIRONMENT_NAME]\",\n\tShort: \"configure an environment\",\n\tArgs:  cobra.RangeArgs(0, 1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.env.configure\")\n\n\t\tvar envName string\n\t\tif len(args) == 1 {\n\t\t\tenvName = args[0]\n\t\t}\n\n\t\tfieldsToSkipPrompt := cliconfig.Environment{}\n\t\tif _flagEnvOperatorEndpoint != \"\" {\n\t\t\toperatorEndpoint, err := validateOperatorEndpoint(_flagEnvOperatorEndpoint)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t\tfieldsToSkipPrompt.OperatorEndpoint = operatorEndpoint\n\t\t}\n\n\t\tif _, err := configureEnv(envName, fieldsToSkipPrompt); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t},\n}\n\nvar _envListCmd = &cobra.Command{\n\tUse:   \"list\",\n\tShort: \"list all configured environments\",\n\tArgs:  cobra.NoArgs,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.env.list\")\n\n\t\tcliConfig, err := readCLIConfig()\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tif _flagOutput == flags.JSONOutputType {\n\t\t\tbytes, err := libjson.Marshal(cliConfig.ConvertToUserFacingCLIConfig())\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t\tfmt.Print(string(bytes))\n\t\t\treturn\n\t\t}\n\n\t\tif len(cliConfig.Environments) == 0 {\n\t\t\tfmt.Println(\"no environments are configured\")\n\t\t\treturn\n\t\t}\n\n\t\tdefaultEnv, err := getDefaultEnv()\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tfor i, env := range cliConfig.Environments {\n\t\t\tfmt.Print(env.String(defaultEnv != nil && *defaultEnv == env.Name))\n\t\t\tif i+1 < len(cliConfig.Environments) {\n\t\t\t\tfmt.Println()\n\t\t\t}\n\t\t}\n\t},\n}\n\nvar _envDefaultCmd = &cobra.Command{\n\tUse:   \"default [ENVIRONMENT_NAME]\",\n\tShort: \"set the default environment\",\n\tArgs:  cobra.RangeArgs(0, 1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.env.default\")\n\n\t\tdefaultEnv, err := getDefaultEnv()\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tvar envName string\n\t\tif len(args) == 0 {\n\t\t\tif defaultEnv != nil {\n\t\t\t\tfmt.Printf(\"current default environment: %s\\n\\n\", *defaultEnv)\n\t\t\t}\n\t\t\tenvName = promptForExistingEnvName(\"name of environment to set as default\")\n\t\t} else {\n\t\t\tenvName = args[0]\n\t\t}\n\n\t\tif defaultEnv != nil && *defaultEnv == envName {\n\t\t\tprint.BoldFirstLine(fmt.Sprintf(\"%s is already the default environment\", envName))\n\t\t\texit.Ok()\n\t\t}\n\n\t\tif err := setDefaultEnv(envName); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tprint.BoldFirstLine(fmt.Sprintf(\"set %s as the default environment\", envName))\n\t},\n}\n\nvar _envRenameCmd = &cobra.Command{\n\tUse:   \"rename EXISTING_NAME NEW_NAME\",\n\tShort: \"rename an environment\",\n\tArgs:  cobra.ExactArgs(2),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.env.rename\")\n\n\t\toldEnvName := args[0]\n\t\tnewEnvName := args[1]\n\n\t\tif err := renameEnv(oldEnvName, newEnvName); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tprint.BoldFirstLine(fmt.Sprintf(\"renamed the %s environment to %s\", oldEnvName, newEnvName))\n\t},\n}\n\nvar _envDeleteCmd = &cobra.Command{\n\tUse:   \"delete [ENVIRONMENT_NAME]\",\n\tShort: \"delete an environment configuration\",\n\tArgs:  cobra.RangeArgs(0, 1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\ttelemetry.Event(\"cli.env.delete\")\n\n\t\tvar envName string\n\t\tif len(args) == 1 {\n\t\t\tenvName = args[0]\n\t\t} else {\n\t\t\tenvName = promptForExistingEnvName(\"name of environment to delete\")\n\t\t}\n\n\t\tprevDefault, err := getDefaultEnv()\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tif err := removeEnvFromCLIConfig(envName); err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tnewDefault, err := getDefaultEnv()\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tprint.BoldFirstLine(fmt.Sprintf(\"deleted the %s environment configuration\", envName))\n\t\tif prevDefault != nil && newDefault == nil {\n\t\t\tprint.BoldFirstLine(\"unset the default environment\")\n\t\t} else if newDefault != nil && (prevDefault == nil || *prevDefault != *newDefault) {\n\t\t\tprint.BoldFirstLine(fmt.Sprintf(\"set the default environment to %s\", *newDefault))\n\t\t}\n\t},\n}\n"
  },
  {
    "path": "cli/cmd/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n)\n\nconst (\n\t_errStrCantMakeRequest = \"unable to make request\"\n\t_errStrRead            = \"unable to read\"\n)\n\nfunc errStrFailedToConnect(u url.URL) string {\n\treturn \"failed to connect to \" + urls.TrimQueryParamsURL(u)\n}\n\nconst (\n\tErrInvalidProvider                     = \"cli.invalid_provider\"\n\tErrInvalidLegacyProvider               = \"cli.invalid_legacy_provider\"\n\tErrNoAvailableEnvironment              = \"cli.no_available_environment\"\n\tErrEnvironmentNotSet                   = \"cli.environment_not_set\"\n\tErrEnvironmentNotFound                 = \"cli.environment_not_found\"\n\tErrFieldNotFoundInEnvironment          = \"cli.field_not_found_in_environment\"\n\tErrInvalidOperatorEndpoint             = \"cli.invalid_operator_endpoint\"\n\tErrNoOperatorLoadBalancer              = \"cli.no_operator_load_balancer\"\n\tErrCortexYAMLNotFound                  = \"cli.cortex_yaml_not_found\"\n\tErrDockerCtrlC                         = \"cli.docker_ctrl_c\"\n\tErrResponseUnknown                     = \"cli.response_unknown\"\n\tErrMissingAWSCredentials               = \"cli.missing_aws_credentials\"\n\tErrCredentialsInClusterConfig          = \"cli.credentials_in_cluster_config\"\n\tErrClusterUp                           = \"cli.cluster_up\"\n\tErrClusterConfigure                    = \"cli.cluster_configure\"\n\tErrClusterDebug                        = \"cli.cluster_debug\"\n\tErrClusterRefresh                      = \"cli.cluster_refresh\"\n\tErrClusterDown                         = \"cli.cluster_down\"\n\tErrSpecifyAtLeastOneFlag               = \"cli.specify_at_least_one_flag\"\n\tErrMinInstancesLowerThan               = \"cli.min_instances_lower_than\"\n\tErrMaxInstancesLowerThan               = \"cli.max_instances_lower_than\"\n\tErrMinInstancesGreaterThanMaxInstances = \"cli.min_instances_greater_than_max_instances\"\n\tErrNodeGroupNotFound                   = \"cli.nodegroup_not_found\"\n\tErrMutuallyExclusiveFlags              = \"cli.mutually_exclusive_flags\"\n\tErrClusterAccessConfigRequired         = \"cli.cluster_access_config_or_prompts_required\"\n\tErrShellCompletionNotSupported         = \"cli.shell_completion_not_supported\"\n\tErrNoTerminalWidth                     = \"cli.no_terminal_width\"\n\tErrDeployFromTopLevelDir               = \"cli.deploy_from_top_level_dir\"\n\tErrAPINameMustBeProvided               = \"cli.api_name_must_be_provided\"\n\tErrAPINotFoundInConfig                 = \"cli.api_not_found_in_config\"\n\tErrClusterUIDsLimitInBucket            = \"cli.cluster_uids_limit_in_bucket\"\n)\n\nfunc ErrorInvalidProvider(providerStr, cliConfigPath string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidProvider,\n\t\tMessage: fmt.Sprintf(\"\\\"%s\\\" is not a supported provider (only aws is supported); remove the environment(s) which use the %s provider from %s, or delete %s (it will be recreated on subsequent CLI commands)\", providerStr, providerStr, cliConfigPath, cliConfigPath),\n\t})\n}\n\nfunc ErrorInvalidLegacyProvider(providerStr, cliConfigPath string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidLegacyProvider,\n\t\tMessage: fmt.Sprintf(\"the %s provider is no longer supported on cortex v%s; remove the environment(s) which use the %s provider from %s, or delete %s (it will be recreated on subsequent CLI commands)\", providerStr, consts.CortexVersionMinor, providerStr, cliConfigPath, cliConfigPath),\n\t})\n}\n\nfunc ErrorNoAvailableEnvironment() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoAvailableEnvironment,\n\t\tMessage: \"no environments are configured; run `cortex cluster up` to create a cluster, or run `cortex env configure` to connect to an existing cluster\",\n\t})\n}\n\nfunc ErrorEnvironmentNotSet() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEnvironmentNotSet,\n\t\tMessage: \"no environment was provided and the default environment is not set; specify the environment to use via the `-e/--env` flag, or run `cortex env default` to set the default environment\",\n\t})\n}\n\nfunc ErrorEnvironmentNotFound(envName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEnvironmentNotFound,\n\t\tMessage: fmt.Sprintf(\"unable to find environment named \\\"%s\\\"\", envName),\n\t})\n}\n\nfunc ErrorFieldNotFoundInEnvironment(fieldName string, envName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFieldNotFoundInEnvironment,\n\t\tMessage: fmt.Sprintf(\"%s was not found in %s environment\", fieldName, envName),\n\t})\n}\n\nfunc ErrorInvalidOperatorEndpoint(endpoint string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidOperatorEndpoint,\n\t\tMessage: fmt.Sprintf(\"%s is not a cortex operator endpoint; run `cortex cluster info` to show your operator endpoint or run `cortex cluster up` to spin up a new cluster\", endpoint),\n\t})\n}\n\nfunc ErrorNoOperatorLoadBalancer(whichLB string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoOperatorLoadBalancer,\n\t\tMessage: fmt.Sprintf(\"unable to locate %s load balancer\", whichLB),\n\t})\n}\n\nfunc ErrorCortexYAMLNotFound() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCortexYAMLNotFound,\n\t\tMessage: \"no api config file was specified, and ./cortex.yaml does not exist; create cortex.yaml, or reference an existing config file by running `cortex deploy <config_file_path>`\",\n\t})\n}\n\nfunc ErrorDockerCtrlC() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:        ErrDockerCtrlC,\n\t\tNoPrint:     true,\n\t\tNoTelemetry: true,\n\t})\n}\n\nfunc ErrorResponseUnknown(body string, statusCode int) error {\n\tmsg := body\n\tif strings.TrimSpace(body) == \"\" {\n\t\tmsg = fmt.Sprintf(\"empty response (status code %d)\", statusCode)\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrResponseUnknown,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorClusterUp(out string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterUp,\n\t\tMessage: out,\n\t\tNoPrint: true,\n\t})\n}\n\nfunc ErrorClusterConfigure(out string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterConfigure,\n\t\tMessage: out,\n\t\tNoPrint: true,\n\t})\n}\n\nfunc ErrorClusterDebug(out string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterDebug,\n\t\tMessage: out,\n\t\tNoPrint: true,\n\t})\n}\n\nfunc ErrorClusterRefresh(out string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterRefresh,\n\t\tMessage: out,\n\t\tNoPrint: true,\n\t})\n}\n\nfunc ErrorClusterDown(out string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterDown,\n\t\tMessage: out,\n\t\tNoPrint: true,\n\t})\n}\n\nfunc ErrorSpecifyAtLeastOneFlag(flagsToSpecify ...string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSpecifyAtLeastOneFlag,\n\t\tMessage: fmt.Sprintf(\"must specify at least one of the following flags: %s\", s.StrsOr(flagsToSpecify)),\n\t})\n}\n\nfunc ErrorMinInstancesLowerThan(minValue int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMinInstancesLowerThan,\n\t\tMessage: fmt.Sprintf(\"min instances cannot be set to a value lower than %d\", minValue),\n\t})\n}\n\nfunc ErrorMaxInstancesLowerThan(minValue int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMaxInstancesLowerThan,\n\t\tMessage: fmt.Sprintf(\"max instances cannot be set to a value lower than %d\", minValue),\n\t})\n}\n\nfunc ErrorMinInstancesGreaterThanMaxInstances(minInstances, maxInstances int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMinInstancesGreaterThanMaxInstances,\n\t\tMessage: fmt.Sprintf(\"min instances (%d) cannot be set to a value higher than max instances (%d)\", minInstances, maxInstances),\n\t})\n}\n\nfunc ErrorNodeGroupNotFound(scalingNodeGroupName, clusterName, clusterRegion string, availableNodeGroups []string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNodeGroupNotFound,\n\t\tMessage: fmt.Sprintf(\"nodegroup %s couldn't be found in the cluster named %s in region %s; the available nodegroups for this cluster are: %s\", scalingNodeGroupName, clusterName, clusterRegion, s.StrsAnd(availableNodeGroups)),\n\t})\n}\n\nfunc ErrorMutuallyExclusiveFlags(flagA, flagB string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMutuallyExclusiveFlags,\n\t\tMessage: fmt.Sprintf(\"flags %s and %s cannot be used at the same time\", flagA, flagB),\n\t})\n}\n\nfunc ErrorClusterAccessConfigRequired(cliFlagsOnly bool) error {\n\tmessage := \"\"\n\tif cliFlagsOnly {\n\t\tmessage = \"please provide the name and region of the cluster using the CLI flags (e.g. via `--name` and `--region`)\"\n\t} else {\n\t\tmessage = fmt.Sprintf(\"please provide a cluster configuration file which specifies `%s` and `%s` (e.g. via `--config cluster.yaml`) or use the CLI flags to specify the cluster (e.g. via `--name` and `--region`)\", clusterconfig.ClusterNameKey, clusterconfig.RegionKey)\n\t}\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterAccessConfigRequired,\n\t\tMessage: message,\n\t})\n}\n\nfunc ErrorShellCompletionNotSupported(shell string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrShellCompletionNotSupported,\n\t\tMessage: fmt.Sprintf(\"shell completion for %s is not supported (only bash and zsh are supported)\", shell),\n\t})\n}\n\nfunc ErrorNoTerminalWidth() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoTerminalWidth,\n\t\tMessage: \"unable to determine terminal width; please re-run the command without the `--watch` flag\",\n\t})\n}\n\nfunc ErrorDeployFromTopLevelDir(genericDirName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDeployFromTopLevelDir,\n\t\tMessage: fmt.Sprintf(\"cannot deploy from your %s directory - when deploying your API, cortex sends all files in your project directory (i.e. the directory which contains cortex.yaml) to your cluster (see https://docs.cortexlabs.com/v/%s/); therefore it is recommended to create a subdirectory for your project files\", genericDirName, consts.CortexVersionMinor),\n\t})\n}\n\nfunc ErrorAPINameMustBeProvided() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAPINameMustBeProvided,\n\t\tMessage: fmt.Sprintf(\"multiple apis listed; please specify the name of an api\"),\n\t})\n}\n\nfunc ErrorAPINotFoundInConfig(apiName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAPINotFoundInConfig,\n\t\tMessage: fmt.Sprintf(\"api '%s' not found in config\", apiName),\n\t})\n}\n\nfunc ErrorClusterUIDsLimitInBucket(bucket string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterUIDsLimitInBucket,\n\t\tMessage: fmt.Sprintf(\"detected too many top level folders in %s bucket; please empty your bucket and try again\", bucket),\n\t})\n}\n"
  },
  {
    "path": "cli/cmd/get.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/yaml\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst (\n\t_titleEnvironment = \"env\"\n\t_titleRealtimeAPI = \"realtime api\"\n\t_titleAsyncAPI    = \"async api\"\n\t_titleLive        = \"live\"\n\t_titleUpToDate    = \"up-to-date\"\n\t_titleLastUpdated = \"last update\"\n)\n\nvar (\n\t_flagGetEnv   string\n\t_flagGetWatch bool\n)\n\nfunc getInit() {\n\t_getCmd.Flags().SortFlags = false\n\t_getCmd.Flags().StringVarP(&_flagGetEnv, \"env\", \"e\", \"\", \"environment to use\")\n\t_getCmd.Flags().BoolVarP(&_flagGetWatch, \"watch\", \"w\", false, \"re-run the command every 2 seconds\")\n\t_getCmd.Flags().VarP(&_flagOutput, \"output\", \"o\", fmt.Sprintf(\"output format: one of %s\", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), \"|\")))\n\taddVerboseFlag(_getCmd)\n}\n\nvar _getCmd = &cobra.Command{\n\tUse:   \"get [API_NAME] [JOB_ID]\",\n\tShort: \"get information about apis or jobs\",\n\tArgs:  cobra.RangeArgs(0, 2),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tvar envName string\n\t\tif wasFlagProvided(cmd, \"env\") {\n\t\t\tenvName = _flagGetEnv\n\t\t} else if len(args) > 0 {\n\t\t\tvar err error\n\t\t\tenvName, err = getEnvFromFlag(\"\")\n\t\t\tif err != nil {\n\t\t\t\ttelemetry.Event(\"cli.get\")\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t}\n\n\t\tif len(args) == 1 || wasFlagProvided(cmd, \"env\") {\n\t\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\t\tif err != nil {\n\t\t\t\ttelemetry.Event(\"cli.get\")\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t\ttelemetry.Event(\"cli.get\", map[string]interface{}{\"env_name\": env.Name})\n\t\t} else {\n\t\t\ttelemetry.Event(\"cli.get\")\n\t\t}\n\n\t\trerun(_flagGetWatch, func() (string, error) {\n\t\t\tif len(args) == 1 {\n\t\t\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\t\t\tif err != nil {\n\t\t\t\t\texit.Error(err)\n\t\t\t\t}\n\n\t\t\t\tout, err := envStringIfNotSpecified(envName, cmd)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tapiTable, err := getAPI(env, args[0])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\n\t\t\t\tif _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {\n\t\t\t\t\treturn apiTable, nil\n\t\t\t\t}\n\n\t\t\t\treturn out + apiTable, nil\n\t\t\t} else if len(args) == 2 {\n\t\t\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\t\t\tif err != nil {\n\t\t\t\t\texit.Error(err)\n\t\t\t\t}\n\n\t\t\t\tout, err := envStringIfNotSpecified(envName, cmd)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\n\t\t\t\tapisRes, err := cluster.GetAPI(MustGetOperatorConfig(envName), args[0])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\n\t\t\t\tvar jobTable string\n\t\t\t\tif apisRes[0].Metadata.Kind == userconfig.BatchAPIKind {\n\t\t\t\t\tjobTable, err = getBatchJob(env, args[0], args[1])\n\t\t\t\t} else {\n\t\t\t\t\tjobTable, err = getTaskJob(env, args[0], args[1])\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tif _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {\n\t\t\t\t\treturn jobTable, nil\n\t\t\t\t}\n\n\t\t\t\treturn out + jobTable, nil\n\t\t\t} else {\n\t\t\t\tenvs, err := listConfiguredEnvs()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tif len(envs) == 0 {\n\t\t\t\t\treturn \"\", ErrorNoAvailableEnvironment()\n\t\t\t\t}\n\n\t\t\t\tif wasFlagProvided(cmd, \"env\") {\n\t\t\t\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\texit.Error(err)\n\t\t\t\t\t}\n\n\t\t\t\t\tout, err := envStringIfNotSpecified(envName, cmd)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn \"\", err\n\t\t\t\t\t}\n\n\t\t\t\t\tapiTable, err := getAPIsByEnv(env)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn \"\", err\n\t\t\t\t\t}\n\n\t\t\t\t\tif _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {\n\t\t\t\t\t\treturn apiTable, nil\n\t\t\t\t\t}\n\n\t\t\t\t\treturn out + apiTable, nil\n\t\t\t\t}\n\n\t\t\t\tout, err := getAPIsInAllEnvironments()\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\n\t\t\t\treturn out, nil\n\t\t\t}\n\t\t})\n\t},\n}\n\nfunc getAPIsInAllEnvironments() (string, error) {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tvar allRealtimeAPIs []schema.APIResponse\n\tvar allRealtimeAPIEnvs []string\n\tvar allAsyncAPIs []schema.APIResponse\n\tvar allAsyncAPIEnvs []string\n\tvar allBatchAPIs []schema.APIResponse\n\tvar allBatchAPIEnvs []string\n\tvar allTaskAPIs []schema.APIResponse\n\tvar allTaskAPIEnvs []string\n\tvar allTrafficSplitters []schema.APIResponse\n\tvar allTrafficSplitterEnvs []string\n\n\ttype getAPIsOutput struct {\n\t\tEnvName string               `json:\"env_name\"`\n\t\tAPIs    []schema.APIResponse `json:\"apis\"`\n\t\tError   string               `json:\"error\"`\n\t}\n\n\tallAPIsOutput := []getAPIsOutput{}\n\n\terrorsMap := map[string]error{}\n\t// get apis from both environments\n\tfor _, env := range cliConfig.Environments {\n\t\tapisRes, err := cluster.GetAPIs(MustGetOperatorConfig(env.Name))\n\n\t\tapisOutput := getAPIsOutput{\n\t\t\tEnvName: env.Name,\n\t\t\tAPIs:    apisRes,\n\t\t}\n\n\t\tif err == nil {\n\t\t\tfor _, api := range apisRes {\n\t\t\t\tswitch api.Metadata.Kind {\n\t\t\t\tcase userconfig.BatchAPIKind:\n\t\t\t\t\tallBatchAPIEnvs = append(allBatchAPIEnvs, env.Name)\n\t\t\t\t\tallBatchAPIs = append(allBatchAPIs, api)\n\t\t\t\tcase userconfig.RealtimeAPIKind:\n\t\t\t\t\tallRealtimeAPIEnvs = append(allRealtimeAPIEnvs, env.Name)\n\t\t\t\t\tallRealtimeAPIs = append(allRealtimeAPIs, api)\n\t\t\t\tcase userconfig.AsyncAPIKind:\n\t\t\t\t\tallAsyncAPIEnvs = append(allAsyncAPIEnvs, env.Name)\n\t\t\t\t\tallAsyncAPIs = append(allAsyncAPIs, api)\n\t\t\t\tcase userconfig.TaskAPIKind:\n\t\t\t\t\tallTaskAPIEnvs = append(allTaskAPIEnvs, env.Name)\n\t\t\t\t\tallTaskAPIs = append(allTaskAPIs, api)\n\t\t\t\tcase userconfig.TrafficSplitterKind:\n\t\t\t\t\tallTrafficSplitterEnvs = append(allTrafficSplitterEnvs, env.Name)\n\t\t\t\t\tallTrafficSplitters = append(allTrafficSplitters, api)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tapisOutput.Error = err.Error()\n\t\t\terrorsMap[env.Name] = err\n\t\t}\n\n\t\tallAPIsOutput = append(allAPIsOutput, apisOutput)\n\t}\n\n\tvar bytes []byte\n\tif _flagOutput == flags.JSONOutputType {\n\t\tbytes, err = libjson.Marshal(allAPIsOutput)\n\t} else if _flagOutput == flags.YAMLOutputType {\n\t\tbytes, err = yaml.Marshal(allAPIsOutput)\n\t}\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {\n\t\treturn string(bytes), nil\n\t}\n\n\tout := \"\"\n\n\tif len(allRealtimeAPIs) == 0 && len(allAsyncAPIs) == 0 && len(allBatchAPIs) == 0 && len(allTrafficSplitters) == 0 && len(allTaskAPIs) == 0 {\n\t\t// check if any environments errorred\n\t\tif len(errorsMap) != len(cliConfig.Environments) {\n\t\t\tif len(errorsMap) == 0 {\n\t\t\t\treturn console.Bold(\"no apis are deployed\"), nil\n\t\t\t}\n\n\t\t\tvar successfulEnvs []string\n\t\t\tfor _, env := range cliConfig.Environments {\n\t\t\t\tif _, ok := errorsMap[env.Name]; !ok {\n\t\t\t\t\tsuccessfulEnvs = append(successfulEnvs, env.Name)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfmt.Println(console.Bold(fmt.Sprintf(\"no apis are deployed in %s: %s\", s.PluralS(\"environment\", len(successfulEnvs)), s.StrsAnd(successfulEnvs))) + \"\\n\")\n\t\t}\n\n\t\t// Print the first error\n\t\tfor name, err := range errorsMap {\n\t\t\tif err != nil {\n\t\t\t\texit.Error(errors.Wrap(err, \"env \"+name))\n\t\t\t}\n\t\t}\n\t} else {\n\t\tif len(allBatchAPIs) > 0 {\n\t\t\tt := batchAPIsTable(allBatchAPIs, allBatchAPIEnvs)\n\t\t\tout += t.MustFormat()\n\t\t}\n\n\t\tif len(allTaskAPIs) > 0 {\n\t\t\tt := taskAPIsTable(allTaskAPIs, allTaskAPIEnvs)\n\t\t\tif len(allBatchAPIs) > 0 {\n\t\t\t\tout += \"\\n\"\n\t\t\t}\n\t\t\tout += t.MustFormat()\n\t\t}\n\n\t\tif len(allRealtimeAPIs) > 0 {\n\t\t\tt := realtimeAPIsTable(allRealtimeAPIs, allRealtimeAPIEnvs)\n\t\t\tif len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 {\n\t\t\t\tout += \"\\n\"\n\t\t\t}\n\t\t\tout += t.MustFormat()\n\t\t}\n\t\tif len(allAsyncAPIs) > 0 {\n\t\t\tt := asyncAPIsTable(allAsyncAPIs, allAsyncAPIEnvs)\n\t\t\tif len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 || len(allRealtimeAPIs) > 0 {\n\t\t\t\tout += \"\\n\"\n\t\t\t}\n\t\t\tout += t.MustFormat()\n\t\t}\n\n\t\tif len(allTrafficSplitters) > 0 {\n\t\t\tt := trafficSplitterListTable(allTrafficSplitters, allTrafficSplitterEnvs)\n\n\t\t\tif len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 || len(allRealtimeAPIs) > 0 || len(allAsyncAPIs) > 0 {\n\t\t\t\tout += \"\\n\"\n\t\t\t}\n\n\t\t\tout += t.MustFormat()\n\t\t}\n\t}\n\n\tif len(errorsMap) == 1 {\n\t\tout = s.EnsureBlankLineIfNotEmpty(out)\n\t\tout += fmt.Sprintf(\"unable to detect apis from the %s environment; run `cortex get --env %s` if this is unexpected\\n\", errors.FirstKeyInErrorMap(errorsMap), errors.FirstKeyInErrorMap(errorsMap))\n\t} else if len(errorsMap) > 1 {\n\t\tout = s.EnsureBlankLineIfNotEmpty(out)\n\t\tout += fmt.Sprintf(\"unable to detect apis from the %s environments; run `cortex get --env ENV_NAME` if this is unexpected\\n\", s.StrsAnd(errors.NonNilErrorMapKeys(errorsMap)))\n\t}\n\n\treturn out, nil\n}\n\nfunc getAPIsByEnv(env cliconfig.Environment) (string, error) {\n\tapisRes, err := cluster.GetAPIs(MustGetOperatorConfig(env.Name))\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tvar bytes []byte\n\tif _flagOutput == flags.JSONOutputType {\n\t\tbytes, err = libjson.Marshal(apisRes)\n\t} else if _flagOutput == flags.YAMLOutputType {\n\t\tbytes, err = yaml.Marshal(apisRes)\n\t}\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {\n\t\treturn string(bytes), nil\n\t}\n\n\tvar allRealtimeAPIs []schema.APIResponse\n\tvar allAsyncAPIs []schema.APIResponse\n\tvar allBatchAPIs []schema.APIResponse\n\tvar allTaskAPIs []schema.APIResponse\n\tvar allTrafficSplitters []schema.APIResponse\n\n\tfor _, api := range apisRes {\n\t\tswitch api.Metadata.Kind {\n\t\tcase userconfig.BatchAPIKind:\n\t\t\tallBatchAPIs = append(allBatchAPIs, api)\n\t\tcase userconfig.TaskAPIKind:\n\t\t\tallTaskAPIs = append(allTaskAPIs, api)\n\t\tcase userconfig.RealtimeAPIKind:\n\t\t\tallRealtimeAPIs = append(allRealtimeAPIs, api)\n\t\tcase userconfig.AsyncAPIKind:\n\t\t\tallAsyncAPIs = append(allAsyncAPIs, api)\n\t\tcase userconfig.TrafficSplitterKind:\n\t\t\tallTrafficSplitters = append(allTrafficSplitters, api)\n\t\t}\n\t}\n\n\tif len(allRealtimeAPIs) == 0 && len(allAsyncAPIs) == 0 && len(allBatchAPIs) == 0 && len(allTaskAPIs) == 0 && len(allTrafficSplitters) == 0 {\n\t\treturn console.Bold(\"no apis are deployed\"), nil\n\t}\n\n\tout := \"\"\n\n\tif len(allBatchAPIs) > 0 {\n\t\tenvNames := []string{}\n\t\tfor range allBatchAPIs {\n\t\t\tenvNames = append(envNames, env.Name)\n\t\t}\n\n\t\tt := batchAPIsTable(allBatchAPIs, envNames)\n\t\tt.FindHeaderByTitle(_titleEnvironment).Hidden = true\n\n\t\tout += t.MustFormat()\n\t}\n\n\tif len(allTaskAPIs) > 0 {\n\t\tenvNames := []string{}\n\t\tfor range allTaskAPIs {\n\t\t\tenvNames = append(envNames, env.Name)\n\t\t}\n\n\t\tt := taskAPIsTable(allTaskAPIs, envNames)\n\t\tt.FindHeaderByTitle(_titleEnvironment).Hidden = true\n\n\t\tif len(allBatchAPIs) > 0 {\n\t\t\tout += \"\\n\"\n\t\t}\n\n\t\tout += t.MustFormat()\n\t}\n\n\tif len(allRealtimeAPIs) > 0 {\n\t\tenvNames := []string{}\n\t\tfor range allRealtimeAPIs {\n\t\t\tenvNames = append(envNames, env.Name)\n\t\t}\n\n\t\tt := realtimeAPIsTable(allRealtimeAPIs, envNames)\n\t\tt.FindHeaderByTitle(_titleEnvironment).Hidden = true\n\n\t\tif len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 {\n\t\t\tout += \"\\n\"\n\t\t}\n\n\t\tout += t.MustFormat()\n\t}\n\n\tif len(allAsyncAPIs) > 0 {\n\t\tenvNames := []string{}\n\t\tfor range allAsyncAPIs {\n\t\t\tenvNames = append(envNames, env.Name)\n\t\t}\n\n\t\tt := asyncAPIsTable(allAsyncAPIs, envNames)\n\t\tt.FindHeaderByTitle(_titleEnvironment).Hidden = true\n\n\t\tif len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 || len(allRealtimeAPIs) > 0 {\n\t\t\tout += \"\\n\"\n\t\t}\n\n\t\tout += t.MustFormat()\n\t}\n\n\tif len(allTrafficSplitters) > 0 {\n\t\tenvNames := []string{}\n\t\tfor range allTrafficSplitters {\n\t\t\tenvNames = append(envNames, env.Name)\n\t\t}\n\n\t\tt := trafficSplitterListTable(allTrafficSplitters, envNames)\n\t\tt.FindHeaderByTitle(_titleEnvironment).Hidden = true\n\n\t\tif len(allBatchAPIs) > 0 || len(allTaskAPIs) > 0 || len(allRealtimeAPIs) > 0 || len(allAsyncAPIs) > 0 {\n\t\t\tout += \"\\n\"\n\t\t}\n\n\t\tout += t.MustFormat()\n\t}\n\n\treturn out, nil\n}\n\nfunc getAPI(env cliconfig.Environment, apiName string) (string, error) {\n\tapisRes, err := cluster.GetAPI(MustGetOperatorConfig(env.Name), apiName)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tvar bytes []byte\n\tif _flagOutput == flags.JSONOutputType {\n\t\tbytes, err = libjson.Marshal(apisRes)\n\t} else if _flagOutput == flags.YAMLOutputType {\n\t\tbytes, err = yaml.Marshal(apisRes)\n\t}\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {\n\t\treturn string(bytes), nil\n\t}\n\n\tif len(apisRes) == 0 {\n\t\texit.Error(errors.ErrorUnexpected(fmt.Sprintf(\"unable to find api %s\", apiName)))\n\t}\n\n\tapiRes := apisRes[0]\n\n\tswitch apiRes.Metadata.Kind {\n\tcase userconfig.RealtimeAPIKind:\n\t\treturn realtimeAPITable(apiRes, env)\n\tcase userconfig.AsyncAPIKind:\n\t\treturn asyncAPITable(apiRes, env)\n\tcase userconfig.TrafficSplitterKind:\n\t\treturn trafficSplitterTable(apiRes, env)\n\tcase userconfig.BatchAPIKind:\n\t\treturn batchAPITable(apiRes), nil\n\tcase userconfig.TaskAPIKind:\n\t\treturn taskAPITable(apiRes), nil\n\tdefault:\n\t\treturn \"\", errors.ErrorUnexpected(fmt.Sprintf(\"encountered unexpected kind %s for api %s\", apiRes.Metadata.Kind, apiRes.Metadata.Name))\n\t}\n}\n\nfunc apiHistoryTable(apiVersions []schema.APIVersion) string {\n\tt := table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: \"api id\"},\n\t\t\t{Title: \"last deployed\"},\n\t\t},\n\t}\n\n\tt.Rows = make([][]interface{}, len(apiVersions))\n\tfor i, apiVersion := range apiVersions {\n\t\tlastUpdated := time.Unix(apiVersion.LastUpdated, 0)\n\t\tt.Rows[i] = []interface{}{apiVersion.APIID, libtime.SinceStr(&lastUpdated)}\n\t}\n\n\treturn t.MustFormat(&table.Opts{Sort: pointer.Bool(false)})\n}\n\nfunc titleStr(title string) string {\n\treturn \"\\n\" + console.Bold(title) + \"\\n\"\n}\n"
  },
  {
    "path": "cli/cmd/lib_apis.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n)\n\nfunc replicaCountTable(counts *status.ReplicaCounts) table.Table {\n\tvar rows [][]interface{}\n\tfor _, replicaCountType := range status.ReplicaCountTypes {\n\t\tcount := counts.GetCountBy(replicaCountType)\n\t\tcanBeHiddenIfZero := false\n\t\tswitch replicaCountType {\n\t\tcase status.ReplicaCountFailed:\n\t\t\tcanBeHiddenIfZero = true\n\t\tcase status.ReplicaCountKilled:\n\t\t\tcanBeHiddenIfZero = true\n\t\tcase status.ReplicaCountKilledOOM:\n\t\t\tcanBeHiddenIfZero = true\n\t\tcase status.ReplicaCountErrImagePull:\n\t\t\tcanBeHiddenIfZero = true\n\t\tcase status.ReplicaCountUnknown:\n\t\t\tcanBeHiddenIfZero = true\n\t\tcase status.ReplicaCountStalled:\n\t\t\tcanBeHiddenIfZero = true\n\t\t}\n\t\tif count == 0 && canBeHiddenIfZero {\n\t\t\tcontinue\n\t\t}\n\t\trows = append(rows, []interface{}{\n\t\t\treplicaCountType,\n\t\t\tcount,\n\t\t})\n\t}\n\n\treturn table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: _titleReplicaStatus, MinWidth: 32, MaxWidth: 32},\n\t\t\t{Title: _titleReplicaCount},\n\t\t},\n\t\tRows: rows,\n\t}\n}\n"
  },
  {
    "path": "cli/cmd/lib_async_apis.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nfunc asyncAPITable(asyncAPI schema.APIResponse, env cliconfig.Environment) (string, error) {\n\tvar out string\n\n\tt := asyncAPIsTable([]schema.APIResponse{asyncAPI}, []string{env.Name})\n\n\tout += t.MustFormat()\n\n\tif asyncAPI.DashboardURL != nil && *asyncAPI.DashboardURL != \"\" {\n\t\tout += \"\\n\" + console.Bold(\"metrics dashboard: \") + *asyncAPI.DashboardURL + \"\\n\"\n\t}\n\n\tif asyncAPI.Endpoint != nil {\n\t\tout += \"\\n\" + console.Bold(\"endpoint: \") + *asyncAPI.Endpoint + \"\\n\"\n\t}\n\n\tout += \"\\n\" + apiHistoryTable(asyncAPI.APIVersions)\n\n\tif !_flagVerbose {\n\t\treturn out, nil\n\t}\n\n\tout += titleStr(\"configuration\") + strings.TrimSpace(asyncAPI.Spec.UserStr())\n\n\treturn out, nil\n}\n\nfunc asyncDescribeAPITable(asyncAPI schema.APIResponse, env cliconfig.Environment) (string, error) {\n\tif asyncAPI.Metadata == nil {\n\t\treturn \"\", errors.ErrorUnexpected(\"missing metadata from operator response\")\n\t}\n\n\tif asyncAPI.Status == nil {\n\t\treturn \"\", errors.ErrorUnexpected(fmt.Sprintf(\"missing status for %s api\", asyncAPI.Metadata.Name))\n\t}\n\n\tt := asyncAPIsTable([]schema.APIResponse{asyncAPI}, []string{env.Name})\n\tout := t.MustFormat()\n\n\tif asyncAPI.DashboardURL != nil && *asyncAPI.DashboardURL != \"\" {\n\t\tout += \"\\n\" + console.Bold(\"metrics dashboard: \") + *asyncAPI.DashboardURL + \"\\n\"\n\t}\n\n\tif asyncAPI.Endpoint != nil {\n\t\tout += \"\\n\" + console.Bold(\"endpoint: \") + *asyncAPI.Endpoint + \"\\n\"\n\t}\n\n\tt = replicaCountTable(asyncAPI.Status.ReplicaCounts)\n\tout += \"\\n\" + t.MustFormat()\n\n\treturn out, nil\n}\n\nfunc asyncAPIsTable(asyncAPIs []schema.APIResponse, envNames []string) table.Table {\n\trows := make([][]interface{}, 0, len(asyncAPIs))\n\n\tfor i, asyncAPI := range asyncAPIs {\n\t\tif asyncAPI.Metadata == nil || asyncAPI.Status == nil {\n\t\t\tcontinue\n\t\t}\n\t\tlastUpdated := time.Unix(asyncAPI.Metadata.LastUpdated, 0)\n\t\trows = append(rows, []interface{}{\n\t\t\tenvNames[i],\n\t\t\tasyncAPI.Metadata.Name,\n\t\t\tfmt.Sprintf(\"%d/%d\", asyncAPI.Status.Ready, asyncAPI.Status.Requested),\n\t\t\tasyncAPI.Status.UpToDate,\n\t\t\tlibtime.SinceStr(&lastUpdated),\n\t\t})\n\t}\n\n\treturn table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: _titleEnvironment},\n\t\t\t{Title: _titleAsyncAPI},\n\t\t\t{Title: _titleLive},\n\t\t\t{Title: _titleUpToDate},\n\t\t\t{Title: _titleLastUpdated},\n\t\t},\n\t\tRows: rows,\n\t}\n}\n"
  },
  {
    "path": "cli/cmd/lib_aws_creds.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n)\n\nfunc newAWSClient(region string, printToStdout bool) (*aws.Client, error) {\n\tif err := clusterconfig.ValidateRegion(region); err != nil {\n\t\treturn nil, err\n\t}\n\n\tawsClient, err := aws.NewForRegion(region)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif _, _, err := awsClient.CheckCredentials(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif printToStdout {\n\t\tfmt.Println(\"using aws credentials with access key \" + *awsClient.AccessKeyID() + \"\\n\")\n\t}\n\n\treturn awsClient, nil\n}\n\nfunc promptIfNotAdmin(awsClient *aws.Client, disallowPrompt bool) {\n\taccessKeyMsg := \"\"\n\tif accessKey := awsClient.AccessKeyID(); accessKey != nil {\n\t\taccessKeyMsg = fmt.Sprintf(\" (with access key %s)\", *accessKey)\n\t}\n\n\tif !awsClient.IsAdmin() {\n\t\twarningStr := fmt.Sprintf(\"warning: your IAM user%s does not have administrator access. Please attach the AdministratorAccess policy to your IAM user (or to a group that your IAM user belongs to), or visit https://docs.cortexlabs.com/v/%s/ to view the minimum permissions required to run `cortex cluster` commands.\\n\\n\", accessKeyMsg, consts.CortexVersionMinor)\n\t\tif disallowPrompt {\n\t\t\tfmt.Print(warningStr)\n\t\t} else {\n\t\t\tprompt.YesOrExit(warningStr+\"are you sure you want to continue without administrator access?\", \"\", \"\")\n\t\t}\n\t}\n}\n\nfunc warnIfNotAdmin(awsClient *aws.Client) {\n\taccessKeyMsg := \"\"\n\tif accessKey := awsClient.AccessKeyID(); accessKey != nil {\n\t\taccessKeyMsg = fmt.Sprintf(\" (with access key %s)\", *accessKey)\n\t}\n\n\tif !awsClient.IsAdmin() {\n\t\tfmt.Println(fmt.Sprintf(\"warning: your IAM user or assumed role%s does not have administrator access. This may prevent this command from executing correctly, so it is recommended to attach the AdministratorAccess policy to your IAM user or role.\\n\", accessKeyMsg), \"\", \"\")\n\t}\n}\n"
  },
  {
    "path": "cli/cmd/lib_batch_apis.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/yaml\"\n)\n\nconst (\n\t_titleBatchAPI    = \"batch api\"\n\t_titleJobCount    = \"running jobs\"\n\t_titleLatestJobID = \"latest job id\"\n)\n\nfunc batchAPIsTable(batchAPIs []schema.APIResponse, envNames []string) table.Table {\n\trows := make([][]interface{}, 0, len(batchAPIs))\n\n\tfor i, batchAPI := range batchAPIs {\n\t\tif batchAPI.Metadata == nil {\n\t\t\tcontinue\n\t\t}\n\t\tlastAPIUpdated := time.Unix(batchAPI.Metadata.LastUpdated, 0)\n\t\tlatestStartTime := time.Time{}\n\t\tlatestJobID := \"-\"\n\t\trunningJobs := 0\n\n\t\tfor _, job := range batchAPI.BatchJobStatuses {\n\t\t\tif job.StartTime.After(latestStartTime) {\n\t\t\t\tlatestStartTime = job.StartTime\n\t\t\t\tlatestJobID = job.ID + fmt.Sprintf(\" (submitted %s ago)\", libtime.SinceStr(&latestStartTime))\n\t\t\t}\n\n\t\t\tif job.Status.IsInProgress() {\n\t\t\t\trunningJobs++\n\t\t\t}\n\t\t}\n\n\t\trows = append(rows, []interface{}{\n\t\t\tenvNames[i],\n\t\t\tbatchAPI.Metadata.Name,\n\t\t\trunningJobs,\n\t\t\tlatestJobID,\n\t\t\tlibtime.SinceStr(&lastAPIUpdated),\n\t\t})\n\t}\n\n\treturn table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: _titleEnvironment},\n\t\t\t{Title: _titleBatchAPI},\n\t\t\t{Title: _titleJobCount},\n\t\t\t{Title: _titleLatestJobID},\n\t\t\t{Title: _titleLastUpdated},\n\t\t},\n\t\tRows: rows,\n\t}\n}\n\nfunc batchAPITable(batchAPI schema.APIResponse) string {\n\tjobRows := make([][]interface{}, 0, len(batchAPI.BatchJobStatuses))\n\n\tout := \"\"\n\tif len(batchAPI.BatchJobStatuses) == 0 {\n\t\tout = console.Bold(\"no submitted batch jobs\\n\")\n\t} else {\n\t\tfor _, job := range batchAPI.BatchJobStatuses {\n\n\t\t\tjobEndTime := time.Now()\n\t\t\tif job.EndTime != nil {\n\t\t\t\tjobEndTime = *job.EndTime\n\t\t\t}\n\n\t\t\tduration := jobEndTime.Sub(job.StartTime).Truncate(time.Second).String()\n\n\t\t\tjobRows = append(jobRows, []interface{}{\n\t\t\t\tjob.ID,\n\t\t\t\tjob.Status.Message(),\n\t\t\t\tjob.TotalBatchCount,\n\t\t\t\tjob.StartTime.Format(_timeFormat),\n\t\t\t\tduration,\n\t\t\t})\n\t\t}\n\n\t\tt := table.Table{\n\t\t\tHeaders: []table.Header{\n\t\t\t\t{Title: \"job id\"},\n\t\t\t\t{Title: \"status\"},\n\t\t\t\t{Title: \"total batches\"},\n\t\t\t\t{Title: \"start time\"},\n\t\t\t\t{Title: \"duration\"},\n\t\t\t},\n\t\t\tRows: jobRows,\n\t\t}\n\n\t\tout += t.MustFormat()\n\t}\n\n\tif batchAPI.DashboardURL != nil && *batchAPI.DashboardURL != \"\" {\n\t\tout += \"\\n\" + console.Bold(\"metrics dashboard: \") + *batchAPI.DashboardURL + \"\\n\"\n\t}\n\n\tif batchAPI.Endpoint != nil {\n\t\tout += \"\\n\" + console.Bold(\"endpoint: \") + *batchAPI.Endpoint + \"\\n\"\n\t}\n\n\tout += \"\\n\" + apiHistoryTable(batchAPI.APIVersions)\n\n\tif !_flagVerbose {\n\t\treturn out\n\t}\n\n\tout += titleStr(\"batch api configuration\") + batchAPI.Spec.UserStr()\n\n\treturn out\n}\n\nfunc getBatchJob(env cliconfig.Environment, apiName string, jobID string) (string, error) {\n\tresp, err := cluster.GetBatchJob(MustGetOperatorConfig(env.Name), apiName, jobID)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tvar bytes []byte\n\tif _flagOutput == flags.JSONOutputType {\n\t\tbytes, err = libjson.Marshal(resp)\n\t} else if _flagOutput == flags.YAMLOutputType {\n\t\tbytes, err = yaml.Marshal(resp)\n\t}\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {\n\t\treturn string(bytes), nil\n\t}\n\n\tjob := resp.JobStatus\n\n\tout := \"\"\n\n\tjobIntroTable := table.KeyValuePairs{}\n\tjobIntroTable.Add(\"job id\", job.ID)\n\tjobIntroTable.Add(\"status\", job.Status.Message())\n\tout += jobIntroTable.String(&table.KeyValuePairOpts{BoldKeys: pointer.Bool(true)})\n\n\tjobTimingTable := table.KeyValuePairs{}\n\tjobTimingTable.Add(\"start time\", job.StartTime.Format(_timeFormat))\n\n\tjobEndTime := time.Now()\n\tif job.EndTime != nil {\n\t\tjobTimingTable.Add(\"end time\", job.EndTime.Format(_timeFormat))\n\t\tjobEndTime = *job.EndTime\n\t} else {\n\t\tjobTimingTable.Add(\"end time\", \"-\")\n\t}\n\tduration := jobEndTime.Sub(job.StartTime).Truncate(time.Second).String()\n\tjobTimingTable.Add(\"duration\", duration)\n\n\tout += \"\\n\" + jobTimingTable.String(&table.KeyValuePairOpts{BoldKeys: pointer.Bool(true)})\n\n\tsucceeded := \"-\"\n\tfailed := \"-\"\n\tavgTimePerBatch := \"-\"\n\n\tif resp.Metrics != nil {\n\t\tif resp.Metrics.AverageTimePerBatch != nil {\n\t\t\tbatchMetricsDuration := time.Duration(*resp.Metrics.AverageTimePerBatch*1000000000) * time.Nanosecond\n\t\t\tavgTimePerBatch = batchMetricsDuration.Truncate(time.Millisecond).String()\n\t\t}\n\n\t\tsucceeded = s.Int(resp.Metrics.Succeeded)\n\t\tfailed = s.Int(resp.Metrics.Failed)\n\t}\n\n\tt := table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: \"total\"},\n\t\t\t{Title: \"succeeded\"},\n\t\t\t{Title: \"failed attempts\"},\n\t\t\t{Title: \"avg time per batch\"},\n\t\t},\n\t\tRows: [][]interface{}{\n\t\t\t{\n\t\t\t\tjob.TotalBatchCount,\n\t\t\t\tsucceeded,\n\t\t\t\tfailed,\n\t\t\t\tavgTimePerBatch,\n\t\t\t},\n\t\t},\n\t}\n\n\tout += titleStr(\"batch stats\") + t.MustFormat(&table.Opts{BoldHeader: pointer.Bool(false)})\n\n\tif job.Status == status.JobEnqueuing {\n\t\tout += \"\\n\" + \"still enqueuing, workers have not been allocated for this job yet\\n\"\n\t} else if job.Status.IsCompleted() {\n\t\tout += \"\\n\" + \"worker stats are not available because this job is not currently running\\n\"\n\t} else {\n\t\tout += titleStr(\"worker stats\")\n\t\tif job.WorkerCounts != nil {\n\t\t\tt := table.Table{\n\t\t\t\tHeaders: []table.Header{\n\t\t\t\t\t{Title: \"Requested\"},\n\t\t\t\t\t{Title: \"Pending\"},\n\t\t\t\t\t{Title: \"Creating\"},\n\t\t\t\t\t{Title: \"Ready\"},\n\t\t\t\t\t{Title: \"NotReady\"},\n\t\t\t\t\t{Title: \"ErrImagePull\", Hidden: job.WorkerCounts.ErrImagePull == 0},\n\t\t\t\t\t{Title: \"Terminating\", Hidden: job.WorkerCounts.Terminating == 0},\n\t\t\t\t\t{Title: \"Failed\", Hidden: job.WorkerCounts.Failed == 0},\n\t\t\t\t\t{Title: \"Killed\", Hidden: job.WorkerCounts.Killed == 0},\n\t\t\t\t\t{Title: \"KilledOOM\", Hidden: job.WorkerCounts.KilledOOM == 0},\n\t\t\t\t\t{Title: \"Stalled\", Hidden: job.WorkerCounts.Stalled == 0},\n\t\t\t\t\t{Title: \"Unknown\", Hidden: job.WorkerCounts.Unknown == 0},\n\t\t\t\t\t{Title: \"Succeeded\"},\n\t\t\t\t},\n\t\t\t\tRows: [][]interface{}{\n\t\t\t\t\t{\n\t\t\t\t\t\tjob.Workers,\n\t\t\t\t\t\tjob.WorkerCounts.Pending,\n\t\t\t\t\t\tjob.WorkerCounts.Creating,\n\t\t\t\t\t\tjob.WorkerCounts.Ready,\n\t\t\t\t\t\tjob.WorkerCounts.NotReady,\n\t\t\t\t\t\tjob.WorkerCounts.ErrImagePull,\n\t\t\t\t\t\tjob.WorkerCounts.Terminating,\n\t\t\t\t\t\tjob.WorkerCounts.Failed,\n\t\t\t\t\t\tjob.WorkerCounts.Killed,\n\t\t\t\t\t\tjob.WorkerCounts.KilledOOM,\n\t\t\t\t\t\tjob.WorkerCounts.Stalled,\n\t\t\t\t\t\tjob.WorkerCounts.Unknown,\n\t\t\t\t\t\tjob.WorkerCounts.Succeeded,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t}\n\t\t\tout += t.MustFormat(&table.Opts{BoldHeader: pointer.Bool(false)})\n\t\t} else {\n\t\t\tout += \"unable to get worker stats\\n\"\n\t\t}\n\t}\n\n\tout += \"\\n\" + console.Bold(\"job endpoint: \") + resp.Endpoint + \"\\n\"\n\n\tjobSpecStr, err := libjson.Pretty(job.BatchJob)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tout += titleStr(\"job configuration\") + jobSpecStr\n\n\treturn out, nil\n}\n"
  },
  {
    "path": "cli/cmd/lib_cli_config.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\tcr \"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/print\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/yaml\"\n)\n\nvar _cliConfigValidation = &cr.StructValidation{\n\tTreatNullAsEmpty: true,\n\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t{\n\t\t\tStructField: \"Telemetry\",\n\t\t\tBoolPtrValidation: &cr.BoolPtrValidation{\n\t\t\t\tRequired: false,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"DefaultEnvironment\",\n\t\t\tStringPtrValidation: &cr.StringPtrValidation{\n\t\t\t\tRequired:          false,\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Environments\",\n\t\t\tStructListValidation: &cr.StructListValidation{\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t\tStructValidation: &cr.StructValidation{\n\t\t\t\t\tAllowExtraFields: true, // backwards compatibility with previous version cli.yaml that require aws creds\n\t\t\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tStructField: \"Name\",\n\t\t\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\t\t\tRequired:  true,\n\t\t\t\t\t\t\t\tMaxLength: 63,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tKey: \"provider\",\n\t\t\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\t\t\tAllowEmpty: true,\n\t\t\t\t\t\t\t\tValidator: func(provider string) (string, error) {\n\t\t\t\t\t\t\t\t\tif provider == \"\" || provider == \"aws\" {\n\t\t\t\t\t\t\t\t\t\treturn \"\", nil\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\tif provider == \"gcp\" || provider == \"local\" {\n\t\t\t\t\t\t\t\t\t\treturn \"\", ErrorInvalidLegacyProvider(provider, _cliConfigPath)\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\treturn \"\", ErrorInvalidProvider(provider, _cliConfigPath)\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tStructField: \"OperatorEndpoint\",\n\t\t\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\t\t\tRequired:  true,\n\t\t\t\t\t\t\t\tValidator: cliconfig.CortexEndpointValidator,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n}\n\nfunc getEnvFromFlag(envFlag string) (string, error) {\n\tif envFlag != \"\" {\n\t\treturn envFlag, nil\n\t}\n\n\tdefaultEnv, err := getDefaultEnv()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif defaultEnv != nil {\n\t\treturn *defaultEnv, nil\n\t}\n\n\tenvs, err := listConfiguredEnvs()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif len(envs) == 0 {\n\t\treturn \"\", ErrorNoAvailableEnvironment()\n\t}\n\n\treturn \"\", ErrorEnvironmentNotSet()\n}\n\nfunc promptForExistingEnvName(promptMsg string) string {\n\tconfiguredEnvNames, err := listConfiguredEnvNames()\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\n\tfmt.Printf(\"your currently configured environments are: %s\\n\\n\", strings.Join(configuredEnvNames, \", \"))\n\n\tenvNameContainer := &struct {\n\t\tEnvironmentName string\n\t}{}\n\n\terr = cr.ReadPrompt(envNameContainer, &cr.PromptValidation{\n\t\tPromptItemValidations: []*cr.PromptItemValidation{\n\t\t\t{\n\t\t\t\tStructField: \"EnvironmentName\",\n\t\t\t\tPromptOpts: &prompt.Options{\n\t\t\t\t\tPrompt: promptMsg,\n\t\t\t\t},\n\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\tRequired:      true,\n\t\t\t\t\tAllowedValues: configuredEnvNames,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\n\treturn envNameContainer.EnvironmentName\n}\n\nfunc promptEnv(env *cliconfig.Environment, defaults cliconfig.Environment) error {\n\tif env.OperatorEndpoint == \"\" {\n\t\tfmt.Print(\"you can get your cortex operator endpoint using `cortex cluster info` if you already have a cortex cluster running, otherwise run `cortex cluster up` to create a cortex cluster\\n\\n\")\n\t}\n\n\tvalidator := func(endpoint string) (string, error) {\n\t\toperatorURL, err := validateOperatorEndpoint(endpoint)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\treturn operatorURL, nil\n\t}\n\n\tfor true {\n\t\terr := cr.ReadPrompt(env, &cr.PromptValidation{\n\t\t\tSkipNonEmptyFields: true,\n\t\t\tPromptItemValidations: []*cr.PromptItemValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Name\",\n\t\t\t\t\tPromptOpts: &prompt.Options{\n\t\t\t\t\t\tPrompt: \"name of environment to create or update\",\n\t\t\t\t\t},\n\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\tRequired: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"OperatorEndpoint\",\n\t\t\t\t\tPromptOpts: &prompt.Options{\n\t\t\t\t\t\tPrompt: \"cortex operator endpoint\",\n\t\t\t\t\t},\n\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\tRequired:  true,\n\t\t\t\t\t\tDefault:   defaults.OperatorEndpoint,\n\t\t\t\t\t\tValidator: validator,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\treturn nil\n\t}\n\n\treturn nil\n}\n\n// Only validate this during prompt, not when reading from file\nfunc validateOperatorEndpoint(endpoint string) (string, error) {\n\turl, err := cliconfig.CortexEndpointValidator(endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tparsedURL, err := urls.Parse(url)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\turl = parsedURL.String()\n\n\treq, err := http.NewRequest(\"GET\", urls.Join(url, \"/verifycortex\"), nil)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, \"verifying operator endpoint\", url)\n\t}\n\n\tclient := http.Client{\n\t\tTransport: &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t\t},\n\t}\n\n\tresponse, err := client.Do(req)\n\tif err != nil {\n\t\treturn \"\", ErrorInvalidOperatorEndpoint(url)\n\t}\n\tdefer response.Body.Close()\n\n\tif response.StatusCode != 200 {\n\t\treturn \"\", ErrorInvalidOperatorEndpoint(url)\n\t}\n\n\tbodyBytes, err := ioutil.ReadAll(response.Body)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, _errStrRead)\n\t}\n\n\tvar verifyCortex schema.VerifyCortexResponse\n\tif err = json.Unmarshal(bodyBytes, &verifyCortex); err != nil {\n\t\treturn \"\", errors.Wrap(err, endpoint, string(bodyBytes))\n\t}\n\n\treturn url, nil\n}\n\nfunc getDefaultEnv() (*string, error) {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif cliConfig.DefaultEnvironment != nil {\n\t\treturn cliConfig.DefaultEnvironment, nil\n\t}\n\n\tif len(cliConfig.Environments) == 1 {\n\t\tdefaultEnv := cliConfig.Environments[0].Name\n\t\terr := setDefaultEnv(defaultEnv)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &defaultEnv, nil\n\t}\n\n\treturn nil, nil\n}\n\nfunc setDefaultEnv(envName string) error {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tenvExists, err := isEnvConfigured(envName)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !envExists {\n\t\treturn cliconfig.ErrorEnvironmentNotConfigured(envName)\n\t}\n\n\tcliConfig.DefaultEnvironment = &envName\n\n\tif err := writeCLIConfig(cliConfig); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc renameEnv(oldEnvName string, newEnvName string) error {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\trenamedEnv := false\n\n\tfor _, env := range cliConfig.Environments {\n\t\tif env.Name == newEnvName {\n\t\t\treturn cliconfig.ErrorEnvironmentAlreadyConfigured(newEnvName)\n\t\t}\n\n\t\tif env.Name == oldEnvName {\n\t\t\tenv.Name = newEnvName\n\t\t\trenamedEnv = true\n\t\t}\n\t}\n\n\tif !renamedEnv {\n\t\treturn cliconfig.ErrorEnvironmentNotConfigured(oldEnvName)\n\t}\n\n\tif cliConfig.DefaultEnvironment != nil && *cliConfig.DefaultEnvironment == oldEnvName {\n\t\tcliConfig.DefaultEnvironment = &newEnvName\n\t}\n\n\tif err := writeCLIConfig(cliConfig); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc readTelemetryConfig() (bool, error) {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif cliConfig.Telemetry != nil && *cliConfig.Telemetry == false {\n\t\treturn false, nil\n\t}\n\n\treturn true, nil\n}\n\n// Returns false if there is an error reading the CLI config\nfunc isTelemetryEnabled() bool {\n\tenabled, err := readTelemetryConfig()\n\tif err != nil {\n\t\treturn false\n\t}\n\treturn enabled\n}\n\n// Will return nil if not configured\nfunc readEnv(envName string) (*cliconfig.Environment, error) {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, env := range cliConfig.Environments {\n\t\tif env.Name == envName {\n\t\t\treturn env, nil\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\nfunc ReadOrConfigureEnv(envName string) (cliconfig.Environment, error) {\n\texistingEnv, err := readEnv(envName)\n\tif err != nil {\n\t\treturn cliconfig.Environment{}, err\n\t}\n\n\tif existingEnv != nil {\n\t\treturn *existingEnv, nil\n\t}\n\n\tpromptStr := fmt.Sprintf(\"the %s environment is not configured; do you already have a Cortex cluster running?\", envName)\n\tyesMsg := fmt.Sprintf(\"please configure the %s environment to point to your running cluster:\\n\", envName)\n\tnoMsg := \"you can create a cluster by running the `cortex cluster up` command\"\n\tprompt.YesOrExit(promptStr, yesMsg, noMsg)\n\n\tenv, err := configureEnv(envName, cliconfig.Environment{})\n\tif err != nil {\n\t\treturn cliconfig.Environment{}, err\n\t}\n\n\treturn env, nil\n}\n\nfunc getEnvConfigDefaults(envName string) cliconfig.Environment {\n\tdefaults := cliconfig.Environment{}\n\n\tprevEnv, err := readEnv(envName)\n\tif err == nil && prevEnv != nil {\n\t\tdefaults = *prevEnv\n\t}\n\n\tif defaults.OperatorEndpoint == \"\" && os.Getenv(\"CORTEX_OPERATOR_ENDPOINT\") != \"\" {\n\t\tdefaults.OperatorEndpoint = os.Getenv(\"CORTEX_OPERATOR_ENDPOINT\")\n\t}\n\n\treturn defaults\n}\n\n// If envName is \"\", this will prompt for the environment name to configure\nfunc configureEnv(envName string, fieldsToSkipPrompt cliconfig.Environment) (cliconfig.Environment, error) {\n\tenv := cliconfig.Environment{\n\t\tName:             envName,\n\t\tOperatorEndpoint: fieldsToSkipPrompt.OperatorEndpoint,\n\t}\n\n\tdefaults := getEnvConfigDefaults(env.Name)\n\n\terr := promptEnv(&env, defaults)\n\tif err != nil {\n\t\treturn cliconfig.Environment{}, err\n\t}\n\n\tif err := env.Validate(); err != nil {\n\t\treturn cliconfig.Environment{}, err\n\t}\n\n\tif err := addEnvToCLIConfig(env, false); err != nil {\n\t\treturn cliconfig.Environment{}, err\n\t}\n\n\tprint.BoldFirstLine(fmt.Sprintf(\"configured %s environment\", env.Name))\n\n\treturn env, nil\n}\n\nfunc MustGetOperatorConfig(envName string) cluster.OperatorConfig {\n\tclientID := clientID()\n\tenv, err := readEnv(envName)\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\n\tif env == nil {\n\t\texit.Error(ErrorEnvironmentNotFound(envName))\n\t}\n\n\toperatorConfig := cluster.OperatorConfig{\n\t\tTelemetry: isTelemetryEnabled(),\n\t\tClientID:  clientID,\n\t\tEnvName:   env.Name,\n\t}\n\n\tif env.OperatorEndpoint == \"\" {\n\t\texit.Error(ErrorFieldNotFoundInEnvironment(cliconfig.OperatorEndpointKey, env.Name))\n\t}\n\toperatorConfig.OperatorEndpoint = env.OperatorEndpoint\n\n\treturn operatorConfig\n}\n\nfunc listConfiguredEnvs() ([]*cliconfig.Environment, error) {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn cliConfig.Environments, nil\n}\n\nfunc listConfiguredEnvNames() ([]string, error) {\n\tenvList, err := listConfiguredEnvs()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tenvNames := make([]string, len(envList))\n\tfor i, env := range envList {\n\t\tenvNames[i] = env.Name\n\t}\n\n\treturn envNames, nil\n}\n\nfunc isEnvConfigured(envName string) (bool, error) {\n\tenvList, err := listConfiguredEnvs()\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tfor _, env := range envList {\n\t\tif env.Name == envName {\n\t\t\treturn true, nil\n\t\t}\n\t}\n\n\treturn false, nil\n}\n\nfunc addEnvToCLIConfig(newEnv cliconfig.Environment, setAsDefault bool) error {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"unable to configure cli environment\")\n\t}\n\n\treplaced := false\n\tfor i, prevEnv := range cliConfig.Environments {\n\t\tif prevEnv.Name == newEnv.Name {\n\t\t\tcliConfig.Environments[i] = &newEnv\n\t\t\treplaced = true\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif !replaced {\n\t\tcliConfig.Environments = append(cliConfig.Environments, &newEnv)\n\t}\n\n\tif setAsDefault {\n\t\tcliConfig.DefaultEnvironment = &newEnv.Name\n\t}\n\n\tif err := writeCLIConfig(cliConfig); err != nil {\n\t\treturn errors.Wrap(err, \"unable to configure cli environment\")\n\t}\n\n\treturn nil\n}\n\nfunc removeEnvFromCLIConfig(envName string) error {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tprevDefault, err := getDefaultEnv()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar updatedEnvs []*cliconfig.Environment\n\tdeleted := false\n\tfor _, env := range cliConfig.Environments {\n\t\tif env.Name == envName {\n\t\t\tdeleted = true\n\t\t\tcontinue\n\t\t}\n\t\tupdatedEnvs = append(updatedEnvs, env)\n\t}\n\n\tif !deleted {\n\t\treturn cliconfig.ErrorEnvironmentNotConfigured(envName)\n\t}\n\n\tcliConfig.Environments = updatedEnvs\n\n\tif prevDefault != nil && envName == *prevDefault {\n\t\tcliConfig.DefaultEnvironment = nil\n\t}\n\tif len(cliConfig.Environments) == 1 {\n\t\tcliConfig.DefaultEnvironment = &cliConfig.Environments[0].Name\n\t}\n\n\tif err := writeCLIConfig(cliConfig); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// returns the list of environment names, and whether any of them were the default\nfunc getEnvNamesByOperatorEndpoint(operatorEndpoint string) ([]string, bool, error) {\n\tcliConfig, err := readCLIConfig()\n\tif err != nil {\n\t\treturn nil, false, err\n\t}\n\n\tvar envNames []string\n\tisDefaultEnv := false\n\n\tfor _, env := range cliConfig.Environments {\n\t\tif env.OperatorEndpoint != \"\" && s.LastSplit(env.OperatorEndpoint, \"//\") == s.LastSplit(operatorEndpoint, \"//\") {\n\t\t\tenvNames = append(envNames, env.Name)\n\t\t\tif cliConfig.DefaultEnvironment != nil && env.Name == *cliConfig.DefaultEnvironment {\n\t\t\t\tisDefaultEnv = true\n\t\t\t}\n\t\t}\n\t}\n\n\treturn envNames, isDefaultEnv, nil\n}\n\nfunc readCLIConfig() (cliconfig.CLIConfig, error) {\n\tif !files.IsFile(_cliConfigPath) {\n\t\tcliConfig := cliconfig.CLIConfig{}\n\n\t\tif err := cliConfig.Validate(); err != nil {\n\t\t\treturn cliconfig.CLIConfig{}, err // unexpected\n\t\t}\n\n\t\t// create file so that the file created by the manager container maintains current user permissions\n\t\tif err := writeCLIConfig(cliConfig); err != nil {\n\t\t\treturn cliconfig.CLIConfig{}, errors.Wrap(err, \"unable to save CLI configuration file\")\n\t\t}\n\n\t\treturn cliConfig, nil\n\t}\n\n\tcliConfig := cliconfig.CLIConfig{}\n\terrs := cr.ParseYAMLFile(&cliConfig, _cliConfigValidation, _cliConfigPath)\n\tif errors.HasError(errs) {\n\t\treturn cliconfig.CLIConfig{}, errors.FirstError(errs...)\n\t}\n\n\tif err := cliConfig.Validate(); err != nil {\n\t\treturn cliconfig.CLIConfig{}, errors.Wrap(err, _cliConfigPath)\n\t}\n\n\treturn cliConfig, nil\n}\n\nfunc writeCLIConfig(cliConfig cliconfig.CLIConfig) error {\n\tif err := cliConfig.Validate(); err != nil {\n\t\treturn err\n\t}\n\n\tcliConfigBytes, err := yaml.Marshal(cliConfig)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\tif err := files.WriteFile(cliConfigBytes, _cliConfigPath); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "cli/cmd/lib_client_id.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/google/uuid\"\n)\n\nvar _cachedClientID string\n\nfunc clientID() string {\n\tif _cachedClientID != \"\" {\n\t\treturn _cachedClientID\n\t}\n\n\tvar err error\n\t_cachedClientID, err = files.ReadFile(_clientIDPath)\n\tif err != nil || _cachedClientID == \"\" {\n\t\t_cachedClientID = uuid.New().String()\n\t\tfiles.WriteFile([]byte(_cachedClientID), _clientIDPath)\n\t}\n\n\treturn _cachedClientID\n}\n"
  },
  {
    "path": "cli/cmd/lib_cluster_config.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"regexp\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\tcr \"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/maps\"\n\tlibmath \"github.com/cortexlabs/cortex/pkg/lib/math\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterstate\"\n)\n\nvar _cachedClusterConfigRegex = regexp.MustCompile(`^cluster_\\S+\\.yaml$`)\n\nfunc getCachedClusterConfigPath(clusterName string, region string) string {\n\treturn filepath.Join(_localDir, fmt.Sprintf(\"cluster_%s_%s.yaml\", clusterName, region))\n}\n\nfunc existingCachedClusterConfigPaths() []string {\n\tpaths, err := files.ListDir(_localDir, false)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tvar matches []string\n\tfor _, p := range paths {\n\t\tif _cachedClusterConfigRegex.MatchString(path.Base(p)) {\n\t\t\tmatches = append(matches, p)\n\t\t}\n\t}\n\n\treturn matches\n}\n\nfunc readCachedClusterConfigFile(clusterConfig *clusterconfig.Config, filePath string) error {\n\terrs := cr.ParseYAMLFile(clusterConfig, clusterconfig.FullConfigValidation, filePath)\n\tif errors.HasError(errs) {\n\t\treturn errors.FirstError(errs...)\n\t}\n\n\treturn nil\n}\n\nfunc readUserClusterConfigFile(clusterConfig *clusterconfig.Config, filePath string) error {\n\terrs := cr.ParseYAMLFile(clusterConfig, clusterconfig.FullConfigValidation, filePath)\n\tif errors.HasError(errs) {\n\t\treturn errors.Append(errors.FirstError(errs...), fmt.Sprintf(\"\\n\\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t}\n\n\treturn nil\n}\n\nfunc getNewClusterAccessConfig(clusterConfigFile string) (*clusterconfig.AccessConfig, error) {\n\taccessConfig := &clusterconfig.AccessConfig{}\n\n\terrs := cr.ParseYAMLFile(accessConfig, clusterconfig.AccessValidation, clusterConfigFile)\n\tif errors.HasError(errs) {\n\t\treturn nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf(\"\\n\\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t}\n\n\treturn accessConfig, nil\n}\n\nfunc getClusterAccessConfigWithCache(hasClusterFlags bool) (*clusterconfig.AccessConfig, error) {\n\taccessConfig := &clusterconfig.AccessConfig{\n\t\tImageManager: consts.DefaultRegistry() + \"/manager:\" + consts.CortexVersion,\n\t}\n\n\tcachedPaths := existingCachedClusterConfigPaths()\n\tif len(cachedPaths) == 1 {\n\t\tcachedAccessConfig := &clusterconfig.AccessConfig{}\n\t\tcr.ParseYAMLFile(cachedAccessConfig, clusterconfig.AccessValidation, cachedPaths[0])\n\t\taccessConfig.ClusterName = cachedAccessConfig.ClusterName\n\t\taccessConfig.Region = cachedAccessConfig.Region\n\t}\n\n\tif _flagClusterConfig != \"\" {\n\t\terrs := cr.ParseYAMLFile(accessConfig, clusterconfig.AccessValidation, _flagClusterConfig)\n\t\tif errors.HasError(errs) {\n\t\t\treturn nil, errors.Append(errors.FirstError(errs...), fmt.Sprintf(\"\\n\\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t\t}\n\t}\n\n\tif _flagClusterName != \"\" {\n\t\taccessConfig.ClusterName = _flagClusterName\n\t}\n\tif _flagClusterRegion != \"\" {\n\t\taccessConfig.Region = _flagClusterRegion\n\t}\n\n\tif accessConfig.ClusterName == \"\" || accessConfig.Region == \"\" {\n\t\treturn nil, ErrorClusterAccessConfigRequired(hasClusterFlags)\n\t}\n\treturn accessConfig, nil\n}\n\nfunc getInstallClusterConfig(awsClient *aws.Client, clusterConfigFile string) (*clusterconfig.Config, error) {\n\tclusterConfig := &clusterconfig.Config{}\n\n\terr := readUserClusterConfigFile(clusterConfig, clusterConfigFile)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tclusterConfig.Telemetry = isTelemetryEnabled()\n\n\terr = clusterConfig.ValidateOnInstall(awsClient)\n\tif err != nil {\n\t\terr = errors.Append(err, fmt.Sprintf(\"\\n\\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t\treturn nil, errors.Wrap(err, clusterConfigFile)\n\t}\n\n\treturn clusterConfig, nil\n}\n\nfunc getConfigureClusterConfig(awsClient *aws.Client, k8sClient *k8s.Client, stacks clusterstate.ClusterStacks, cachedClusterConfig clusterconfig.Config, newClusterConfigFile string) (*clusterconfig.Config, clusterconfig.ConfigureChanges, error) {\n\tnewUserClusterConfig := &clusterconfig.Config{}\n\n\terr := readUserClusterConfigFile(newUserClusterConfig, newClusterConfigFile)\n\tif err != nil {\n\t\treturn nil, clusterconfig.ConfigureChanges{}, err\n\t}\n\n\tnewUserClusterConfig.Telemetry = isTelemetryEnabled()\n\tcachedClusterConfig.Telemetry = newUserClusterConfig.Telemetry\n\n\tconfigureChanges, err := newUserClusterConfig.ValidateOnConfigure(awsClient, k8sClient, cachedClusterConfig, stacks.NodeGroupsStacks)\n\tif err != nil {\n\t\terr = errors.Append(err, fmt.Sprintf(\"\\n\\ncluster configuration schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t\treturn nil, clusterconfig.ConfigureChanges{}, errors.Wrap(err, newClusterConfigFile)\n\t}\n\n\treturn newUserClusterConfig, configureChanges, nil\n}\n\nfunc confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsClient *aws.Client, disallowPrompt bool) {\n\teksPrice := aws.EKSPrices[clusterConfig.Region]\n\toperatorInstancePrice := aws.InstanceMetadatas[clusterConfig.Region][\"t3.medium\"].Price\n\tprometheusInstancePrice := aws.InstanceMetadatas[clusterConfig.Region][clusterConfig.PrometheusInstanceType].Price\n\toperatorEBSPrice := aws.EBSMetadatas[clusterConfig.Region][\"gp3\"].PriceGB * 20 / 30 / 24\n\tprometheusEBSPrice := aws.EBSMetadatas[clusterConfig.Region][\"gp3\"].PriceGB * 20 / 30 / 24\n\tmetricsEBSPrice := aws.EBSMetadatas[clusterConfig.Region][\"gp2\"].PriceGB * (40 + 2) / 30 / 24\n\tnlbPrice := aws.NLBMetadatas[clusterConfig.Region].Price\n\telbPrice := aws.ELBMetadatas[clusterConfig.Region].Price\n\tnatUnitPrice := aws.NATMetadatas[clusterConfig.Region].Price\n\n\tvar loadBalancersPrice float64\n\tusesELBForAPILoadBalancer := clusterConfig.APILoadBalancerType == clusterconfig.ELBLoadBalancerType\n\tif usesELBForAPILoadBalancer {\n\t\tloadBalancersPrice = nlbPrice + elbPrice\n\t} else {\n\t\tloadBalancersPrice = 2 * nlbPrice\n\t}\n\n\tvar natTotalPrice float64\n\tif clusterConfig.NATGateway == clusterconfig.SingleNATGateway {\n\t\tnatTotalPrice = natUnitPrice\n\t} else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway {\n\t\tnatTotalPrice = natUnitPrice * float64(len(clusterConfig.AvailabilityZones))\n\t}\n\n\theaders := []table.Header{\n\t\t{Title: \"aws resource\"},\n\t\t{Title: \"cost per hour\"},\n\t}\n\n\tvar rows [][]interface{}\n\trows = append(rows, []interface{}{\"1 eks cluster\", s.DollarsMaxPrecision(eksPrice)})\n\n\tngNameToSpotInstancesUsed := map[string]int{}\n\tfixedPrice := eksPrice + 2*(operatorInstancePrice+operatorEBSPrice) + prometheusInstancePrice + prometheusEBSPrice + metricsEBSPrice + loadBalancersPrice + natTotalPrice\n\ttotalMinPrice := fixedPrice\n\ttotalMaxPrice := fixedPrice\n\tfor _, ng := range clusterConfig.NodeGroups {\n\t\tapiInstancePrice := aws.InstanceMetadatas[clusterConfig.Region][ng.InstanceType].Price\n\t\tapiEBSPrice := aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24\n\t\tif ng.InstanceVolumeType == clusterconfig.IO1VolumeType && ng.InstanceVolumeIOPS != nil {\n\t\t\tapiEBSPrice += aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24\n\t\t}\n\t\tif ng.InstanceVolumeType == clusterconfig.GP3VolumeType && ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput != nil {\n\t\t\tapiEBSPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS-3000)*float64(*ng.InstanceVolumeIOPS)/30/24)\n\t\t\tapiEBSPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceThroughput-125)*float64(*ng.InstanceVolumeThroughput)/30/24)\n\t\t}\n\n\t\ttotalMaxPrice += float64(ng.MaxInstances) * (apiInstancePrice + apiEBSPrice)\n\n\t\tworkerInstanceStr := fmt.Sprintf(\"nodegroup %s: %d-%d %s instances\", ng.Name, ng.MinInstances, ng.MaxInstances, ng.InstanceType)\n\t\tif ng.MinInstances == ng.MaxInstances {\n\t\t\tworkerInstanceStr = fmt.Sprintf(\"nodegroup %s: %d %s %s\", ng.Name, ng.MinInstances, ng.InstanceType, s.PluralS(\"instance\", ng.MinInstances))\n\t\t}\n\n\t\tworkerPriceStr := s.DollarsAndTenthsOfCents(apiInstancePrice+apiEBSPrice) + \" each\"\n\t\tif ng.Spot {\n\t\t\tngNameToSpotInstancesUsed[ng.Name]++\n\t\t\tspotPrice, err := awsClient.SpotInstancePrice(ng.InstanceType)\n\t\t\tworkerPriceStr += \" (spot pricing unavailable)\"\n\t\t\tif err == nil && spotPrice != 0 {\n\t\t\t\tworkerPriceStr = fmt.Sprintf(\"%s - %s each (varies based on spot price)\", s.DollarsAndTenthsOfCents(spotPrice+apiEBSPrice), s.DollarsAndTenthsOfCents(apiInstancePrice+apiEBSPrice))\n\t\t\t\tif ng.MinInstances > *ng.SpotConfig.OnDemandBaseCapacity {\n\t\t\t\t\ttotalMinPrice += float64(ng.MinInstances-*ng.SpotConfig.OnDemandBaseCapacity)*(spotPrice+apiEBSPrice)*float64(100-*ng.SpotConfig.OnDemandPercentageAboveBaseCapacity)/100 +\n\t\t\t\t\t\tfloat64(ng.MinInstances-*ng.SpotConfig.OnDemandBaseCapacity)*(apiInstancePrice+apiEBSPrice)*float64(*ng.SpotConfig.OnDemandPercentageAboveBaseCapacity)/100 +\n\t\t\t\t\t\tfloat64(*ng.SpotConfig.OnDemandBaseCapacity)*(apiInstancePrice+apiEBSPrice)\n\t\t\t\t} else {\n\t\t\t\t\ttotalMinPrice += float64(ng.MinInstances) * (apiInstancePrice + apiEBSPrice)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\ttotalMinPrice += float64(ng.MinInstances) * (apiInstancePrice + apiEBSPrice)\n\t\t\t}\n\t\t} else {\n\t\t\ttotalMinPrice += float64(ng.MinInstances) * (apiInstancePrice + apiEBSPrice)\n\t\t}\n\n\t\trows = append(rows, []interface{}{workerInstanceStr, workerPriceStr})\n\t}\n\n\toperatorNodeGroupPrice := 2 * (operatorInstancePrice + operatorEBSPrice)\n\tprometheusNodeGroupPrice := prometheusInstancePrice + prometheusEBSPrice + metricsEBSPrice\n\trows = append(rows, []interface{}{\"2 t3.medium instances (cortex system)\", s.DollarsAndTenthsOfCents(operatorNodeGroupPrice) + \" total\"})\n\trows = append(rows, []interface{}{fmt.Sprintf(\"1 %s instance (prometheus)\", clusterConfig.PrometheusInstanceType), s.DollarsAndTenthsOfCents(prometheusNodeGroupPrice)})\n\tif usesELBForAPILoadBalancer {\n\t\trows = append(rows, []interface{}{\"1 network load balancer\", s.DollarsMaxPrecision(nlbPrice)})\n\t\trows = append(rows, []interface{}{\"1 classic load balancer\", s.DollarsMaxPrecision(elbPrice)})\n\t} else {\n\t\trows = append(rows, []interface{}{\"2 network load balancers\", s.DollarsMaxPrecision(loadBalancersPrice) + \" total\"})\n\t}\n\n\tif clusterConfig.NATGateway == clusterconfig.SingleNATGateway {\n\t\trows = append(rows, []interface{}{\"1 nat gateway\", s.DollarsMaxPrecision(natUnitPrice)})\n\t} else if clusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway {\n\t\trows = append(rows, []interface{}{fmt.Sprintf(\"%d nat gateways\", len(clusterConfig.AvailabilityZones)), s.DollarsMaxPrecision(natUnitPrice) + \" each\"})\n\t}\n\n\titems := table.Table{\n\t\tHeaders: headers,\n\t\tRows:    rows,\n\t}\n\tfmt.Println(items.MustFormat(&table.Opts{Sort: pointer.Bool(false)}))\n\n\tpriceStr := s.DollarsAndCents(totalMaxPrice)\n\tsuffix := \"\"\n\tif totalMinPrice != totalMaxPrice {\n\t\tpriceStr = fmt.Sprintf(\"%s - %s\", s.DollarsAndCents(totalMinPrice), s.DollarsAndCents(totalMaxPrice))\n\t\tif len(ngNameToSpotInstancesUsed) > 0 && len(ngNameToSpotInstancesUsed) < len(clusterConfig.NodeGroups) {\n\t\t\tsuffix = \" based on cluster size and spot instance pricing/availability\"\n\t\t} else if len(ngNameToSpotInstancesUsed) == len(clusterConfig.NodeGroups) {\n\t\t\tsuffix = \" based on spot instance pricing/availability\"\n\t\t} else if len(ngNameToSpotInstancesUsed) == 0 {\n\t\t\tsuffix = \" based on cluster size\"\n\t\t}\n\t}\n\n\tfmt.Printf(\"your cluster will cost %s per hour%s\\n\\n\", priceStr, suffix)\n\n\tprivateSubnetMsg := \"\"\n\tif clusterConfig.SubnetVisibility == clusterconfig.PrivateSubnetVisibility {\n\t\tprivateSubnetMsg = \", and will use private subnets for all EC2 instances\"\n\t}\n\tfmt.Printf(\"cortex will also create an s3 bucket (%s) and a cloudwatch log group (%s)%s\\n\\n\", clusterConfig.Bucket, clusterConfig.ClusterName, privateSubnetMsg)\n\n\tif clusterConfig.OperatorLoadBalancerScheme == clusterconfig.InternalLoadBalancerScheme {\n\t\tfmt.Print(fmt.Sprintf(\"warning: you've configured the operator load balancer to be internal; you must configure VPC Peering to connect your CLI to your cluster operator (see https://docs.cortexlabs.com/v/%s/)\\n\\n\", consts.CortexVersionMinor))\n\t}\n\n\tif len(clusterConfig.Subnets) > 0 {\n\t\tfmt.Print(\"warning: you've configured your cluster to be installed in an existing VPC; if your cluster doesn't spin up or function as expected, please double-check your VPC configuration (here are the requirements: https://eksctl.io/usage/vpc-networking/#use-existing-vpc-other-custom-configuration)\\n\\n\")\n\t}\n\n\tif len(clusterConfig.NodeGroups) > 1 && len(ngNameToSpotInstancesUsed) > 0 {\n\t\tfmt.Printf(\"warning: you've enabled spot instances for %s %s; spot instances are not guaranteed to be available so please take that into account for production clusters; see https://docs.cortexlabs.com/v/%s/ for more information\\n\\n\", s.PluralS(\"nodegroup\", len(ngNameToSpotInstancesUsed)), s.StrsAnd(maps.StrMapKeysInt(ngNameToSpotInstancesUsed)), consts.CortexVersionMinor)\n\t}\n\n\tif !disallowPrompt {\n\t\texitMessage := fmt.Sprintf(\"cluster configuration can be modified via the cluster config file; see https://docs.cortexlabs.com/v/%s/ for more information\", consts.CortexVersionMinor)\n\t\tprompt.YesOrExit(\"would you like to continue?\", \"\", exitMessage)\n\t}\n}\n\nfunc confirmConfigureClusterConfig(configureChanges clusterconfig.ConfigureChanges, oldCc, newCc clusterconfig.Config, disallowPrompt bool) {\n\tfmt.Printf(\"your %s cluster in region %s will be updated as follows:\\n\\n\", newCc.ClusterName, newCc.Region)\n\n\tfor _, fieldToUpdate := range configureChanges.FieldsToUpdate {\n\t\tfmt.Printf(\"￮ %s will be updated\\n\", fieldToUpdate)\n\t}\n\n\tfor _, ngName := range configureChanges.NodeGroupsToUpdate {\n\t\tngOld := oldCc.GetNodeGroupByName(ngName)\n\t\tngNew := newCc.GetNodeGroupByName(ngName)\n\t\tfmt.Printf(\"￮ %s\\n\", ngNew.UpdatePlan(ngOld))\n\t}\n\n\tfor _, ngName := range configureChanges.NodeGroupsToAdd {\n\t\tfmt.Printf(\"￮ nodegroup %s will be added\\n\", ngName)\n\t}\n\n\tfor _, ngName := range configureChanges.NodeGroupsToRemove {\n\t\tfmt.Printf(\"￮ nodegroup %s will be removed\\n\", ngName)\n\t}\n\n\t// EKS node groups that don't appear in the old/new cluster config; this is unlikely but can happen\n\tfor _, ngName := range configureChanges.GetGhostEKSNodeGroups() {\n\t\tfmt.Printf(\"￮ EKS nodegroup %s will be removed\\n\", ngName)\n\t}\n\n\tfmt.Println()\n\n\tif !disallowPrompt {\n\t\texitMessage := fmt.Sprintf(\"cluster configuration can be modified via the cluster config file; see https://docs.cortexlabs.com/v/%s/ for more information\", consts.CortexVersionMinor)\n\t\tprompt.YesOrExit(fmt.Sprintf(\"your cluster named \\\"%s\\\" in %s will be updated according to the configuration above, are you sure you want to continue?\", newCc.ClusterName, newCc.Region), \"\", exitMessage)\n\t}\n}\n"
  },
  {
    "path": "cli/cmd/lib_manager.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/signal\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/cli/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/archive\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/docker\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/yaml\"\n\tdockertypes \"github.com/docker/docker/api/types\"\n\t\"github.com/docker/docker/api/types/container\"\n)\n\ntype dockerCopyFromPath struct {\n\tcontainerPath string\n\tlocalDir      string\n}\n\ntype dockerCopyToPath struct {\n\tinput         *archive.Input\n\tcontainerPath string\n}\n\nfunc runManager(containerConfig *container.Config, addNewLineAfterPull bool, copyToPaths []dockerCopyToPath, copyFromPaths []dockerCopyFromPath) (string, *int, error) {\n\tcontainerConfig.Env = append(containerConfig.Env, \"CORTEX_CLI_VERSION=\"+consts.CortexVersion)\n\n\t// Add a slight delay before running the command to ensure logs don't start until after the container is attached\n\tcontainerConfig.Cmd[0] = \"sleep 0.1 && /root/check_cortex_version.sh && \" + containerConfig.Cmd[0]\n\n\tdockerClient, err := docker.GetDockerClient()\n\tif err != nil {\n\t\treturn \"\", nil, err\n\t}\n\n\tpulledImage, err := docker.PullImage(containerConfig.Image, docker.NoAuth, docker.PrintDots)\n\tif err != nil {\n\t\tif strings.Contains(err.Error(), \"auth\") {\n\t\t\terr = errors.Append(err, fmt.Sprintf(\"\\n\\nif your manager image is stored in a private repository: run `docker login` (if you haven't already), download your image with `docker pull %s`, and try this command again)\", containerConfig.Image))\n\t\t}\n\t\treturn \"\", nil, err\n\t}\n\n\tif pulledImage && addNewLineAfterPull {\n\t\tfmt.Println()\n\t}\n\n\tcontainerInfo, err := dockerClient.ContainerCreate(context.Background(), containerConfig, nil, nil, \"\")\n\tif err != nil {\n\t\treturn \"\", nil, docker.WrapDockerError(err)\n\t}\n\n\tremoveContainer := func() {\n\t\t_ = dockerClient.ContainerRemove(context.Background(), containerInfo.ID, dockertypes.ContainerRemoveOptions{\n\t\t\tRemoveVolumes: true,\n\t\t\tForce:         true,\n\t\t})\n\t}\n\n\tdefer removeContainer()\n\n\t// Make sure to remove container immediately on ctrl+c\n\tc := make(chan os.Signal, 1)\n\tsignal.Notify(c, os.Interrupt, syscall.SIGTERM)\n\tcaughtCtrlC := false\n\n\troutines.RunWithPanicHandler(func() {\n\t\t<-c\n\t\tcaughtCtrlC = true\n\t\tremoveContainer()\n\t\texit.Error(ErrorDockerCtrlC())\n\t}, false)\n\n\tfor _, copyPath := range copyToPaths {\n\t\terr = docker.CopyToContainer(containerInfo.ID, copyPath.input, copyPath.containerPath)\n\t\tif err != nil {\n\t\t\treturn \"\", nil, err\n\t\t}\n\t}\n\n\terr = dockerClient.ContainerStart(context.Background(), containerInfo.ID, dockertypes.ContainerStartOptions{})\n\tif err != nil {\n\t\treturn \"\", nil, docker.WrapDockerError(err)\n\t}\n\n\t// Use ContainerAttach() since that allows logs to be streamed even if they don't end in new lines\n\tlogsOutput, err := dockerClient.ContainerAttach(context.Background(), containerInfo.ID, dockertypes.ContainerAttachOptions{\n\t\tStream: true,\n\t\tStdout: true,\n\t\tStderr: true,\n\t})\n\tif err != nil {\n\t\treturn \"\", nil, docker.WrapDockerError(err)\n\t}\n\tdefer logsOutput.Close()\n\n\tvar outputBuffer bytes.Buffer\n\ttee := io.TeeReader(logsOutput.Reader, &outputBuffer)\n\n\t_, err = io.Copy(os.Stdout, tee)\n\tif err != nil && err != io.EOF {\n\t\treturn \"\", nil, errors.WithStack(err)\n\t}\n\n\toutput := strings.ReplaceAll(outputBuffer.String(), \"\\r\\n\", \"\\n\")\n\n\t// Let the ctrl+c handler run its course\n\tif caughtCtrlC {\n\t\ttime.Sleep(5 * time.Second)\n\t}\n\n\tinfo, err := dockerClient.ContainerInspect(context.Background(), containerInfo.ID)\n\tif err != nil {\n\t\treturn \"\", nil, errors.WithStack(err)\n\t}\n\n\tif info.State.ExitCode == 0 {\n\t\tfor _, copyPath := range copyFromPaths {\n\t\t\terr = docker.CopyFromContainer(containerInfo.ID, copyPath.containerPath, copyPath.localDir)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\tif info.State.Running {\n\t\treturn output, nil, nil\n\t}\n\n\treturn output, &info.State.ExitCode, nil\n}\n\nfunc runManagerWithClusterConfig(entrypoint string, clusterConfig *clusterconfig.Config, awsClient *aws.Client, copyToPaths []dockerCopyToPath, copyFromPaths []dockerCopyFromPath, extraEnvs []string) (string, *int, error) {\n\tclusterConfigBytes, err := yaml.Marshal(clusterConfig)\n\tif err != nil {\n\t\treturn \"\", nil, errors.WithStack(err)\n\t}\n\n\tcachedClusterConfigPath := getCachedClusterConfigPath(clusterConfig.ClusterName, clusterConfig.Region)\n\tif err := files.WriteFile(clusterConfigBytes, cachedClusterConfigPath); err != nil {\n\t\treturn \"\", nil, err\n\t}\n\n\tcontainerClusterConfigPath := \"/in/\" + filepath.Base(cachedClusterConfigPath)\n\tcopyToPaths = append(copyToPaths, dockerCopyToPath{\n\t\tinput: &archive.Input{\n\t\t\tFiles: []archive.FileInput{\n\t\t\t\t{\n\t\t\t\t\tSource: cachedClusterConfigPath,\n\t\t\t\t\tDest:   containerClusterConfigPath,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tcontainerPath: \"/\",\n\t})\n\n\tenvs := []string{\n\t\t\"AWS_ACCESS_KEY_ID=\" + *awsClient.AccessKeyID(),\n\t\t\"AWS_SECRET_ACCESS_KEY=\" + *awsClient.SecretAccessKey(),\n\t\t\"CORTEX_TELEMETRY_DISABLE=\" + os.Getenv(\"CORTEX_TELEMETRY_DISABLE\"),\n\t\t\"CORTEX_TELEMETRY_SENTRY_DSN=\" + os.Getenv(\"CORTEX_TELEMETRY_SENTRY_DSN\"),\n\t\t\"CORTEX_TELEMETRY_SEGMENT_WRITE_KEY=\" + os.Getenv(\"CORTEX_TELEMETRY_SEGMENT_WRITE_KEY\"),\n\t\t\"CORTEX_DEV_DEFAULT_IMAGE_REGISTRY=\" + os.Getenv(\"CORTEX_DEV_DEFAULT_IMAGE_REGISTRY\"),\n\t\t\"CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD=\" + os.Getenv(\"CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD\"),\n\t\t\"CORTEX_CLUSTER_CONFIG_FILE=\" + containerClusterConfigPath,\n\t}\n\tenvs = append(envs, extraEnvs...)\n\tcontainerConfig := &container.Config{\n\t\tImage:        clusterConfig.ImageManager,\n\t\tEntrypoint:   []string{\"/bin/bash\", \"-c\"},\n\t\tCmd:          []string{fmt.Sprintf(\"eval $(python /root/cluster_config_env.py %s) && %s\", containerClusterConfigPath, entrypoint)},\n\t\tTty:          true,\n\t\tAttachStdout: true,\n\t\tAttachStderr: true,\n\t\tEnv:          envs,\n\t}\n\n\tif sessionToken := awsClient.SessionToken(); sessionToken != nil {\n\t\tcontainerConfig.Env = append(containerConfig.Env, \"AWS_SESSION_TOKEN=\"+*sessionToken)\n\t}\n\n\toutput, exitCode, err := runManager(containerConfig, false, copyToPaths, copyFromPaths)\n\tif err != nil {\n\t\treturn \"\", nil, err\n\t}\n\n\treturn output, exitCode, nil\n}\n\nfunc runManagerAccessCommand(entrypoint string, accessConfig clusterconfig.AccessConfig, awsClient *aws.Client, copyToPaths []dockerCopyToPath, copyFromPaths []dockerCopyFromPath) (string, *int, error) {\n\tcontainerConfig := &container.Config{\n\t\tImage:        accessConfig.ImageManager,\n\t\tEntrypoint:   []string{\"/bin/bash\", \"-c\"},\n\t\tCmd:          []string{entrypoint},\n\t\tTty:          true,\n\t\tAttachStdout: true,\n\t\tAttachStderr: true,\n\t\tEnv: []string{\n\t\t\t\"AWS_ACCESS_KEY_ID=\" + *awsClient.AccessKeyID(),\n\t\t\t\"AWS_SECRET_ACCESS_KEY=\" + *awsClient.SecretAccessKey(),\n\t\t\t\"CORTEX_CLUSTER_NAME=\" + accessConfig.ClusterName,\n\t\t\t\"CORTEX_REGION=\" + accessConfig.Region,\n\t\t\t\"CORTEX_TELEMETRY_DISABLE=\" + os.Getenv(\"CORTEX_TELEMETRY_DISABLE\"),\n\t\t\t\"CORTEX_TELEMETRY_SENTRY_DSN=\" + os.Getenv(\"CORTEX_TELEMETRY_SENTRY_DSN\"),\n\t\t\t\"CORTEX_TELEMETRY_SEGMENT_WRITE_KEY=\" + os.Getenv(\"CORTEX_TELEMETRY_SEGMENT_WRITE_KEY\"),\n\t\t},\n\t}\n\n\tif sessionToken := awsClient.SessionToken(); sessionToken != nil {\n\t\tcontainerConfig.Env = append(containerConfig.Env, \"AWS_SESSION_TOKEN=\"+*sessionToken)\n\t}\n\n\toutput, exitCode, err := runManager(containerConfig, true, copyToPaths, copyFromPaths)\n\tif err != nil {\n\t\treturn \"\", nil, err\n\t}\n\n\treturn output, exitCode, nil\n}\n"
  },
  {
    "path": "cli/cmd/lib_realtime_apis.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nfunc realtimeAPITable(realtimeAPI schema.APIResponse, env cliconfig.Environment) (string, error) {\n\tvar out string\n\n\tt := realtimeAPIsTable([]schema.APIResponse{realtimeAPI}, []string{env.Name})\n\tout += t.MustFormat()\n\n\tif realtimeAPI.DashboardURL != nil && *realtimeAPI.DashboardURL != \"\" {\n\t\tout += \"\\n\" + console.Bold(\"metrics dashboard: \") + *realtimeAPI.DashboardURL + \"\\n\"\n\t}\n\n\tif realtimeAPI.Endpoint != nil {\n\t\tout += \"\\n\" + console.Bold(\"endpoint: \") + *realtimeAPI.Endpoint + \"\\n\"\n\t}\n\n\tout += \"\\n\" + apiHistoryTable(realtimeAPI.APIVersions)\n\n\tif !_flagVerbose {\n\t\treturn out, nil\n\t}\n\n\tout += titleStr(\"configuration\") + strings.TrimSpace(realtimeAPI.Spec.UserStr())\n\n\treturn out, nil\n}\n\nfunc realtimeDescribeAPITable(realtimeAPI schema.APIResponse, env cliconfig.Environment) (string, error) {\n\tif realtimeAPI.Metadata == nil {\n\t\treturn \"\", errors.ErrorUnexpected(\"missing metadata from operator response\")\n\t}\n\n\tif realtimeAPI.Status == nil {\n\t\treturn \"\", errors.ErrorUnexpected(fmt.Sprintf(\"missing status for %s api\", realtimeAPI.Metadata.Name))\n\t}\n\n\tt := realtimeAPIsTable([]schema.APIResponse{realtimeAPI}, []string{env.Name})\n\tout := t.MustFormat()\n\n\tif realtimeAPI.DashboardURL != nil && *realtimeAPI.DashboardURL != \"\" {\n\t\tout += \"\\n\" + console.Bold(\"metrics dashboard: \") + *realtimeAPI.DashboardURL + \"\\n\"\n\t}\n\n\tif realtimeAPI.Endpoint != nil {\n\t\tout += \"\\n\" + console.Bold(\"endpoint: \") + *realtimeAPI.Endpoint + \"\\n\"\n\t}\n\n\tt = replicaCountTable(realtimeAPI.Status.ReplicaCounts)\n\tout += \"\\n\" + t.MustFormat()\n\n\treturn out, nil\n}\n\nfunc realtimeAPIsTable(realtimeAPIs []schema.APIResponse, envNames []string) table.Table {\n\trows := make([][]interface{}, 0, len(realtimeAPIs))\n\n\tfor i, realtimeAPI := range realtimeAPIs {\n\t\tif realtimeAPI.Metadata == nil || realtimeAPI.Status == nil {\n\t\t\tcontinue\n\t\t}\n\t\tlastUpdated := time.Unix(realtimeAPI.Metadata.LastUpdated, 0)\n\t\trows = append(rows, []interface{}{\n\t\t\tenvNames[i],\n\t\t\trealtimeAPI.Metadata.Name,\n\t\t\tfmt.Sprintf(\"%d/%d\", realtimeAPI.Status.Ready, realtimeAPI.Status.Requested),\n\t\t\trealtimeAPI.Status.UpToDate,\n\t\t\tlibtime.SinceStr(&lastUpdated),\n\t\t})\n\t}\n\n\treturn table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: _titleEnvironment},\n\t\t\t{Title: _titleRealtimeAPI},\n\t\t\t{Title: _titleLive},\n\t\t\t{Title: _titleUpToDate},\n\t\t\t{Title: _titleLastUpdated},\n\t\t},\n\t\tRows: rows,\n\t}\n}\n"
  },
  {
    "path": "cli/cmd/lib_task_apis.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/yaml\"\n)\n\nconst (\n\t_titleTaskAPI         = \"task api\"\n\t_titleTaskJobCount    = \"running jobs\"\n\t_titleLatestTaskJobID = \"latest job id\"\n)\n\nfunc taskAPIsTable(taskAPIs []schema.APIResponse, envNames []string) table.Table {\n\trows := make([][]interface{}, 0, len(taskAPIs))\n\n\tfor i, taskAPI := range taskAPIs {\n\t\tif taskAPI.Metadata == nil {\n\t\t\tcontinue\n\t\t}\n\t\tlastAPIUpdated := time.Unix(taskAPI.Metadata.LastUpdated, 0)\n\t\tlatestStartTime := time.Time{}\n\t\tlatestJobID := \"-\"\n\t\trunningJobs := 0\n\n\t\tfor _, job := range taskAPI.TaskJobStatuses {\n\t\t\tif job.StartTime.After(latestStartTime) {\n\t\t\t\tlatestStartTime = job.StartTime\n\t\t\t\tlatestJobID = job.ID + fmt.Sprintf(\" (submitted %s ago)\", libtime.SinceStr(&latestStartTime))\n\t\t\t}\n\n\t\t\tif job.Status.IsInProgress() {\n\t\t\t\trunningJobs++\n\t\t\t}\n\t\t}\n\n\t\trows = append(rows, []interface{}{\n\t\t\tenvNames[i],\n\t\t\ttaskAPI.Metadata.Name,\n\t\t\trunningJobs,\n\t\t\tlatestJobID,\n\t\t\tlibtime.SinceStr(&lastAPIUpdated),\n\t\t})\n\t}\n\n\treturn table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: _titleEnvironment},\n\t\t\t{Title: _titleTaskAPI},\n\t\t\t{Title: _titleTaskJobCount},\n\t\t\t{Title: _titleLatestTaskJobID},\n\t\t\t{Title: _titleLastUpdated},\n\t\t},\n\t\tRows: rows,\n\t}\n}\n\nfunc taskAPITable(taskAPI schema.APIResponse) string {\n\tjobRows := make([][]interface{}, 0, len(taskAPI.TaskJobStatuses))\n\n\tout := \"\"\n\tif len(taskAPI.TaskJobStatuses) == 0 {\n\t\tout = console.Bold(\"no submitted task jobs\\n\")\n\t} else {\n\t\tfor _, job := range taskAPI.TaskJobStatuses {\n\t\t\tjobEndTime := time.Now()\n\t\t\tif job.EndTime != nil {\n\t\t\t\tjobEndTime = *job.EndTime\n\t\t\t}\n\n\t\t\tduration := jobEndTime.Sub(job.StartTime).Truncate(time.Second).String()\n\n\t\t\tjobRows = append(jobRows, []interface{}{\n\t\t\t\tjob.ID,\n\t\t\t\tjob.Status.Message(),\n\t\t\t\tjob.StartTime.Format(_timeFormat),\n\t\t\t\tduration,\n\t\t\t})\n\t\t}\n\n\t\tt := table.Table{\n\t\t\tHeaders: []table.Header{\n\t\t\t\t{Title: \"task job id\"},\n\t\t\t\t{Title: \"status\"},\n\t\t\t\t{Title: \"start time\"},\n\t\t\t\t{Title: \"duration\"},\n\t\t\t},\n\t\t\tRows: jobRows,\n\t\t}\n\n\t\tout += t.MustFormat()\n\t}\n\n\tif taskAPI.DashboardURL != nil && *taskAPI.DashboardURL != \"\" {\n\t\tout += \"\\n\" + console.Bold(\"metrics dashboard: \") + *taskAPI.DashboardURL + \"\\n\"\n\t}\n\n\tif taskAPI.Endpoint != nil {\n\t\tout += \"\\n\" + console.Bold(\"endpoint: \") + *taskAPI.Endpoint + \"\\n\"\n\t}\n\n\tout += \"\\n\" + apiHistoryTable(taskAPI.APIVersions)\n\n\tif !_flagVerbose {\n\t\treturn out\n\t}\n\n\tout += titleStr(\"task api configuration\") + taskAPI.Spec.UserStr()\n\n\treturn out\n}\n\nfunc getTaskJob(env cliconfig.Environment, apiName string, jobID string) (string, error) {\n\tresp, err := cluster.GetTaskJob(MustGetOperatorConfig(env.Name), apiName, jobID)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tvar bytes []byte\n\tif _flagOutput == flags.JSONOutputType {\n\t\tbytes, err = libjson.Marshal(resp)\n\t} else if _flagOutput == flags.YAMLOutputType {\n\t\tbytes, err = yaml.Marshal(resp)\n\t}\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif _flagOutput == flags.JSONOutputType || _flagOutput == flags.YAMLOutputType {\n\t\treturn string(bytes), nil\n\t}\n\n\tjob := resp.JobStatus\n\n\tout := \"\"\n\n\tjobIntroTable := table.KeyValuePairs{}\n\tjobIntroTable.Add(\"job id\", job.ID)\n\tjobIntroTable.Add(\"status\", job.Status.Message())\n\tout += jobIntroTable.String(&table.KeyValuePairOpts{BoldKeys: pointer.Bool(true)})\n\n\tjobTimingTable := table.KeyValuePairs{}\n\tjobTimingTable.Add(\"start time\", job.StartTime.Format(_timeFormat))\n\n\tjobEndTime := time.Now()\n\tif job.EndTime != nil {\n\t\tjobTimingTable.Add(\"end time\", job.EndTime.Format(_timeFormat))\n\t\tjobEndTime = *job.EndTime\n\t} else {\n\t\tjobTimingTable.Add(\"end time\", \"-\")\n\t}\n\tduration := jobEndTime.Sub(job.StartTime).Truncate(time.Second).String()\n\tjobTimingTable.Add(\"duration\", duration)\n\n\tout += \"\\n\" + jobTimingTable.String(&table.KeyValuePairOpts{BoldKeys: pointer.Bool(true)})\n\n\tif job.Status.IsCompleted() {\n\t\tout += \"\\n\" + \"worker stats are not available because this job is not currently running\\n\"\n\t} else {\n\t\tout += titleStr(\"worker stats\")\n\t\tif job.WorkerCounts != nil {\n\t\t\tt := table.Table{\n\t\t\t\tHeaders: []table.Header{\n\t\t\t\t\t{Title: \"Requested\"},\n\t\t\t\t\t{Title: \"Pending\"},\n\t\t\t\t\t{Title: \"Creating\"},\n\t\t\t\t\t{Title: \"Ready\"},\n\t\t\t\t\t{Title: \"NotReady\"},\n\t\t\t\t\t{Title: \"ErrImagePull\", Hidden: job.WorkerCounts.ErrImagePull == 0},\n\t\t\t\t\t{Title: \"Terminating\", Hidden: job.WorkerCounts.Terminating == 0},\n\t\t\t\t\t{Title: \"Failed\", Hidden: job.WorkerCounts.Failed == 0},\n\t\t\t\t\t{Title: \"Killed\", Hidden: job.WorkerCounts.Killed == 0},\n\t\t\t\t\t{Title: \"KilledOOM\", Hidden: job.WorkerCounts.KilledOOM == 0},\n\t\t\t\t\t{Title: \"Stalled\", Hidden: job.WorkerCounts.Stalled == 0},\n\t\t\t\t\t{Title: \"Unknown\", Hidden: job.WorkerCounts.Unknown == 0},\n\t\t\t\t\t{Title: \"Succeeded\"},\n\t\t\t\t},\n\t\t\t\tRows: [][]interface{}{\n\t\t\t\t\t{\n\t\t\t\t\t\tjob.Workers,\n\t\t\t\t\t\tjob.WorkerCounts.Pending,\n\t\t\t\t\t\tjob.WorkerCounts.Creating,\n\t\t\t\t\t\tjob.WorkerCounts.Ready,\n\t\t\t\t\t\tjob.WorkerCounts.NotReady,\n\t\t\t\t\t\tjob.WorkerCounts.ErrImagePull,\n\t\t\t\t\t\tjob.WorkerCounts.Terminating,\n\t\t\t\t\t\tjob.WorkerCounts.Failed,\n\t\t\t\t\t\tjob.WorkerCounts.Killed,\n\t\t\t\t\t\tjob.WorkerCounts.KilledOOM,\n\t\t\t\t\t\tjob.WorkerCounts.Stalled,\n\t\t\t\t\t\tjob.WorkerCounts.Unknown,\n\t\t\t\t\t\tjob.WorkerCounts.Succeeded,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t}\n\t\t\tout += t.MustFormat(&table.Opts{BoldHeader: pointer.Bool(false)})\n\t\t} else {\n\t\t\tout += \"unable to get worker stats\\n\"\n\t\t}\n\t}\n\n\tout += \"\\n\" + console.Bold(\"job endpoint: \") + resp.Endpoint + \"\\n\"\n\n\tjobSpecStr, err := libjson.Pretty(job.TaskJob)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tout += titleStr(\"job configuration\") + jobSpecStr\n\n\treturn out, nil\n}\n"
  },
  {
    "path": "cli/cmd/lib_traffic_splitters.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/cliconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nconst (\n\t_titleTrafficSplitter   = \"traffic splitter\"\n\t_trafficSplitterWeights = \"weights\"\n\t_titleAPIs              = \"apis\"\n)\n\nfunc trafficSplitterTable(trafficSplitter schema.APIResponse, env cliconfig.Environment) (string, error) {\n\tvar out string\n\n\tlastUpdated := time.Unix(trafficSplitter.Spec.LastUpdated, 0)\n\n\tt, err := trafficSplitTable(trafficSplitter, env)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tout += t.MustFormat()\n\n\tout += \"\\n\" + console.Bold(\"last updated: \") + libtime.SinceStr(&lastUpdated)\n\n\tif trafficSplitter.Endpoint != nil {\n\t\tout += \"\\n\" + console.Bold(\"endpoint: \") + *trafficSplitter.Endpoint + \"\\n\"\n\t}\n\n\tout += \"\\n\" + apiHistoryTable(trafficSplitter.APIVersions)\n\n\tif !_flagVerbose {\n\t\treturn out, nil\n\t}\n\n\tout += titleStr(\"configuration\") + strings.TrimSpace(trafficSplitter.Spec.UserStr())\n\n\treturn out, nil\n}\n\nfunc trafficSplitTable(trafficSplitter schema.APIResponse, env cliconfig.Environment) (table.Table, error) {\n\trows := make([][]interface{}, 0, len(trafficSplitter.Spec.APIs))\n\n\tfor _, api := range trafficSplitter.Spec.APIs {\n\t\tapisRes, err := cluster.GetAPI(MustGetOperatorConfig(env.Name), api.Name)\n\t\tif err != nil {\n\t\t\treturn table.Table{}, err\n\t\t}\n\n\t\tapiRes := apisRes[0]\n\t\tif apiRes.Metadata == nil || apiRes.Status == nil {\n\t\t\tcontinue\n\t\t}\n\t\tlastUpdated := time.Unix(apiRes.Metadata.LastUpdated, 0)\n\n\t\tapiName := apiRes.Metadata.Name\n\t\tif api.Shadow {\n\t\t\tapiName += \" (shadow)\"\n\t\t}\n\t\trows = append(rows, []interface{}{\n\t\t\tenv.Name,\n\t\t\tapiName,\n\t\t\tapi.Weight,\n\t\t\tfmt.Sprintf(\"%d/%d\", apiRes.Status.Ready, apiRes.Status.Requested),\n\t\t\tapiRes.Status.UpToDate,\n\t\t\tlibtime.SinceStr(&lastUpdated),\n\t\t})\n\t}\n\n\treturn table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: _titleEnvironment},\n\t\t\t{Title: _titleAPIs},\n\t\t\t{Title: _trafficSplitterWeights},\n\t\t\t{Title: _titleLive},\n\t\t\t{Title: _titleUpToDate},\n\t\t\t{Title: _titleLastUpdated},\n\t\t},\n\t\tRows: rows,\n\t}, nil\n}\n\nfunc trafficSplitterListTable(trafficSplitter []schema.APIResponse, envNames []string) table.Table {\n\trows := make([][]interface{}, 0, len(trafficSplitter))\n\tfor i, splitAPI := range trafficSplitter {\n\t\tif splitAPI.Metadata == nil || splitAPI.NumTrafficSplitterTargets == nil {\n\t\t\tcontinue\n\t\t}\n\t\tlastUpdated := time.Unix(splitAPI.Metadata.LastUpdated, 0)\n\t\trows = append(rows, []interface{}{\n\t\t\tenvNames[i],\n\t\t\tsplitAPI.Metadata.Name,\n\t\t\ts.Int32(*splitAPI.NumTrafficSplitterTargets),\n\t\t\tlibtime.SinceStr(&lastUpdated),\n\t\t})\n\t}\n\n\treturn table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: _titleEnvironment},\n\t\t\t{Title: _titleTrafficSplitter},\n\t\t\t{Title: _titleAPIs},\n\t\t\t{Title: _titleLastUpdated},\n\t\t},\n\t\tRows: rows,\n\t}\n}\n"
  },
  {
    "path": "cli/cmd/lib_watch.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\tlibmath \"github.com/cortexlabs/cortex/pkg/lib/math\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n)\n\nfunc getTerminalWidth() int {\n\tcmd := exec.Command(\"stty\", \"size\")\n\tcmd.Stdin = os.Stdin\n\tout, err := cmd.Output()\n\tif err != nil {\n\t\treturn 0\n\t}\n\tdimensions := strings.Split(strings.TrimSpace(string(out)), \" \")\n\tif len(dimensions) != 2 {\n\t\treturn 0\n\t}\n\twidthStr := dimensions[1]\n\twidth, ok := s.ParseInt(widthStr)\n\tif !ok {\n\t\treturn 0\n\t}\n\treturn width\n}\n\nfunc watchHeader() string {\n\ttimeStr := libtime.LocalHourNow()\n\twidth := getTerminalWidth()\n\tnumExtraChars := 4\n\tpadding := strings.Repeat(\" \", libmath.MaxInt(width-len(_cmdStr)-len(timeStr)-numExtraChars, 0))\n\treturn fmt.Sprintf(\"$ %s  %s%s\", _cmdStr, padding, libtime.LocalHourNow())\n}\n\nfunc rerun(watchFlag bool, f func() (string, error)) {\n\tif watchFlag {\n\t\tprint(\"\\033[H\\033[2J\") // clear the screen\n\n\t\tvar prevStrSlice []string\n\n\t\tfor true {\n\t\t\tnextStr, err := f()\n\t\t\tif err != nil {\n\t\t\t\tfmt.Println()\n\t\t\t\texit.Error(err)\n\t\t\t}\n\n\t\t\tnextStr = watchHeader() + \"\\n\\n\" + s.EnsureSingleTrailingNewLine(nextStr)\n\t\t\tnextStrSlice := strings.Split(nextStr, \"\\n\")\n\n\t\t\tterminalWidth := getTerminalWidth()\n\t\t\tif terminalWidth <= 0 {\n\t\t\t\texit.Error(ErrorNoTerminalWidth())\n\t\t\t}\n\n\t\t\tnextNumLines := 0\n\t\t\tfor _, strLine := range nextStrSlice {\n\t\t\t\tnextNumLines += (len(strLine)-1)/terminalWidth + 1\n\t\t\t}\n\t\t\tprevNumLines := 0\n\t\t\tfor _, strLine := range prevStrSlice {\n\t\t\t\tprevNumLines += (len(strLine)-1)/terminalWidth + 1\n\t\t\t}\n\n\t\t\tfor i := prevNumLines; i > nextNumLines; i-- {\n\t\t\t\tfmt.Printf(\"\\033[%dA\\033[2K\", 1) // move the cursor up and clear the line\n\t\t\t}\n\n\t\t\tfor i := 0; i < prevNumLines; i++ {\n\t\t\t\tfmt.Printf(\"\\033[%dA\", 1) // move the cursor up\n\t\t\t}\n\n\t\t\tfor _, strLine := range nextStrSlice {\n\t\t\t\tfmt.Printf(\"\\033[2K%s\\n\", strLine) // clear the line and print the new line\n\t\t\t}\n\n\t\t\tprevStrSlice = nextStrSlice\n\n\t\t\ttime.Sleep(time.Second * 2)\n\t\t}\n\t} else {\n\t\tstr, err := f()\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tfmt.Print(s.EnsureSingleTrailingNewLine(str))\n\t}\n}\n"
  },
  {
    "path": "cli/cmd/logs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar (\n\t_flagLogsEnv            string\n\t_flagLogsDisallowPrompt bool\n\t_flagRandomPod          bool\n\t_logsOutput             = `Navigate to the link below and click \"Run Query\":\n\n%s\n\nNOTE: there may be 1-2 minutes of delay for the logs to show up in the results of CloudWatch Insight queries\n`\n)\n\nfunc logsInit() {\n\t_logsCmd.Flags().SortFlags = false\n\t_logsCmd.Flags().StringVarP(&_flagLogsEnv, \"env\", \"e\", \"\", \"environment to use\")\n\t_logsCmd.Flags().BoolVarP(&_flagLogsDisallowPrompt, \"yes\", \"y\", false, \"skip prompts\")\n\t_logsCmd.Flags().BoolVarP(&_flagRandomPod, \"random-pod\", \"\", false, \"stream logs from a random pod\")\n}\n\nvar _logsCmd = &cobra.Command{\n\tUse:   \"logs API_NAME [JOB_ID]\",\n\tShort: \"get the logs for a workload\",\n\tArgs:  cobra.RangeArgs(1, 2),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tenvName, err := getEnvFromFlag(_flagLogsEnv)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.logs\")\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.logs\")\n\t\t\texit.Error(err)\n\t\t}\n\t\ttelemetry.Event(\"cli.logs\", map[string]interface{}{\"env_name\": env.Name, \"random_pod\": _flagRandomPod})\n\n\t\terr = printEnvIfNotSpecified(env.Name, cmd)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\toperatorConfig := MustGetOperatorConfig(env.Name)\n\t\tapiName := args[0]\n\n\t\tif len(args) == 1 {\n\t\t\tif _flagRandomPod {\n\t\t\t\terr := cluster.StreamLogs(operatorConfig, apiName)\n\t\t\t\tif err != nil {\n\t\t\t\t\texit.Error(err)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlogResponse, err := cluster.GetLogs(operatorConfig, apiName)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t\tfmt.Printf(_logsOutput, logResponse.LogURL)\n\t\t\treturn\n\t\t}\n\n\t\tjobID := args[1]\n\t\tif _flagRandomPod {\n\t\t\terr := cluster.StreamJobLogs(operatorConfig, apiName, jobID)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\n\t\tlogResponse, err := cluster.GetJobLogs(operatorConfig, apiName, jobID)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\t\tfmt.Printf(_logsOutput, logResponse.LogURL)\n\t},\n}\n"
  },
  {
    "path": "cli/cmd/refresh.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/print\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar (\n\t_flagRefreshEnv   string\n\t_flagRefreshForce bool\n)\n\nfunc refreshInit() {\n\t_refreshCmd.Flags().SortFlags = false\n\t_refreshCmd.Flags().StringVarP(&_flagRefreshEnv, \"env\", \"e\", \"\", \"environment to use\")\n\t_refreshCmd.Flags().BoolVarP(&_flagRefreshForce, \"force\", \"f\", false, \"override the in-progress api update\")\n\t_refreshCmd.Flags().VarP(&_flagOutput, \"output\", \"o\", fmt.Sprintf(\"output format: one of %s\", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), \"|\")))\n}\n\nvar _refreshCmd = &cobra.Command{\n\tUse:   \"refresh API_NAME\",\n\tShort: \"restart all replicas for an api (without downtime)\",\n\tArgs:  cobra.ExactArgs(1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tenvName, err := getEnvFromFlag(_flagRefreshEnv)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.refresh\")\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.refresh\")\n\t\t\texit.Error(err)\n\t\t}\n\t\ttelemetry.Event(\"cli.refresh\", map[string]interface{}{\"env_name\": env.Name})\n\n\t\terr = printEnvIfNotSpecified(env.Name, cmd)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\trefreshResponse, err := cluster.Refresh(MustGetOperatorConfig(env.Name), args[0], _flagRefreshForce)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tif _flagOutput == flags.JSONOutputType {\n\t\t\tbytes, err := libjson.Marshal(refreshResponse)\n\t\t\tif err != nil {\n\t\t\t\texit.Error(err)\n\t\t\t}\n\t\t\tfmt.Print(string(bytes))\n\t\t\treturn\n\t\t}\n\n\t\tprint.BoldFirstLine(refreshResponse.Message)\n\t},\n}\n"
  },
  {
    "path": "cli/cmd/root.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/cli/types/flags\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\thomedir \"github.com/mitchellh/go-homedir\"\n\t\"github.com/spf13/cobra\"\n\t\"github.com/spf13/pflag\"\n)\n\nvar (\n\t_cmdStr string\n\n\t_configFileExts = []string{\"yaml\", \"yml\"}\n\t_flagVerbose    bool\n\t_flagOutput     = flags.PrettyOutputType\n\n\t_credentialsCacheDir string\n\t_localDir            string\n\t_cliConfigPath       string\n\t_clientIDPath        string\n\t_emailPath           string\n\t_debugPath           string\n\t_cwd                 string\n\t_homeDir             string\n)\n\nfunc init() {\n\tcwd, err := os.Getwd()\n\tif err != nil {\n\t\terr := errors.Wrap(err, \"unable to determine current working directory\")\n\t\texit.Error(err)\n\t}\n\t_cwd = s.EnsureSuffix(cwd, \"/\")\n\n\thomeDir, err := homedir.Dir()\n\tif err != nil {\n\t\terr := errors.Wrap(err, \"unable to determine home directory\")\n\t\texit.Error(err)\n\t}\n\t_homeDir = s.EnsureSuffix(homeDir, \"/\")\n\n\t_localDir = os.Getenv(\"CORTEX_CLI_CONFIG_DIR\")\n\tif _localDir != \"\" {\n\t\t_localDir = files.UserRelToAbsPath(_localDir)\n\t} else {\n\t\t_localDir = filepath.Join(homeDir, \".cortex\")\n\t}\n\n\terr = os.MkdirAll(_localDir, os.ModePerm)\n\tif err != nil {\n\t\terr := errors.Wrap(err, \"unable to write to home directory\", _localDir)\n\t\texit.Error(err)\n\t}\n\n\t// ~/.cortex/credentials/\n\t_credentialsCacheDir = filepath.Join(_localDir, \"credentials\")\n\terr = os.MkdirAll(_credentialsCacheDir, os.ModePerm)\n\tif err != nil {\n\t\terr := errors.Wrap(err, \"unable to write to home directory\", _localDir)\n\t\texit.Error(err)\n\t}\n\n\t_cliConfigPath = filepath.Join(_localDir, \"cli.yaml\")\n\t_clientIDPath = filepath.Join(_localDir, \"client-id.txt\")\n\t_emailPath = filepath.Join(_localDir, \"email.txt\")\n\t_debugPath = filepath.Join(_localDir, \"cortex-debug.tgz\")\n\n\tcobra.EnablePrefixMatching = true\n\n\t_cmdStr = \"cortex\"\n\tfor _, arg := range os.Args[1:] {\n\t\tif arg == \"-w\" || arg == \"--watch\" {\n\t\t\tcontinue\n\t\t}\n\t\t_cmdStr += \" \" + arg\n\t}\n\n\tenableTelemetry, err := readTelemetryConfig()\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\tif enableTelemetry {\n\t\tinitTelemetry()\n\t}\n\n\tclusterInit()\n\tcompletionInit()\n\tdeleteInit()\n\tdescribeInit()\n\tdeployInit()\n\tenvInit()\n\tgetInit()\n\tlogsInit()\n\trefreshInit()\n\tversionInit()\n}\n\nfunc initTelemetry() {\n\tcID := clientID()\n\n\tinvoker := os.Getenv(\"CORTEX_CLI_INVOKER\")\n\tif invoker == \"\" {\n\t\tinvoker = \"direct\"\n\t}\n\n\ttelemetry.Init(telemetry.Config{\n\t\tEnabled: true,\n\t\tUserID:  cID,\n\t\tProperties: map[string]string{\n\t\t\t\"client_id\": cID,\n\t\t\t\"invoker\":   invoker,\n\t\t},\n\t\tEnvironment: \"cli\",\n\t\tLogErrors:   false,\n\t\tBackoffMode: telemetry.NoBackoff,\n\t})\n}\n\nvar _rootCmd = &cobra.Command{\n\tUse:     \"cortex\",\n\tAliases: []string{\"cx\"},\n\tShort:   \"cost-effective serverless computing\",\n}\n\nfunc Execute() {\n\tdefer exit.RecoverAndExit()\n\n\tcobra.EnableCommandSorting = false\n\n\t_rootCmd.AddCommand(_deployCmd)\n\t_rootCmd.AddCommand(_getCmd)\n\t_rootCmd.AddCommand(_describeCmd)\n\t_rootCmd.AddCommand(_logsCmd)\n\t_rootCmd.AddCommand(_refreshCmd)\n\t_rootCmd.AddCommand(_deleteCmd)\n\n\t_rootCmd.AddCommand(_clusterCmd)\n\n\t_rootCmd.AddCommand(_envCmd)\n\t_rootCmd.AddCommand(_versionCmd)\n\t_rootCmd.AddCommand(_completionCmd)\n\n\tupdateRootUsage()\n\n\t_rootCmd.Execute()\n\n\texit.Ok()\n}\n\nfunc updateRootUsage() {\n\tdefaultUsageFunc := _rootCmd.UsageFunc()\n\tusage := _rootCmd.UsageString()\n\n\t_rootCmd.SetUsageFunc(func(cmd *cobra.Command) error {\n\t\tif cmd != _rootCmd {\n\t\t\treturn defaultUsageFunc(cmd)\n\t\t}\n\n\t\tusage = strings.Replace(usage, \"Usage:\\n  cortex [command]\\n\\nAliases:\\n  cortex, cx\\n\\n\", \"\", 1)\n\t\tusage = strings.Replace(usage, \"Available Commands:\", \"api commands:\", 1)\n\t\tusage = strings.Replace(usage, \"\\n  cluster\", \"\\n\\ncluster commands:\\n  cluster\", 1)\n\t\tusage = strings.Replace(usage, \"\\n  env \", \"\\n\\nother commands:\\n  env \", 1)\n\t\tusage = strings.Replace(usage, \"\\n\\nUse \\\"cortex [command] --help\\\" for more information about a command.\", \"\", 1)\n\n\t\tcmd.Print(usage)\n\n\t\treturn nil\n\t})\n}\n\nfunc addVerboseFlag(cmd *cobra.Command) {\n\tcmd.Flags().BoolVarP(&_flagVerbose, \"verbose\", \"v\", false, \"show additional information (only applies to pretty output format)\")\n}\n\nfunc wasFlagProvided(cmd *cobra.Command, flagName string) bool {\n\tflagWasProvided := false\n\tcmd.Flags().VisitAll(func(flag *pflag.Flag) {\n\t\tif flag.Name == flagName && flag.Changed && flag.Value.String() != \"\" {\n\t\t\tflagWasProvided = true\n\t\t}\n\t})\n\n\treturn flagWasProvided\n}\n\nfunc printEnvIfNotSpecified(envName string, cmd *cobra.Command) error {\n\tout, err := envStringIfNotSpecified(envName, cmd)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif out != \"\" {\n\t\tfmt.Print(out)\n\t}\n\n\treturn nil\n}\n\nfunc envStringIfNotSpecified(envName string, cmd *cobra.Command) (string, error) {\n\tenvNames, err := listConfiguredEnvNames()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif _flagOutput == flags.PrettyOutputType && !wasFlagProvided(cmd, \"env\") && len(envNames) > 1 {\n\t\treturn fmt.Sprintf(\"using %s environment\\n\\n\", envName), nil\n\t}\n\n\treturn \"\", nil\n}\n"
  },
  {
    "path": "cli/cmd/version.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cmd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/cli/cluster\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/spf13/cobra\"\n)\n\nvar _flagVersionEnv string\n\nfunc versionInit() {\n\t_versionCmd.Flags().SortFlags = false\n\t_versionCmd.Flags().StringVarP(&_flagVersionEnv, \"env\", \"e\", \"\", \"environment to use\")\n}\n\nvar _versionCmd = &cobra.Command{\n\tUse:   \"version\",\n\tShort: \"print the cli and cluster versions\",\n\tArgs:  cobra.NoArgs,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tenvName, err := getEnvFromFlag(_flagVersionEnv)\n\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.version\")\n\t\t\tfmt.Println(\"cli version: \" + consts.CortexVersion)\n\t\t\treturn\n\t\t}\n\n\t\tenv, err := ReadOrConfigureEnv(envName)\n\t\tif err != nil {\n\t\t\ttelemetry.Event(\"cli.version\")\n\t\t\texit.Error(err)\n\t\t}\n\t\ttelemetry.Event(\"cli.version\", map[string]interface{}{\"env_name\": env.Name})\n\n\t\terr = printEnvIfNotSpecified(env.Name, cmd)\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tfmt.Println(\"cli version: \" + consts.CortexVersion)\n\n\t\tinfoResponse, err := cluster.Info(MustGetOperatorConfig(env.Name))\n\t\tif err != nil {\n\t\t\texit.Error(err)\n\t\t}\n\n\t\tfmt.Println(\"cluster version: \" + infoResponse.ClusterConfig.APIVersion)\n\t},\n}\n"
  },
  {
    "path": "cli/lib/routines/routines.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage routines\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n)\n\nfunc RunWithPanicHandler(f func(), exitOnPanic bool) {\n\tgo func() {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\terr := errors.CastRecoverError(r)\n\t\t\t\terrors.PrintStacktrace(err)\n\t\t\t\tif exitOnPanic {\n\t\t\t\t\texit.Error(err)\n\t\t\t\t}\n\t\t\t\ttelemetry.Error(err)\n\t\t\t}\n\t\t}()\n\t\tf()\n\t}()\n}\n"
  },
  {
    "path": "cli/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"github.com/cortexlabs/cortex/cli/cmd\"\n)\n\nfunc main() {\n\tcmd.Execute()\n}\n"
  },
  {
    "path": "cli/types/cliconfig/cli_config.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cliconfig\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\ntype CLIConfig struct {\n\tTelemetry          *bool          `json:\"telemetry,omitempty\" yaml:\"telemetry,omitempty\"`\n\tDefaultEnvironment *string        `json:\"default_environment\" yaml:\"default_environment\"`\n\tEnvironments       []*Environment `json:\"environments\" yaml:\"environments\"`\n}\n\ntype UserFacingCLIConfig struct {\n\tDefaultEnvironment *string        `json:\"default_environment\" yaml:\"default_environment\"`\n\tEnvironments       []*Environment `json:\"environments\" yaml:\"environments\"`\n}\n\nfunc (cliConfig *CLIConfig) Validate() error {\n\tenvNames := strset.New()\n\n\tfor _, env := range cliConfig.Environments {\n\t\tif envNames.Has(env.Name) {\n\t\t\treturn errors.Wrap(ErrorDuplicateEnvironmentNames(env.Name), EnvironmentsKey)\n\t\t}\n\n\t\tenvNames.Add(env.Name)\n\n\t\tif err := env.Validate(); err != nil {\n\t\t\treturn errors.Wrap(err, EnvironmentsKey)\n\t\t}\n\t}\n\n\t// Backwards compatibility: ignore local default env\n\tdefaultEnv := cliConfig.DefaultEnvironment\n\tif defaultEnv != nil && *defaultEnv == \"local\" && !envNames.Has(*defaultEnv) {\n\t\tcliConfig.DefaultEnvironment = nil\n\t}\n\n\treturn nil\n}\n\nfunc (cliConfig *CLIConfig) ConvertToUserFacingCLIConfig() UserFacingCLIConfig {\n\tenvs := cliConfig.Environments\n\tif envs == nil {\n\t\tenvs = []*Environment{}\n\t}\n\treturn UserFacingCLIConfig{\n\t\tDefaultEnvironment: cliConfig.DefaultEnvironment,\n\t\tEnvironments:       envs,\n\t}\n}\n"
  },
  {
    "path": "cli/types/cliconfig/config_key.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cliconfig\n\nconst (\n\tEnvironmentsKey       = \"environments\"\n\tDefaultEnvironmentKey = \"default_environment\"\n\tNameKey               = \"name\"\n\tOperatorEndpointKey   = \"operator_endpoint\"\n)\n"
  },
  {
    "path": "cli/types/cliconfig/environment.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cliconfig\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\tcr \"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n)\n\ntype Environment struct {\n\tName             string `json:\"name\" yaml:\"name\"`\n\tOperatorEndpoint string `json:\"operator_endpoint\" yaml:\"operator_endpoint\"`\n}\n\nfunc (env Environment) String(isDefault bool) string {\n\tvar envStr string\n\n\tif isDefault {\n\t\tenvStr += console.Bold(env.Name + \" (default)\")\n\t} else {\n\t\tenvStr += console.Bold(env.Name)\n\t}\n\n\tenvStr += fmt.Sprintf(\"\\ncortex operator endpoint: %s\\n\", env.OperatorEndpoint)\n\n\treturn envStr\n}\n\nfunc CortexEndpointValidator(val string) (string, error) {\n\turlStr := strings.TrimSpace(val)\n\n\tparsedURL, err := urls.Parse(urlStr)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\t// default https\n\tif parsedURL.Scheme == \"\" {\n\t\tparsedURL.Scheme = \"https\"\n\t}\n\n\treturn parsedURL.String(), nil\n}\n\nfunc (env *Environment) Validate() error {\n\tif env.Name == \"\" {\n\t\treturn errors.Wrap(cr.ErrorMustBeDefined(), NameKey)\n\t}\n\n\tvalidOperatorURL, err := CortexEndpointValidator(env.OperatorEndpoint)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tenv.OperatorEndpoint = validOperatorURL\n\treturn nil\n}\n"
  },
  {
    "path": "cli/types/cliconfig/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cliconfig\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrEnvironmentNotConfigured     = \"cliconfig.environment_not_configured\"\n\tErrEnvironmentAlreadyConfigured = \"cliconfig.environment_already_configured\"\n\tErrDuplicateEnvironmentNames    = \"cliconfig.duplicate_environment_names\"\n)\n\nfunc ErrorEnvironmentNotConfigured(envName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEnvironmentNotConfigured,\n\t\tMessage: fmt.Sprintf(\"there is no environment named %s\", envName),\n\t})\n}\n\nfunc ErrorEnvironmentAlreadyConfigured(envName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEnvironmentAlreadyConfigured,\n\t\tMessage: fmt.Sprintf(\"there is already an environment named %s\", envName),\n\t})\n}\n\nfunc ErrorDuplicateEnvironmentNames(envName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDuplicateEnvironmentNames,\n\t\tMessage: fmt.Sprintf(\"duplicate environment names (%s is defined more than once)\", s.UserStr(envName)),\n\t})\n}\n"
  },
  {
    "path": "cli/types/flags/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage flags\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrInvalidOutputType = \"flags.invalid_output_type\"\n)\n\nfunc ErrorInvalidOutputType(invalidOutputType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidOutputType,\n\t\tMessage: fmt.Sprintf(\"invalid value \\\"%s\\\" specified for -o/--output; valid values are %s\", invalidOutputType, s.StrsAnd(OutputTypeStrings())),\n\t})\n}\n"
  },
  {
    "path": "cli/types/flags/output_type.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage flags\n\ntype OutputType int\n\nconst (\n\tUnknownOutputType OutputType = iota\n\tPrettyOutputType\n\tJSONOutputType\n\tYAMLOutputType\n)\n\nvar _outputTypes = []string{\n\t\"unknown\",\n\t\"pretty\",\n\t\"json\",\n\t\"yaml\",\n}\n\nfunc OutputTypeFromString(s string) OutputType {\n\tfor i := 0; i < len(_outputTypes); i++ {\n\t\tif s == _outputTypes[i] {\n\t\t\treturn OutputType(i)\n\t\t}\n\t}\n\treturn UnknownOutputType\n}\n\nfunc OutputTypeStrings() []string {\n\treturn _outputTypes[1:]\n}\n\nfunc OutputTypeStringsExcluding(outputType OutputType) []string {\n\tvar outputTypes []string\n\tfor _, _outputType := range _outputTypes[1:] {\n\t\tif OutputTypeFromString(_outputType) != outputType {\n\t\t\toutputTypes = append(outputTypes, _outputType)\n\t\t}\n\t}\n\treturn outputTypes\n}\n\nfunc (t OutputType) String() string {\n\treturn _outputTypes[t]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (t OutputType) MarshalText() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (t *OutputType) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_outputTypes); i++ {\n\t\tif enum == _outputTypes[i] {\n\t\t\t*t = OutputType(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*t = UnknownOutputType\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (t *OutputType) UnmarshalBinary(data []byte) error {\n\treturn t.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (t OutputType) MarshalBinary() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\nfunc (t *OutputType) Set(value string) error {\n\toutput := OutputTypeFromString(value)\n\tif output == UnknownOutputType {\n\t\treturn ErrorInvalidOutputType(value)\n\t}\n\t*t = output\n\treturn nil\n}\n\nfunc (t OutputType) Type() string {\n\treturn \"string\"\n}\n"
  },
  {
    "path": "cmd/activator/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/activator\"\n\t\"github.com/cortexlabs/cortex/pkg/autoscaler\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"go.uber.org/zap\"\n\tistioinformers \"istio.io/client-go/pkg/informers/externalversions\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\tkinformers \"k8s.io/client-go/informers\"\n)\n\nfunc main() {\n\tvar (\n\t\tport          int\n\t\tadminPort     int\n\t\tinCluster     bool\n\t\tautoscalerURL string\n\t\tnamespace     string\n\t)\n\n\tflag.IntVar(&port, \"port\", 8000, \"port where the activator server will be exposed\")\n\tflag.IntVar(&adminPort, \"admin-port\", 15000, \"port where the admin server will be exposed\")\n\tflag.BoolVar(&inCluster, \"in-cluster\", false, \"use when autoscaler runs in-cluster\")\n\tflag.StringVar(&autoscalerURL, \"autoscaler-url\", \"\", \"the URL for the cortex autoscaler endpoint\")\n\tflag.StringVar(&namespace, \"namespace\", os.Getenv(\"CORTEX_NAMESPACE\"),\n\t\t\"kubernetes namespace where the cortex APIs are deployed \"+\n\t\t\t\"(can be set through the CORTEX_NAMESPACE env variable)\",\n\t)\n\tflag.Parse()\n\n\tlog := logging.GetLogger()\n\tdefer func() {\n\t\t_ = log.Sync()\n\t}()\n\n\tswitch {\n\tcase autoscalerURL == \"\":\n\t\tlog.Fatal(\"--autoscaler-url is a required option\")\n\tcase namespace == \"\":\n\t\tlog.Fatal(\"--namespace is a required option\")\n\t}\n\n\tawsClient, err := aws.New()\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\t_, userID, err := awsClient.CheckCredentials()\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\ttelemetryEnabled := strings.ToLower(os.Getenv(\"CORTEX_TELEMETRY_DISABLE\")) != \"true\"\n\n\terr = telemetry.Init(telemetry.Config{\n\t\tEnabled: telemetryEnabled,\n\t\tUserID:  userID,\n\t\tProperties: map[string]string{\n\t\t\t\"kind\":       userconfig.RealtimeAPIKind.String(),\n\t\t\t\"image_type\": \"activator\",\n\t\t},\n\t\tEnvironment: \"operator\",\n\t\tLogErrors:   true,\n\t\tBackoffMode: telemetry.BackoffDuplicateMessages,\n\t})\n\tif err != nil {\n\t\tlog.Fatalw(\"failed to initialize telemetry\", zap.Error(err))\n\t}\n\tdefer telemetry.Close()\n\n\tk8sClient, err := k8s.New(namespace, inCluster, nil, runtime.NewScheme())\n\tif err != nil {\n\t\texit(log, err, \"failed to initialize kubernetes client\")\n\t}\n\n\tistioClient := k8sClient.IstioClientSet()\n\tkubeClient := k8sClient.ClientSet()\n\tautoscalerClient := autoscaler.NewClient(autoscalerURL)\n\n\tprometheusStatsReporter := activator.NewPrometheusStatsReporter()\n\n\tistioInformerFactory := istioinformers.NewSharedInformerFactoryWithOptions(\n\t\tistioClient, 10*time.Second, // TODO: check how much makes sense\n\t\tistioinformers.WithNamespace(namespace),\n\t\tistioinformers.WithTweakListOptions(informerFilter),\n\t)\n\tvirtualServiceInformer := istioInformerFactory.Networking().V1beta1().VirtualServices().Informer()\n\tvirtualServiceClient := istioClient.NetworkingV1beta1().VirtualServices(namespace)\n\n\tkubeInformerFactory := kinformers.NewSharedInformerFactoryWithOptions(\n\t\tkubeClient, 2*time.Second, // TODO: check how much makes sense\n\t\tkinformers.WithNamespace(namespace),\n\t\tkinformers.WithTweakListOptions(informerFilter),\n\t)\n\tdeploymentInformer := kubeInformerFactory.Apps().V1().Deployments().Informer()\n\n\tact := activator.New(\n\t\tvirtualServiceClient,\n\t\tdeploymentInformer,\n\t\tvirtualServiceInformer,\n\t\tautoscalerClient,\n\t\tprometheusStatsReporter,\n\t\tlog,\n\t)\n\n\thandler := activator.NewHandler(act, log)\n\n\tadminHandler := http.NewServeMux()\n\tadminHandler.Handle(\"/metrics\", prometheusStatsReporter)\n\n\tservers := map[string]*http.Server{\n\t\t\"activator\": {\n\t\t\tAddr:    \":\" + strconv.Itoa(port),\n\t\t\tHandler: handler,\n\t\t},\n\t\t\"admin\": {\n\t\t\tAddr:    \":\" + strconv.Itoa(adminPort),\n\t\t\tHandler: adminHandler,\n\t\t},\n\t}\n\n\tstopCh := make(chan struct{})\n\tgo virtualServiceInformer.Run(stopCh)\n\tgo deploymentInformer.Run(stopCh)\n\tdefer func() {\n\t\tstopCh <- struct{}{}\n\t}()\n\n\terrCh := make(chan error)\n\tfor name, server := range servers {\n\t\tgo func(name string, server *http.Server) {\n\t\t\tlog.Infof(\"Starting %s server on %s\", name, server.Addr)\n\t\t\terrCh <- server.ListenAndServe()\n\t\t}(name, server)\n\t}\n\n\tsigint := make(chan os.Signal, 1)\n\tsignal.Notify(sigint, os.Interrupt, syscall.SIGTERM)\n\n\tselect {\n\tcase err = <-errCh:\n\t\texit(log, err, \"failed to start activator server\")\n\tcase <-sigint:\n\t\tlog.Info(\"Received INT or TERM signal, handling a graceful shutdown...\")\n\n\t\tfor name, server := range servers {\n\t\t\tlog.Infof(\"Shutting down %s server\", name)\n\t\t\tif err = server.Shutdown(context.Background()); err != nil {\n\t\t\t\t// Error from closing listeners, or context timeout:\n\t\t\t\tlog.Warnw(\"HTTP server Shutdown Error\", zap.Error(err))\n\t\t\t\ttelemetry.Error(errors.Wrap(err, \"HTTP server Shutdown Error\"))\n\t\t\t}\n\t\t}\n\t\tlog.Info(\"Shutdown complete, exiting...\")\n\t}\n}\n\nfunc informerFilter(listOptions *kmeta.ListOptions) {\n\tlistOptions.LabelSelector = kmeta.FormatLabelSelector(&kmeta.LabelSelector{\n\t\tMatchLabels: map[string]string{\n\t\t\t\"apiKind\": userconfig.RealtimeAPIKind.String(),\n\t\t},\n\t\tMatchExpressions: []kmeta.LabelSelectorRequirement{\n\t\t\t{\n\t\t\t\tKey:      \"apiName\",\n\t\t\t\tOperator: kmeta.LabelSelectorOpExists,\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc exit(log *zap.SugaredLogger, err error, wrapStrs ...string) {\n\tif err == nil {\n\t\tos.Exit(0)\n\t}\n\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\ttelemetry.Error(err)\n\tif !errors.IsNoPrint(err) {\n\t\tlog.Fatal(err)\n\t}\n\tos.Exit(1)\n}\n"
  },
  {
    "path": "cmd/async-gateway/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"flag\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\n\tgateway \"github.com/cortexlabs/cortex/pkg/async-gateway\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/handlers\"\n\t\"github.com/gorilla/mux\"\n\t\"go.uber.org/zap\"\n)\n\nconst (\n\t_defaultPort = \"8080\"\n)\n\n// usage: ./gateway -bucket <bucket> -region <region> -port <port>\nfunc main() {\n\tlog := logging.GetLogger()\n\tdefer func() {\n\t\t_ = log.Sync()\n\t}()\n\n\tvar (\n\t\tbucket     = flag.String(\"bucket\", \"\", \"bucket\")\n\t\tclusterUID = flag.String(\"cluster-uid\", \"\", \"cluster uid\")\n\t\tport       = flag.String(\"port\", _defaultPort, \"port on which the gateway server runs on\")\n\t)\n\tflag.Parse()\n\n\tswitch {\n\tcase *bucket == \"\":\n\t\tlog.Fatal(\"missing required option: -bucket\")\n\tcase *clusterUID == \"\":\n\t\tlog.Fatal(\"missing required option: -cluster-uid\")\n\t}\n\n\tawsClient, err := aws.New()\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\t_, userID, err := awsClient.CheckCredentials()\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\ttelemetryEnabled := strings.ToLower(os.Getenv(\"CORTEX_TELEMETRY_DISABLE\")) != \"true\"\n\terr = telemetry.Init(telemetry.Config{\n\t\tEnabled: telemetryEnabled,\n\t\tUserID:  userID,\n\t\tProperties: map[string]string{\n\t\t\t\"kind\":       userconfig.AsyncAPIKind.String(),\n\t\t\t\"image_type\": \"async-gateway\",\n\t\t},\n\t\tEnvironment: \"api\",\n\t\tLogErrors:   true,\n\t\tBackoffMode: telemetry.BackoffDuplicateMessages,\n\t})\n\tif err != nil {\n\t\tlog.Fatalw(\"failed to initialize telemetry\", zap.Error(err))\n\t}\n\tdefer telemetry.Close()\n\n\tsess := awsClient.Session()\n\ts3Storage := gateway.NewS3(sess, *bucket)\n\n\tsvc := gateway.NewService(*clusterUID, s3Storage, log, *sess)\n\tep := gateway.NewEndpoint(svc, log)\n\n\trouter := mux.NewRouter()\n\trouter.HandleFunc(\"/\", ep.CreateWorkload).Methods(\"POST\")\n\trouter.HandleFunc(\n\t\t\"/healthz\",\n\t\tfunc(w http.ResponseWriter, r *http.Request) {\n\t\t\tw.Header().Set(\"Content-Type\", \"text/plain\")\n\t\t\tw.WriteHeader(http.StatusOK)\n\t\t\t_, _ = w.Write([]byte(\"ok\"))\n\t\t},\n\t)\n\trouter.HandleFunc(\"/{id}\", ep.GetWorkload).Methods(\"GET\")\n\n\t// inspired by our nginx config\n\tcorsOptions := []handlers.CORSOption{\n\t\thandlers.AllowedOrigins([]string{\"*\"}),\n\t\t// custom headers are not supported currently, since \"*\" is not supported in AllowedHeaders(); here are some common ones:\n\t\thandlers.AllowedHeaders([]string{\"Content-Type\", \"X-Requested-With\", \"User-Agent\", \"Accept\", \"Accept-Language\", \"Content-Language\", \"Origin\"}),\n\t\thandlers.AllowedMethods([]string{\"GET\", \"HEAD\", \"POST\", \"PUT\", \"OPTIONS\"}),\n\t\thandlers.ExposedHeaders([]string{\"Content-Length\", \"Content-Range\"}),\n\t\thandlers.AllowCredentials(),\n\t}\n\n\tlog.Info(\"Running on port \" + *port)\n\tif err = http.ListenAndServe(\":\"+*port, handlers.CORS(corsOptions...)(router)); err != nil {\n\t\texit(log, err)\n\t}\n}\n\nfunc exit(log *zap.SugaredLogger, err error, wrapStrs ...string) {\n\tif err == nil {\n\t\tos.Exit(0)\n\t}\n\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\ttelemetry.Error(err)\n\tif !errors.IsNoPrint(err) {\n\t\tlog.Fatal(err)\n\t}\n\tos.Exit(1)\n}\n"
  },
  {
    "path": "cmd/autoscaler/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"strconv\"\n\t\"strings\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/autoscaler\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n\tpromapi \"github.com/prometheus/client_golang/api\"\n\tpromv1 \"github.com/prometheus/client_golang/api/prometheus/v1\"\n\t\"go.uber.org/zap\"\n\tistioclient \"istio.io/client-go/pkg/clientset/versioned\"\n\tistioinformers \"istio.io/client-go/pkg/informers/externalversions\"\n\t\"k8s.io/apimachinery/pkg/api/meta\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\tclientgoscheme \"k8s.io/client-go/kubernetes/scheme\"\n\t\"k8s.io/client-go/tools/cache\"\n)\n\nfunc main() {\n\tvar (\n\t\tport          int\n\t\tinCluster     bool\n\t\tprometheusURL string\n\t\tnamespace     string\n\t)\n\n\tflag.IntVar(&port, \"port\", 8000, \"port where the autoscaler server will be exposed\")\n\tflag.BoolVar(&inCluster, \"in-cluster\", false, \"use when autoscaler runs in-cluster\")\n\tflag.StringVar(&prometheusURL, \"prometheus-url\", os.Getenv(\"CORTEX_PROMETHEUS_URL\"),\n\t\t\"prometheus url (can be set through the CORTEX_PROMETHEUS_URL env variable)\",\n\t)\n\tflag.StringVar(&namespace, \"namespace\", os.Getenv(\"CORTEX_NAMESPACE\"),\n\t\t\"kubernetes namespace where the cortex APIs are deployed \"+\n\t\t\t\"(can be set through the CORTEX_NAMESPACE env variable)\",\n\t)\n\tflag.Parse()\n\n\tlog := logging.GetLogger()\n\tdefer func() {\n\t\t_ = log.Sync()\n\t}()\n\n\tswitch {\n\tcase prometheusURL == \"\":\n\t\tlog.Fatal(\"--prometheus-url is a required option\")\n\tcase namespace == \"\":\n\t\tlog.Fatal(\"--namespace is a required option\")\n\t}\n\n\tawsClient, err := aws.New()\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\t_, userID, err := awsClient.CheckCredentials()\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\ttelemetryEnabled := strings.ToLower(os.Getenv(\"CORTEX_TELEMETRY_DISABLE\")) != \"true\"\n\n\terr = telemetry.Init(telemetry.Config{\n\t\tEnabled: telemetryEnabled,\n\t\tUserID:  userID,\n\t\tProperties: map[string]string{\n\t\t\t\"kind\":       userconfig.RealtimeAPIKind.String(),\n\t\t\t\"image_type\": \"autoscaler\",\n\t\t},\n\t\tEnvironment: \"operator\",\n\t\tLogErrors:   true,\n\t\tBackoffMode: telemetry.BackoffDuplicateMessages,\n\t})\n\tif err != nil {\n\t\tlog.Fatalw(\"failed to initialize telemetry\", zap.Error(err))\n\t}\n\tdefer telemetry.Close()\n\n\tscheme := runtime.NewScheme()\n\tif err := clientgoscheme.AddToScheme(scheme); err != nil {\n\t\texit(log, err, \"failed to add k8s client-go-scheme to scheme\")\n\t}\n\n\tk8sClient, err := k8s.New(namespace, inCluster, nil, scheme)\n\tif err != nil {\n\t\texit(log, err, \"failed to initialize kubernetes client\")\n\t}\n\n\t//goland:noinspection GoNilness\n\tistioClient, err := istioclient.NewForConfig(k8sClient.RestConfig)\n\tif err != nil {\n\t\texit(log, err, \"failed to initialize istio client\")\n\t}\n\n\tpromClient, err := promapi.NewClient(\n\t\tpromapi.Config{\n\t\t\tAddress: prometheusURL,\n\t\t},\n\t)\n\tif err != nil {\n\t\texit(log, err, \"failed to initialize prometheus client\")\n\t}\n\n\tpromAPIClient := promv1.NewAPI(promClient)\n\n\trealtimeScaler := autoscaler.NewRealtimeScaler(k8sClient, promAPIClient, log)\n\tasyncScaler := autoscaler.NewAsyncScaler(k8sClient, promAPIClient)\n\n\tautoScaler := autoscaler.New(log)\n\tautoScaler.AddScaler(realtimeScaler, userconfig.RealtimeAPIKind)\n\tautoScaler.AddScaler(asyncScaler, userconfig.AsyncAPIKind)\n\tdefer autoScaler.Stop()\n\n\tistioInformerFactory := istioinformers.NewSharedInformerFactoryWithOptions(\n\t\tistioClient, 10*time.Second, // TODO: check how much makes sense\n\t\tistioinformers.WithNamespace(namespace),\n\t\tistioinformers.WithTweakListOptions(informerFilter),\n\t)\n\tvirtualServiceInformer := istioInformerFactory.Networking().V1beta1().VirtualServices().Informer()\n\tvirtualServiceInformer.AddEventHandler(\n\t\tcache.ResourceEventHandlerFuncs{\n\t\t\tAddFunc: func(obj interface{}) {\n\t\t\t\tresource, err := meta.Accessor(obj)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorw(\"failed to access resource metadata\", zap.Error(err))\n\t\t\t\t\ttelemetry.Error(err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tif resource.GetNamespace() != namespace {\n\t\t\t\t\t// filter out virtual services that are not in the cortex namespace\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tapi, err := apiResourceFromLabels(resource.GetLabels())\n\t\t\t\tif err != nil {\n\t\t\t\t\t// filter out non-cortex apis\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tif err := autoScaler.AddAPI(api); err != nil {\n\t\t\t\t\tlog.Errorw(\"failed to add API to autoscaler\",\n\t\t\t\t\t\tzap.Error(err),\n\t\t\t\t\t\tzap.String(\"apiName\", api.Name),\n\t\t\t\t\t\tzap.String(\"apiKind\", api.Kind.String()),\n\t\t\t\t\t)\n\t\t\t\t\ttelemetry.Error(err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t},\n\t\t\tDeleteFunc: func(obj interface{}) {\n\t\t\t\tresource, err := meta.Accessor(obj)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Errorw(\"failed to access resource metadata\", zap.Error(err))\n\t\t\t\t}\n\n\t\t\t\tif resource.GetNamespace() != namespace {\n\t\t\t\t\t// filter out virtual services that are not in the cortex namespace\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tapi, err := apiResourceFromLabels(resource.GetLabels())\n\t\t\t\tif err != nil {\n\t\t\t\t\t// filter out non-cortex apis\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\tautoScaler.RemoveAPI(api)\n\t\t\t},\n\t\t},\n\t)\n\n\thandler := autoscaler.NewHandler(autoScaler)\n\trouter := mux.NewRouter()\n\trouter.HandleFunc(\"/awaken\", handler.Awaken).Methods(http.MethodPost)\n\trouter.HandleFunc(\"/healthz\", func(w http.ResponseWriter, r *http.Request) {\n\t\tw.Header().Set(\"Content-Type\", \"text/plain; charset=utf-8\")\n\t\tw.WriteHeader(http.StatusOK)\n\t\t_, _ = w.Write([]byte(\"ok\"))\n\t}).Methods(http.MethodGet)\n\n\tserver := &http.Server{\n\t\tAddr:    \":\" + strconv.Itoa(port),\n\t\tHandler: router,\n\t}\n\n\tstopCh := make(chan struct{})\n\tgo virtualServiceInformer.Run(stopCh)\n\tdefer func() { stopCh <- struct{}{} }()\n\n\terrCh := make(chan error)\n\tgo func() {\n\t\tlog.Infof(\"Starting autoscaler server on %s\", server.Addr)\n\t\terrCh <- server.ListenAndServe()\n\t}()\n\n\tsigint := make(chan os.Signal, 1)\n\tsignal.Notify(sigint, os.Interrupt, syscall.SIGTERM)\n\n\tselect {\n\tcase err = <-errCh:\n\t\texit(log, err, \"failed to start autoscaler server\")\n\tcase <-sigint:\n\t\tlog.Info(\"Received INT or TERM signal, handling a graceful shutdown...\")\n\t\tlog.Info(\"Shutting down server\")\n\t\tif err = server.Shutdown(context.Background()); err != nil {\n\t\t\t// Error from closing listeners, or context timeout:\n\t\t\tlog.Warnw(\"HTTP server Shutdown Error\", zap.Error(err))\n\t\t}\n\t\tlog.Info(\"Shutdown complete, exiting...\")\n\t}\n}\n\nfunc apiResourceFromLabels(labels map[string]string) (userconfig.Resource, error) {\n\tapiName, ok := labels[\"apiName\"]\n\tif !ok {\n\t\treturn userconfig.Resource{}, fmt.Errorf(\"apiName key does not exist\")\n\t}\n\n\tapiKind, ok := labels[\"apiKind\"]\n\tif !ok {\n\t\treturn userconfig.Resource{}, fmt.Errorf(\"apiKind key does not exist\")\n\t}\n\n\treturn userconfig.Resource{\n\t\tName: apiName,\n\t\tKind: userconfig.KindFromString(apiKind),\n\t}, nil\n}\n\nfunc informerFilter(listOptions *kmeta.ListOptions) {\n\tlistOptions.LabelSelector = kmeta.FormatLabelSelector(&kmeta.LabelSelector{\n\t\tMatchExpressions: []kmeta.LabelSelectorRequirement{\n\t\t\t{\n\t\t\t\tKey:      \"apiName\",\n\t\t\t\tOperator: kmeta.LabelSelectorOpExists,\n\t\t\t},\n\t\t\t{\n\t\t\t\tKey:      \"apiKind\",\n\t\t\t\tOperator: kmeta.LabelSelectorOpExists,\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc exit(log *zap.SugaredLogger, err error, wrapStrs ...string) {\n\tif err == nil {\n\t\tos.Exit(0)\n\t}\n\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\ttelemetry.Error(err)\n\tif !errors.IsNoPrint(err) {\n\t\tlog.Fatal(err)\n\t}\n\tos.Exit(1)\n}\n"
  },
  {
    "path": "cmd/dequeuer/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"strconv\"\n\t\"syscall\"\n\n\t\"github.com/DataDog/datadog-go/statsd\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/dequeuer\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/probe\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"go.uber.org/zap\"\n)\n\nfunc main() {\n\tvar (\n\t\tclusterConfigPath string\n\t\tclusterUID        string\n\t\tprobesPath        string\n\t\tqueueURL          string\n\t\tuserContainerPort int\n\t\tapiName           string\n\t\tjobID             string\n\t\tstatsdAddress     string\n\t\tapiKind           string\n\t\tadminPort         int\n\t\tworkers           int\n\t)\n\tflag.StringVar(&clusterConfigPath, \"cluster-config\", \"\", \"cluster config path\")\n\tflag.StringVar(&clusterUID, \"cluster-uid\", \"\", \"cluster unique identifier\")\n\tflag.StringVar(&probesPath, \"probes-path\", \"\", \"path to the probes spec\")\n\tflag.StringVar(&queueURL, \"queue\", \"\", \"target queue URL from which the api messages will be dequeued\")\n\tflag.StringVar(&apiKind, \"api-kind\", \"\", fmt.Sprintf(\"api kind (%s|%s)\", userconfig.BatchAPIKind.String(), userconfig.AsyncAPIKind.String()))\n\tflag.StringVar(&apiName, \"api-name\", \"\", \"api name\")\n\tflag.StringVar(&jobID, \"job-id\", \"\", \"job ID\")\n\tflag.StringVar(&statsdAddress, \"statsd-address\", \"\", \"address to push statsd metrics\")\n\tflag.IntVar(&userContainerPort, \"user-port\", 8080, \"target port to which the dequeued messages will be sent to\")\n\tflag.IntVar(&adminPort, \"admin-port\", 0, \"port where the admin server (for the probes) will be exposed\")\n\tflag.IntVar(&workers, \"workers\", 1, \"number of workers pulling from the queue\")\n\n\tflag.Parse()\n\n\tversion := os.Getenv(\"CORTEX_VERSION\")\n\tif version == \"\" {\n\t\tversion = consts.CortexVersion\n\t}\n\n\tlog := logging.GetLogger()\n\tdefer func() {\n\t\t_ = log.Sync()\n\t}()\n\n\tswitch {\n\tcase clusterConfigPath == \"\":\n\t\tlog.Fatal(\"--cluster-config is a required option\")\n\tcase probesPath == \"\":\n\t\tlog.Fatal(\"--probes-path is a required option\")\n\tcase queueURL == \"\":\n\t\tlog.Fatal(\"--queue is a required option\")\n\tcase apiName == \"\":\n\t\tlog.Fatal(\"--api-name is a required option\")\n\tcase apiKind == \"\":\n\t\tlog.Fatal(\"--api-kind is a required option\")\n\tcase adminPort == 0:\n\t\tlog.Fatal(\"--admin-port is a required option\")\n\t}\n\n\ttargetURL := \"http://127.0.0.1:\" + strconv.Itoa(userContainerPort)\n\n\tclusterConfig, err := clusterconfig.NewForFile(clusterConfigPath)\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\tawsClient, err := awslib.NewForRegion(clusterConfig.Region)\n\tif err != nil {\n\t\texit(log, err, \"failed to create aws client\")\n\t}\n\n\t_, userID, err := awsClient.CheckCredentials()\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\terr = telemetry.Init(telemetry.Config{\n\t\tEnabled: clusterConfig.Telemetry,\n\t\tUserID:  userID,\n\t\tProperties: map[string]string{\n\t\t\t\"kind\":       apiKind,\n\t\t\t\"image_type\": \"dequeuer\",\n\t\t},\n\t\tEnvironment: \"api\",\n\t\tLogErrors:   true,\n\t\tBackoffMode: telemetry.BackoffDuplicateMessages,\n\t})\n\tif err != nil {\n\t\tlog.Fatalw(\"failed to initialize telemetry\", \"error\", err)\n\t}\n\tdefer telemetry.Close()\n\n\tvar probes []*probe.Probe\n\tif files.IsFile(probesPath) {\n\t\tprobes, err = dequeuer.ProbesFromFile(probesPath, log)\n\t\tif err != nil {\n\t\t\texit(log, err, fmt.Sprintf(\"unable to read probes from %s\", probesPath))\n\t\t}\n\t}\n\n\tif !dequeuer.HasTCPProbeTargetingUserPod(probes, userContainerPort) {\n\t\tprobes = append(probes, probe.NewDefaultProbe(fmt.Sprintf(\"http://localhost:%d\", userContainerPort), log))\n\t}\n\n\tadminHandler := http.NewServeMux()\n\tadminHandler.Handle(\"/healthz\", dequeuer.HealthcheckHandler(func() bool {\n\t\treturn probe.AreProbesHealthy(probes)\n\t}))\n\n\tvar dequeuerConfig dequeuer.SQSDequeuerConfig\n\tvar messageHandler dequeuer.MessageHandler\n\n\tswitch apiKind {\n\tcase userconfig.BatchAPIKind.String():\n\t\tif jobID == \"\" {\n\t\t\tlog.Fatal(\"--job-id is a required option\")\n\t\t}\n\n\t\tconfig := dequeuer.BatchMessageHandlerConfig{\n\t\t\tRegion:    clusterConfig.Region,\n\t\t\tAPIName:   apiName,\n\t\t\tJobID:     jobID,\n\t\t\tQueueURL:  queueURL,\n\t\t\tTargetURL: targetURL,\n\t\t}\n\n\t\tmetricsClient, err := statsd.New(statsdAddress)\n\t\tif err != nil {\n\t\t\texit(log, err, \"unable to initialize metrics client\")\n\t\t}\n\n\t\tmessageHandler = dequeuer.NewBatchMessageHandler(config, awsClient, metricsClient, log)\n\t\tdequeuerConfig = dequeuer.SQSDequeuerConfig{\n\t\t\tRegion:           clusterConfig.Region,\n\t\t\tQueueURL:         queueURL,\n\t\t\tStopIfNoMessages: true,\n\t\t\tWorkers:          workers,\n\t\t}\n\n\tcase userconfig.AsyncAPIKind.String():\n\t\tif clusterUID == \"\" {\n\t\t\tlog.Fatal(\"--cluster-uid is a required option\")\n\t\t}\n\n\t\tconfig := dequeuer.AsyncMessageHandlerConfig{\n\t\t\tClusterUID: clusterUID,\n\t\t\tBucket:     clusterConfig.Bucket,\n\t\t\tAPIName:    apiName,\n\t\t\tTargetURL:  targetURL,\n\t\t}\n\n\t\tasyncStatsReporter := dequeuer.NewAsyncPrometheusStatsReporter()\n\t\tmessageHandler = dequeuer.NewAsyncMessageHandler(config, awsClient, asyncStatsReporter, log)\n\t\tdequeuerConfig = dequeuer.SQSDequeuerConfig{\n\t\t\tRegion:           clusterConfig.Region,\n\t\t\tQueueURL:         queueURL,\n\t\t\tStopIfNoMessages: false,\n\t\t\tWorkers:          workers,\n\t\t}\n\n\t\t// report prometheus metrics for async api kinds\n\t\tadminHandler.Handle(\"/metrics\", asyncStatsReporter)\n\tdefault:\n\t\texit(log, err, fmt.Sprintf(\"kind %s is not supported\", apiKind))\n\t}\n\n\terrCh := make(chan error)\n\n\tgo func() {\n\t\tserver := &http.Server{\n\t\t\tAddr:    \":\" + strconv.Itoa(adminPort),\n\t\t\tHandler: adminHandler,\n\t\t}\n\t\tlog.Infof(\"Starting %s server on %s\", consts.AdminPortName, server.Addr)\n\t\terrCh <- server.ListenAndServe()\n\t}()\n\n\tsigint := make(chan os.Signal, 1)\n\tsignal.Notify(sigint, os.Interrupt, syscall.SIGTERM)\n\n\tsqsDequeuer, err := dequeuer.NewSQSDequeuer(dequeuerConfig, awsClient, log)\n\tif err != nil {\n\t\texit(log, err, \"failed to create sqs dequeuer\")\n\t}\n\n\tgo func() {\n\t\tlog.Info(\"Starting dequeuer...\")\n\t\terrCh <- sqsDequeuer.Start(messageHandler, func() bool {\n\t\t\treturn probe.AreProbesHealthy(probes)\n\t\t})\n\t}()\n\n\tvar stopChs []chan struct{}\n\tfor _, p := range probes {\n\t\tstopChs = append(stopChs, p.StartProbing())\n\t}\n\n\tdefer func() {\n\t\tfor _, stopCh := range stopChs {\n\t\t\tstopCh <- struct{}{}\n\t\t}\n\t}()\n\n\tselect {\n\tcase err = <-errCh:\n\t\texit(log, err, \"error during message dequeueing or error from admin server\")\n\tcase <-sigint:\n\t\tlog.Info(\"Received INT or TERM signal, handling a graceful shutdown...\")\n\t\tsqsDequeuer.Shutdown()\n\t\tlog.Info(\"Shutdown complete, exiting...\")\n\t}\n}\n\nfunc exit(log *zap.SugaredLogger, err error, wrapStrs ...string) {\n\tif err == nil {\n\t\tos.Exit(0)\n\t}\n\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\ttelemetry.Error(err)\n\tif !errors.IsNoPrint(err) {\n\t\tlog.Fatal(err)\n\t}\n\n\tos.Exit(1)\n}\n"
  },
  {
    "path": "cmd/enqueuer/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"flag\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/enqueuer\"\n\t\"go.uber.org/zap\"\n\t\"go.uber.org/zap/zapcore\"\n)\n\nfunc createLogger() (*zap.Logger, error) {\n\tlogLevelEnv := strings.ToLower(os.Getenv(\"CORTEX_LOG_LEVEL\"))\n\tdisableJSONLogging := os.Getenv(\"CORTEX_DISABLE_JSON_LOGGING\")\n\n\tvar logLevelZap zapcore.Level\n\tswitch logLevelEnv {\n\tcase \"debug\":\n\t\tlogLevelZap = zapcore.DebugLevel\n\tcase \"warning\":\n\t\tlogLevelZap = zapcore.WarnLevel\n\tcase \"error\":\n\t\tlogLevelZap = zapcore.ErrorLevel\n\tdefault:\n\t\tlogLevelZap = zapcore.InfoLevel\n\t}\n\n\tencoderConfig := zap.NewProductionEncoderConfig()\n\tencoderConfig.MessageKey = \"message\"\n\n\tencoding := \"json\"\n\tif strings.ToLower(disableJSONLogging) == \"true\" {\n\t\tencoding = \"console\"\n\t}\n\n\treturn zap.Config{\n\t\tLevel:            zap.NewAtomicLevelAt(logLevelZap),\n\t\tEncoding:         encoding,\n\t\tEncoderConfig:    encoderConfig,\n\t\tOutputPaths:      []string{\"stdout\"},\n\t\tErrorOutputPaths: []string{\"stderr\"},\n\t}.Build()\n}\n\nfunc main() {\n\tvar (\n\t\tclusterUID string\n\t\tregion     string\n\t\tbucket     string\n\t\tqueueURL   string\n\t\tapiName    string\n\t\tjobID      string\n\t)\n\tflag.StringVar(&clusterUID, \"cluster-uid\", os.Getenv(\"CORTEX_CLUSTER_UID\"), \"cluster UID (can be set throught the CORTEX_CLUSTER_UID env variable)\")\n\tflag.StringVar(&region, \"region\", os.Getenv(\"CORTEX_REGION\"), \"cluster region (can be set throught the CORTEX_REGION env variable)\")\n\tflag.StringVar(&bucket, \"bucket\", os.Getenv(\"CORTEX_BUCKET\"), \"cortex S3 bucket (can be set throught the CORTEX_BUCKET env variable)\")\n\tflag.StringVar(&queueURL, \"queue\", \"\", \"target queue URL to where the api messages will be enqueued\")\n\tflag.StringVar(&apiName, \"apiName\", \"\", \"api name\")\n\tflag.StringVar(&jobID, \"jobID\", \"\", \"job ID\")\n\n\tflag.Parse()\n\n\tversion := os.Getenv(\"CORTEX_VERSION\")\n\tif version == \"\" {\n\t\tversion = consts.CortexVersion\n\t}\n\n\tlog, err := createLogger()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer func() {\n\t\t_ = log.Sync()\n\t}()\n\n\tswitch {\n\tcase clusterUID == \"\":\n\t\tlog.Fatal(\"-cluster-uid is a required option\")\n\tcase region == \"\":\n\t\tlog.Fatal(\"-region is a required option\")\n\tcase bucket == \"\":\n\t\tlog.Fatal(\"-bucket is a required option\")\n\tcase queueURL == \"\":\n\t\tlog.Fatal(\"-queue is a required option\")\n\tcase apiName == \"\":\n\t\tlog.Fatal(\"-apiName is a required option\")\n\tcase jobID == \"\":\n\t\tlog.Fatal(\"-jobID is a required option\")\n\t}\n\n\tenvConfig := enqueuer.EnvConfig{\n\t\tClusterUID: clusterUID,\n\t\tRegion:     region,\n\t\tVersion:    version,\n\t\tBucket:     bucket,\n\t\tAPIName:    apiName,\n\t\tJobID:      jobID,\n\t}\n\n\teqr, err := enqueuer.NewEnqueuer(envConfig, queueURL, log)\n\tif err != nil {\n\t\tlog.Fatal(\"failed to create enqueuer\", zap.Error(err))\n\t}\n\n\ttotalBatches, err := eqr.Enqueue()\n\tif err != nil {\n\t\tlog.Fatal(\"failed to enqueue batches\", zap.Error(err))\n\t}\n\n\tif err = eqr.UploadBatchCount(totalBatches); err != nil {\n\t\tlog.Fatal(\"failed to upload batch count\", zap.Error(err))\n\t}\n\n\tlog.Info(\"done enqueuing batches\", zap.Int(\"batchCount\", totalBatches))\n}\n"
  },
  {
    "path": "cmd/operator/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/cron\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/endpoints\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/asyncapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/handlers\"\n\t\"github.com/gorilla/mux\"\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n)\n\nvar operatorLogger = logging.GetLogger()\n\nconst _operatorPortStr = \"8888\"\n\nfunc main() {\n\tif err := config.Init(); err != nil {\n\t\texit.ErrorNoTelemetry(errors.Wrap(err, \"init\"))\n\t}\n\n\ttelemetry.Event(\"operator.init\")\n\n\tcron.Run(operator.DeleteEvictedPods, operator.ErrorHandler(\"delete evicted pods\"), time.Hour)\n\tcron.Run(operator.ClusterTelemetry, operator.ErrorHandler(\"instance telemetry\"), 1*time.Hour)\n\tcron.Run(operator.CostBreakdown, operator.ErrorHandler(\"cost breakdown metrics\"), 5*time.Minute)\n\n\t_, err := operator.UpdateMemoryCapacityConfigMap()\n\tif err != nil {\n\t\texit.Error(errors.Wrap(err, \"init\"))\n\t}\n\n\tcron.Run(taskapi.ManageJobResources, operator.ErrorHandler(\"manage task jobs\"), taskapi.ManageJobResourcesCronPeriod)\n\n\tdeployments, err := config.K8s.ListDeploymentsWithLabelKeys(\"apiName\")\n\tif err != nil {\n\t\texit.Error(errors.Wrap(err, \"init\"))\n\t}\n\n\tfor i := range deployments {\n\t\tdeployment := deployments[i]\n\t\tapiKind := deployment.Labels[\"apiKind\"]\n\t\tswitch apiKind {\n\t\tcase userconfig.AsyncAPIKind.String():\n\t\t\tif err := asyncapi.UpdateAPIMetricsCron(&deployment); err != nil {\n\t\t\t\toperatorLogger.Fatal(errors.Wrap(err, \"init\"))\n\t\t\t}\n\t\t}\n\t}\n\n\trouter := mux.NewRouter()\n\n\trouterWithoutAuth := router.NewRoute().Subrouter()\n\trouterWithoutAuth.Use(endpoints.PanicMiddleware)\n\trouterWithoutAuth.HandleFunc(\"/verifycortex\", endpoints.VerifyCortex).Methods(\"GET\")\n\n\trouterWithoutAuth.HandleFunc(\"/batch/{apiName}\", endpoints.SubmitBatchJob).Methods(\"POST\")\n\trouterWithoutAuth.HandleFunc(\"/batch/{apiName}\", endpoints.GetBatchJob).Methods(\"GET\")\n\trouterWithoutAuth.HandleFunc(\"/batch/{apiName}\", endpoints.StopBatchJob).Methods(\"DELETE\")\n\trouterWithoutAuth.HandleFunc(\"/tasks/{apiName}\", endpoints.SubmitTaskJob).Methods(\"POST\")\n\trouterWithoutAuth.HandleFunc(\"/tasks/{apiName}\", endpoints.GetTaskJob).Methods(\"GET\")\n\trouterWithoutAuth.HandleFunc(\"/tasks/{apiName}\", endpoints.StopTaskJob).Methods(\"DELETE\")\n\n\t// prometheus metrics\n\trouterWithoutAuth.Handle(\"/metrics\", promhttp.Handler()).Methods(\"GET\")\n\n\trouterWithAuth := router.NewRoute().Subrouter()\n\n\trouterWithAuth.Use(endpoints.PanicMiddleware)\n\trouterWithAuth.Use(endpoints.APIVersionCheckMiddleware)\n\trouterWithAuth.Use(endpoints.AWSAuthMiddleware)\n\trouterWithAuth.Use(endpoints.ClientIDMiddleware)\n\n\trouterWithAuth.HandleFunc(\"/info\", endpoints.Info).Methods(\"GET\")\n\trouterWithAuth.HandleFunc(\"/deploy\", endpoints.Deploy).Methods(\"POST\")\n\trouterWithAuth.HandleFunc(\"/refresh/{apiName}\", endpoints.Refresh).Methods(\"POST\")\n\trouterWithAuth.HandleFunc(\"/delete/{apiName}\", endpoints.Delete).Methods(\"DELETE\")\n\trouterWithAuth.HandleFunc(\"/get\", endpoints.GetAPIs).Methods(\"GET\")\n\trouterWithAuth.HandleFunc(\"/get/{apiName}\", endpoints.GetAPI).Methods(\"GET\")\n\trouterWithAuth.HandleFunc(\"/get/{apiName}/{apiID}\", endpoints.GetAPIByID).Methods(\"GET\")\n\trouterWithAuth.HandleFunc(\"/describe/{apiName}\", endpoints.DescribeAPI).Methods(\"GET\")\n\trouterWithAuth.HandleFunc(\"/streamlogs/{apiName}\", endpoints.ReadLogs)\n\trouterWithAuth.HandleFunc(\"/logs/{apiName}\", endpoints.GetLogURL).Methods(\"GET\")\n\n\toperatorLogger.Info(\"Running on port \" + _operatorPortStr)\n\n\t// inspired by our nginx config\n\tcorsOptions := []handlers.CORSOption{\n\t\thandlers.AllowedOrigins([]string{\"*\"}),\n\t\t// custom headers are not supported currently, since \"*\" is not supported in AllowedHeaders(); here are some common ones:\n\t\thandlers.AllowedHeaders([]string{\"Content-Type\", \"X-Requested-With\", \"User-Agent\", \"Accept\", \"Accept-Language\", \"Content-Language\", \"Origin\"}),\n\t\thandlers.AllowedMethods([]string{\"GET\", \"HEAD\", \"POST\", \"PUT\", \"OPTIONS\"}),\n\t\thandlers.ExposedHeaders([]string{\"Content-Length\", \"Content-Range\"}),\n\t\thandlers.AllowCredentials(),\n\t}\n\n\toperatorLogger.Fatal(http.ListenAndServe(\":\"+_operatorPortStr, handlers.CORS(corsOptions...)(router)))\n}\n"
  },
  {
    "path": "cmd/proxy/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"strconv\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/proxy\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"go.uber.org/zap\"\n)\n\nconst (\n\t_reportInterval        = 10 * time.Second\n\t_requestSampleInterval = 1 * time.Second\n)\n\nfunc main() {\n\tvar (\n\t\tport              int\n\t\tadminPort         int\n\t\tuserContainerPort int\n\t\tmaxConcurrency    int\n\t\tmaxQueueLength    int\n\t\thasTCPProbe       bool\n\t\tclusterConfigPath string\n\t)\n\n\tflag.IntVar(&port, \"port\", 8000, \"port where the proxy server will be exposed\")\n\tflag.IntVar(&adminPort, \"admin-port\", 15000, \"port where the admin server (for metrics and probes) will be exposed\")\n\tflag.IntVar(&userContainerPort, \"user-port\", 8080, \"port where the proxy will redirect to the traffic to\")\n\tflag.IntVar(&maxConcurrency, \"max-concurrency\", 0, \"max concurrency allowed for user container\")\n\tflag.IntVar(&maxQueueLength, \"max-queue-length\", 0, \"max request queue length for user container\")\n\tflag.BoolVar(&hasTCPProbe, \"has-tcp-probe\", false, \"tcp probe to the user-provided container port\")\n\tflag.StringVar(&clusterConfigPath, \"cluster-config\", \"\", \"cluster config path\")\n\tflag.Parse()\n\n\tlog := logging.GetLogger()\n\tdefer func() {\n\t\t_ = log.Sync()\n\t}()\n\n\tswitch {\n\tcase maxConcurrency == 0:\n\t\tlog.Fatal(\"--max-concurrency flag is required\")\n\tcase maxQueueLength == 0:\n\t\tlog.Fatal(\"--max-queue-length flag is required\")\n\tcase clusterConfigPath == \"\":\n\t\tlog.Fatal(\"--cluster-config flag is required\")\n\t}\n\n\tclusterConfig, err := clusterconfig.NewForFile(clusterConfigPath)\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\tawsClient, err := aws.NewForRegion(clusterConfig.Region)\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\t_, userID, err := awsClient.CheckCredentials()\n\tif err != nil {\n\t\texit(log, err)\n\t}\n\n\terr = telemetry.Init(telemetry.Config{\n\t\tEnabled: clusterConfig.Telemetry,\n\t\tUserID:  userID,\n\t\tProperties: map[string]string{\n\t\t\t\"kind\":       userconfig.RealtimeAPIKind.String(),\n\t\t\t\"image_type\": \"proxy\",\n\t\t},\n\t\tEnvironment: \"api\",\n\t\tLogErrors:   true,\n\t\tBackoffMode: telemetry.BackoffDuplicateMessages,\n\t})\n\tif err != nil {\n\t\tlog.Fatalw(\"failed to initialize telemetry\", zap.Error(err))\n\t}\n\tdefer telemetry.Close()\n\n\ttarget := \"http://127.0.0.1:\" + strconv.Itoa(userContainerPort)\n\thttpProxy := proxy.NewReverseProxy(target, maxQueueLength, maxQueueLength)\n\n\trequestCounterStats := &proxy.RequestStats{}\n\tbreaker := proxy.NewBreaker(\n\t\tproxy.BreakerParams{\n\t\t\tQueueDepth:      maxQueueLength,\n\t\t\tMaxConcurrency:  maxConcurrency,\n\t\t\tInitialCapacity: maxConcurrency,\n\t\t},\n\t)\n\n\tpromStats := proxy.NewPrometheusStatsReporter()\n\n\tgo func() {\n\t\treportTicker := time.NewTicker(_reportInterval)\n\t\tdefer reportTicker.Stop()\n\n\t\trequestSamplingTicker := time.NewTicker(_requestSampleInterval)\n\t\tdefer requestSamplingTicker.Stop()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-reportTicker.C:\n\t\t\t\tgo func() {\n\t\t\t\t\treport := requestCounterStats.Report()\n\t\t\t\t\tpromStats.Report(report)\n\t\t\t\t}()\n\t\t\tcase <-requestSamplingTicker.C:\n\t\t\t\tgo func() {\n\t\t\t\t\trequestCounterStats.Append(breaker.InFlight())\n\t\t\t\t}()\n\t\t\t}\n\t\t}\n\t}()\n\n\tadminHandler := http.NewServeMux()\n\tadminHandler.Handle(\"/metrics\", promStats)\n\tadminHandler.Handle(\"/healthz\", readinessTCPHandler(userContainerPort, hasTCPProbe, log))\n\n\tservers := map[string]*http.Server{\n\t\t\"proxy\": {\n\t\t\tAddr:    \":\" + strconv.Itoa(port),\n\t\t\tHandler: proxy.Handler(breaker, httpProxy),\n\t\t},\n\t\t\"admin\": {\n\t\t\tAddr:    \":\" + strconv.Itoa(adminPort),\n\t\t\tHandler: adminHandler,\n\t\t},\n\t}\n\n\terrCh := make(chan error)\n\tfor name, server := range servers {\n\t\tgo func(name string, server *http.Server) {\n\t\t\tlog.Infof(\"Starting %s server on %s\", name, server.Addr)\n\t\t\terrCh <- server.ListenAndServe()\n\t\t}(name, server)\n\t}\n\n\tsigint := make(chan os.Signal, 1)\n\tsignal.Notify(sigint, os.Interrupt, syscall.SIGTERM)\n\n\tselect {\n\tcase err = <-errCh:\n\t\texit(log, errors.Wrap(err, \"failed to start proxy server\"))\n\tcase <-sigint:\n\t\tlog.Info(\"Received INT or TERM signal, handling a graceful shutdown...\")\n\n\t\tfor name, server := range servers {\n\t\t\tlog.Infof(\"Shutting down %s server\", name)\n\t\t\tif err = server.Shutdown(context.Background()); err != nil {\n\t\t\t\t// Error from closing listeners, or context timeout:\n\t\t\t\tlog.Warnw(\"HTTP server Shutdown Error\", zap.Error(err))\n\t\t\t\ttelemetry.Error(errors.Wrap(err, \"HTTP server Shutdown Error\"))\n\t\t\t}\n\t\t}\n\t\tlog.Info(\"Shutdown complete, exiting...\")\n\t}\n}\n\nfunc exit(log *zap.SugaredLogger, err error, wrapStrs ...string) {\n\tif err == nil {\n\t\tos.Exit(0)\n\t}\n\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\ttelemetry.Error(err)\n\tif !errors.IsNoPrint(err) {\n\t\tlog.Fatal(err)\n\t}\n\tos.Exit(1)\n}\n\nfunc readinessTCPHandler(port int, enableTCPProbe bool, logger *zap.SugaredLogger) http.HandlerFunc {\n\treturn func(w http.ResponseWriter, r *http.Request) {\n\t\tif enableTCPProbe {\n\t\t\tctx := r.Context()\n\t\t\taddress := net.JoinHostPort(\"localhost\", fmt.Sprintf(\"%d\", port))\n\n\t\t\tvar d net.Dialer\n\t\t\tconn, err := d.DialContext(ctx, \"tcp\", address)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Warn(errors.Wrap(err, \"TCP probe to user-provided container port failed\"))\n\t\t\t\tw.WriteHeader(http.StatusInternalServerError)\n\t\t\t\t_, _ = w.Write([]byte(\"unhealthy\"))\n\t\t\t\treturn\n\t\t\t}\n\t\t\t_ = conn.Close()\n\t\t}\n\n\t\tw.WriteHeader(http.StatusOK)\n\t\t_, _ = w.Write([]byte(\"healthy\"))\n\t}\n}\n"
  },
  {
    "path": "dev/build_cli.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\n$ROOT/build/cli.sh upload\n"
  },
  {
    "path": "dev/create_user.py",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nimport boto3\nimport configparser\nfrom pathlib import Path\nimport sys\nimport os\n\n# Usage: python create_user.py $CORTEX_CLUSTER_NAME $CORTEX_ACCOUNT_ID $CORTEX_REGION\n\ncluster_name = sys.argv[1]\naccount_id = sys.argv[2]\ncortex_region = sys.argv[3]\n\ndir_path = os.path.dirname(os.path.realpath(__file__))\n\nwith open(f\"{dir_path}/minimum_aws_policy.json\", \"r\") as f:\n    policy_string = f.read()\npolicy_string = policy_string.replace(\"$CORTEX_CLUSTER_NAME\", cluster_name)\npolicy_string = policy_string.replace(\"$CORTEX_REGION\", cortex_region)\npolicy_string = policy_string.replace(\"$CORTEX_ACCOUNT_ID\", account_id)\n\nuser_name = f\"dev-{cluster_name}-{cortex_region}\"\n\niam_client = boto3.client(\"iam\", region_name=cortex_region)\n\ntry:\n    iam_client.get_user(UserName=user_name)\nexcept iam_client.exceptions.NoSuchEntityException:\n    iam_client.create_user(UserName=user_name)\n\npartition = \"aws\"\nif \"us-gov\" in cortex_region:\n    partition = \"aws-us-gov\"\n\npolicy_arn = f\"arn:{partition}:iam::{account_id}:policy/{user_name}\"\n\ntry:\n    iam_client.get_policy(PolicyArn=policy_arn)\nexcept iam_client.exceptions.NoSuchEntityException:\n    iam_client.create_policy(\n        PolicyName=user_name,\n        PolicyDocument=policy_string,\n    )\n\npolicy_versions = iam_client.list_policy_versions(PolicyArn=policy_arn)[\"Versions\"]\n\nif len(policy_versions) == 5:\n    policy_versions.sort(key=lambda x: x[\"CreateDate\"])\n    oldest_version = policy_versions[0][\"VersionId\"]\n    iam_client.delete_policy_version(PolicyArn=policy_arn, VersionId=oldest_version)\n\n\niam_client.create_policy_version(\n    PolicyArn=policy_arn, PolicyDocument=policy_string, SetAsDefault=True\n)\n\niam_client.attach_user_policy(UserName=user_name, PolicyArn=policy_arn)\n\naws_credentials_path = Path(\"~/.aws/credentials\").expanduser()\nif not aws_credentials_path.exists():\n    aws_credentials_path.parent.mkdir(parents=True, exist_ok=True)\n    aws_credentials_path.touch()\n\naws_credentials = configparser.ConfigParser()\naws_credentials.read(aws_credentials_path)\nif not user_name in aws_credentials:\n    access_keys = iam_client.list_access_keys(UserName=user_name)[\"AccessKeyMetadata\"]\n    if len(access_keys) == 2:\n        access_keys.sort(key=lambda x: x[\"CreateDate\"])\n        iam_client.delete_access_key(UserName=user_name, AccessKeyId=access_keys[0][\"AccessKeyId\"])\n    key = iam_client.create_access_key(UserName=user_name)[\"AccessKey\"]\n    aws_credentials[user_name] = {\n        \"aws_access_key_id\": key[\"AccessKeyId\"],\n        \"aws_secret_access_key\": key[\"SecretAccessKey\"],\n    }\n\n    with open(aws_credentials_path, \"w\") as configfile:\n        aws_credentials.write(configfile)\n\naws_access_key_id = aws_credentials[user_name][\"aws_access_key_id\"]\naws_secret_access_key = aws_credentials[user_name][\"aws_secret_access_key\"]\n\nprint(f\"export AWS_ACCESS_KEY_ID={aws_access_key_id}\")\nprint(f\"export AWS_SECRET_ACCESS_KEY={aws_secret_access_key}\")\n"
  },
  {
    "path": "dev/delete_ecr_repos.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport boto3\n\necr = boto3.client(\"ecr\")\n\nresponse = ecr.describe_repositories(maxResults=1000)\n\nfor repo in response[\"repositories\"]:\n    ecr.delete_repository(\n        registryId=repo[\"registryId\"],\n        repositoryName=repo[\"repositoryName\"],\n        force=True,\n    )\n    print(f\"deleted{repo['repositoryName']}\")\n\nprint(\"done\")\n"
  },
  {
    "path": "dev/export_images.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -euo pipefail\n\n# usage: ./dev/export_images.sh <region> <aws account id>\n# e.g. ./dev/export_images.sh us-east-1 123456789\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\n# CORTEX_VERSION\ncortex_version=master\n\n# user set variables\necr_region=$1\naws_account_id=$2\n\nsource_registry=quay.io/cortexlabs  # this can also be docker.io/cortexlabs\n\ndestination_ecr_prefix=\"cortexlabs\"\ndestination_registry=\"${aws_account_id}.dkr.ecr.${ecr_region}.amazonaws.com/${destination_ecr_prefix}\"\n\nif [[ -f $HOME/.docker/config.json && $(cat $HOME/.docker/config.json | grep \"ecr-login\" | wc -l) -ne 0 ]]; then\n    echo \"skipping docker login because you are using ecr-login with Amazon ECR Docker Credential Helper\"\nelse\n    aws ecr get-login-password --region $ecr_region | docker login --username AWS --password-stdin $destination_registry\nfi\n\nsource $ROOT/build/images.sh\n\n# create the image repositories\nfor image in \"${all_images[@]}\"; do\n    repository_name=$destination_ecr_prefix/$image\n    if aws ecr describe-repositories --repository-names=$repository_name --region=$ecr_region >/dev/null 2>&1; then\n        echo \"repository '$repository_name' already exists\"\n    else\n        aws ecr create-repository --repository-name=$repository_name --region=$ecr_region | cat\n    fi\ndone\necho\n\n# pull the images from source registry and push them to ECR\nfor image in \"${all_images[@]}\"; do\n    echo \"copying $image:$cortex_version from $source_registry to $destination_registry\"\n    skopeo copy --src-no-creds \"docker://$source_registry/$image:$cortex_version\" \"docker://$destination_registry/$image:$cortex_version\"\n    echo\ndone\necho \"done ✓\"\n"
  },
  {
    "path": "dev/find_missing_docs_links.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport os\nimport re\nimport asyncio\nimport aiohttp\n\nscript_path = os.path.realpath(__file__)\nroot = os.path.dirname(os.path.dirname(script_path))\ndocs_root = os.path.join(root, \"docs\")\n\nskip_http = False\nif len(sys.argv) == 2 and sys.argv[1] == \"--skip-http\":\n    skip_http = True\n\n\ndef main():\n    files = get_docs_file_paths()\n\n    link_infos = []\n    for file in files:\n        link_infos += get_links_from_file(file)\n\n    errors = check_links(link_infos)\n\n    for error in errors:\n        print(error)\n\n\ndef get_docs_file_paths():\n    file_paths = []\n    for root, dirs, files in os.walk(docs_root):\n        for file in files:\n            if file.endswith(\".md\"):\n                file_paths.append(os.path.join(root, file))\n    return file_paths\n\n\n# link_info is (src_file, line_number, original_link_text, target_file, header)\ndef get_links_from_file(file):\n    link_infos = []\n    n = 1\n    with open(file) as f:\n        for line in f:\n            for link in re.findall(r\"\\]\\((.+?)\\)\", line):\n                if is_external_link(link):\n                    link_infos.append((file, n, link, None, None))\n                    continue\n                if link.startswith(\"#\"):\n                    link_infos.append((file, n, link, file, link[1:]))\n                    continue\n                if link.endswith(\".md\"):\n                    target = os.path.normpath(os.path.join(file, \"..\", link))\n                    link_infos.append((file, n, link, target, None))\n                    continue\n                if \".md#\" in link:\n                    parts = link.split(\"#\")\n                    if len(parts) == 2:\n                        target = os.path.normpath(os.path.join(file, \"..\", parts[0]))\n                        link_infos.append((file, n, link, target, parts[1]))\n                    continue\n\n                # Unexpected link format, will be handled later\n                link_infos.append((file, n, link, None, None))\n\n            n += 1\n\n    return link_infos\n\n\ndef check_links(link_infos):\n    errors = []\n    http_link_infos = []\n\n    for link_info in link_infos:\n        src_file, line_num, original_link_text, target_file, header = link_info\n        if is_external_link(original_link_text):\n            http_link_infos.append(link_info)\n            continue\n\n        if not target_file and not header:\n            errors.append(err_str(src_file, line_num, original_link_text, \"unknown link format\")),\n            continue\n\n        error = check_local_link(src_file, line_num, original_link_text, target_file, header)\n        if error:\n            errors.append(error)\n\n    # fail fast if there are local link errors\n    if len(errors) > 0:\n        return errors\n\n    if not skip_http:\n        asyncio.get_event_loop().run_until_complete(check_all_http_links(http_link_infos, errors))\n\n    return errors\n\n\nasync def check_all_http_links(http_link_infos, errors):\n    links = set()\n\n    async with aiohttp.ClientSession() as session:\n        tasks = []\n        for link_info in http_link_infos:\n            src_file, line_num, link, _, _ = link_info\n            if link in links:\n                continue\n            if \"://localhost:\" in link:\n                continue\n            links.add(link)\n            tasks.append(\n                asyncio.ensure_future(check_http_link(session, src_file, line_num, link, errors))\n            )\n        await asyncio.gather(*tasks)\n\n\nasync def check_http_link(session, src_file, line_num, link, errors):\n    num_tries = 1\n    while True:\n        try:\n            async with session.get(link, timeout=5) as resp:\n                if resp.status != 200:\n                    errors.append(\n                        err_str(src_file, line_num, link, f\"http response code {resp.status}\")\n                    )\n                return\n        except asyncio.TimeoutError:\n            if num_tries > 2:\n                errors.append(err_str(src_file, line_num, link, \"http timeout\"))\n                return\n            num_tries += 1\n\n\ndef check_local_link(src_file, line_num, original_link_text, target_file, header):\n    if not os.path.isfile(target_file):\n        return err_str(src_file, line_num, original_link_text, \"file does not exist\")\n\n    if header:\n        found_header = False\n        with open(target_file) as f:\n            for line in f:\n                if not line.startswith(\"#\"):\n                    continue\n                if header_matches(line, header):\n                    found_header = True\n                    break\n        if not found_header:\n            return err_str(src_file, line_num, original_link_text, \"header does not exist\")\n\n    return None\n\n\ndef header_matches(text, header):\n    text_words = re.findall(r\"[a-zA-Z]+\", text.lower())\n    for word in re.findall(r\"[a-zA-Z]+\", header.lower()):\n        if word not in text_words:\n            return False\n    return True\n\n\ndef err_str(src_file, line_num, original_link_text, reason):\n    clean_src_file = src_file.split(\"cortexlabs/cortex/\")[-1]\n    return f\"{clean_src_file}:{line_num}: {original_link_text} ({reason})\"\n\n\ndef is_external_link(link):\n    return link.startswith(\"http://\") or link.startswith(\"https://\")\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "dev/format.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nif ! command -v gofmt >/dev/null 2>&1; then\n  echo \"gofmt must be installed\"\n  exit 1\nfi\n\nif ! command -v black >/dev/null 2>&1; then\n  echo \"black must be installed\"\n  exit 1\nfi\n\ngofmt -s -w \"$ROOT\"/cli \"${ROOT}\"/pkg\n\nblack --quiet --line-length=100 --exclude .idea/ \"$ROOT\"\n\n# Trim trailing whitespace\nif [[ \"$OSTYPE\" == \"darwin\"* ]]; then\n  output=$(cd \"$ROOT\" && find . -type f \\\n  ! -path \"./vendor/*\" \\\n  ! -path \"./bin/*\" \\\n  ! -path \"./.git/*\" \\\n  ! -path \"./.idea/*\" \\\n  ! -name \".*\" \\\n  -print0 | \\\n  xargs -0 sed -i '' -e's/[[:space:]]*$//')\nelse\n  output=$(cd \"$ROOT\" && find . -type f \\\n  ! -path \"./vendor/*\" \\\n  ! -path \"./bin/*\" \\\n  ! -path \"./.git/*\" \\\n  ! -path \"./.idea/*\" \\\n  ! -name \".*\" \\\n  -print0 | \\\n  xargs -0 sed -i 's/[[:space:]]*$//')\nfi\n\n# Add new line to end of file\n(cd \"$ROOT\" && find . -type f \\\n! -path \"./vendor/*\" \\\n! -path \"./bin/*\" \\\n! -path \"./.git/*\" \\\n! -path \"./.idea/*\" \\\n! -name \".*\" \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(tail -c 1 \"$0\")\" && echo \"\" >> \"$0\"' || true)\n\n# Remove repeated new lines at end of file\n(cd \"$ROOT\" && find . -type f \\\n! -path \"./vendor/*\" \\\n! -path \"./bin/*\" \\\n! -path \"./.git/*\" \\\n! -path \"./.idea/*\" \\\n! -name \".*\" \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(tail -c 2 \"$0\")\" || [ ! -s \"$0\" ] || (trimmed=$(printf \"%s\" \"$(< $0)\") && echo \"$trimmed\" > \"$0\")' || true)\n\n# Remove new lines at beginning of file\n(cd \"$ROOT\" && find . -type f \\\n! -path \"./vendor/*\" \\\n! -path \"./bin/*\" \\\n! -path \"./.git/*\" \\\n! -path \"./.idea/*\" \\\n! -name \".*\" \\\n-print0 | \\\nxargs -0 -L1 bash -c 'test \"$(head -c 1 \"$0\")\" || [ ! -s \"$0\" ] || (trimmed=$(sed '\"'\"'/./,$!d'\"'\"' \"$0\") && echo \"$trimmed\" > \"$0\")' || true)\n"
  },
  {
    "path": "dev/generate_cli_md.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -e\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nout_file=$ROOT/docs/clients/cli.md\nrm -f $out_file\n\necho \"building cli ...\"\nmake --no-print-directory -C $ROOT cli\n\n# Clear default environments\ncli_config_backup_path=$HOME/.cortex/cli-bak-$RANDOM.yaml\nmv $HOME/.cortex/cli.yaml $cli_config_backup_path\n\necho \"# CLI commands\" >> $out_file\n\ncommands=(\n  \"deploy\"\n  \"get\"\n  \"describe\"\n  \"logs\"\n  \"refresh\"\n  \"delete\"\n  \"cluster up\"\n  \"cluster info\"\n  \"cluster configure\"\n  \"cluster down\"\n  \"cluster export\"\n  \"cluster health\"\n  \"env configure\"\n  \"env list\"\n  \"env default\"\n  \"env rename\"\n  \"env delete\"\n  \"version\"\n  \"completion\"\n)\n\necho \"running help commands ...\"\n\nfor cmd in \"${commands[@]}\"; do\n  echo '' >> $out_file\n  echo \"## ${cmd}\" >> $out_file\n  echo '' >> $out_file\n  echo '```text' >> $out_file\n  $ROOT/bin/cortex help ${cmd} >> $out_file\n  echo '```' >> $out_file\ndone\n\n# Bring back CLI config\nmv -f $cli_config_backup_path $HOME/.cortex/cli.yaml\n\necho \"updated $out_file\"\n"
  },
  {
    "path": "dev/generate_python_client_md.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nset -euo pipefail\n\nif [[ \"$OSTYPE\" != \"linux\"* ]]; then\n  echo \"error: this script is only designed to run on linux\"\n  exit 1\nfi\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\ndocs_path=\"$ROOT/docs/clients/python.md\"\n\npip3 uninstall -y cortex\n\ncd $ROOT/python/client\n\npip3 install -e .\n\npydoc-markdown -m cortex -m cortex.client --render-toc > $docs_path\n\n# title\nsed -i \"s/# Table of Contents/# Python client/g\" $docs_path\n\n# delete links\nsed -i \"/<a name=/d\" $docs_path\n\n# delete sentry wrapper\nsed -i \"/@sentry_wrapper/d\" $docs_path\n# delete unnecessary section headers\nsed -i \"/_](#cortex\\.client\\.Client\\.__init__)/d\" $docs_path\nsed -i \"/\\* \\[Client](#cortex\\.client\\.Client)/d\" $docs_path\n# fix section link/header\nsed -i \"s/\\* \\[cortex\\.client](#cortex\\.client)/\\* [cortex\\.client\\.Client](#cortex-client-client)/g\" $docs_path\nsed -i \"s/# cortex\\.client/# cortex\\.client\\.Client/g\" $docs_path\n# delete unnecessary section body\nsed -i \"/# cortex.client.Client/,/## deploy/{//!d}\" $docs_path\nsed -i \"s/# cortex.client.Client/# cortex.client.Client\\n/g\" $docs_path\n\n# fix table of contents links\nsed -i \"s/](#cortex\\./](#/g\" $docs_path\nsed -i \"s/](#client\\.Client\\./](#/g\" $docs_path\n\n# indentation\nsed -i \"s/    \\* /  \\* /g\" $docs_path\nsed -i \"s/#### /## /g\" $docs_path\n\n# whitespace\nsed -i 's/[[:space:]]*$//' $docs_path\ntruncate -s -1 $docs_path\n\n# Cortex version comment\nsed -i \"s/^## deploy$/## deploy\\n\\n<!-- CORTEX_VERSION_MINOR -->/g\" $docs_path\nsed -i \"s/^## deploy\\\\\\_from\\\\\\_file$/## deploy\\\\\\_from\\\\\\_file\\n\\n<!-- CORTEX_VERSION_MINOR -->/g\" $docs_path\n\npip3 uninstall -y cortex\nrm -rf $ROOT/python/client/cortex.egg-info\n"
  },
  {
    "path": "dev/get_operator_url.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport boto3\n\n\ndef main():\n    cluster_name = sys.argv[1]\n    region = sys.argv[2]\n    operator_url = get_operator_url(cluster_name, region)\n    print(\"https://\" + operator_url)\n\n\ndef get_operator_url(cluster_name, region):\n    client_elbv2 = boto3.client(\"elbv2\", region_name=region)\n\n    paginator = client_elbv2.get_paginator(\"describe_load_balancers\")\n    for load_balancer_page in paginator.paginate(PaginationConfig={\"PageSize\": 20}):\n        load_balancers = {\n            load_balancer[\"LoadBalancerArn\"]: load_balancer\n            for load_balancer in load_balancer_page[\"LoadBalancers\"]\n        }\n        tag_descriptions = client_elbv2.describe_tags(ResourceArns=list(load_balancers.keys()))[\n            \"TagDescriptions\"\n        ]\n        for tag_description in tag_descriptions:\n            foundClusterNameTag = False\n            foundLoadBalancerTag = False\n            for tags in tag_description[\"Tags\"]:\n                if tags[\"Key\"] == \"cortex.dev/cluster-name\" and tags[\"Value\"] == cluster_name:\n                    foundClusterNameTag = True\n                if tags[\"Key\"] == \"cortex.dev/load-balancer\" and tags[\"Value\"] == \"operator\":\n                    foundLoadBalancerTag = True\n            if foundClusterNameTag and foundLoadBalancerTag:\n                load_balancer = load_balancers[tag_description[\"ResourceArn\"]]\n                return load_balancer[\"DNSName\"]\n\n\n# usage: python get_operator_url.py CLUSTER_NAME REGION\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "dev/load.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/debug\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"go.uber.org/atomic\"\n)\n\n// usage: go run load.go <url> <sample.json OR sample json string>\n\n// configuration options (either set _numConcurrent > 0 or _requestInterval > 0, and configure the corresponding section)\nconst (\n\t// constant in-flight requests\n\t_numConcurrent        = 3\n\t_requestDelay         = 0 * time.Millisecond\n\t_numRequestsPerThread = 0 // 0 means loop infinitely\n\t_numMainLoops         = 1 // only relevant if _numRequestsPerThread > 0\n\n\t// constant requests per second\n\t_requestInterval        = 0 * time.Millisecond\n\t_numRequests     uint64 = 0 // 0 means loop infinitely\n\t_maxInFlight            = 5\n\n\t// other options\n\t_printSuccessDots = true\n\t_printBody        = false\n\t_printHTTPErrors  = true\n\t_printGoErrors    = true\n)\n\ntype Counter struct {\n\tsync.Mutex\n\tcount int64\n}\n\nvar (\n\t_requestCount = atomic.Uint64{}\n\t_successCount = atomic.Uint64{}\n\t_httpErrCount = atomic.Uint64{} // HTTP error response codes\n\t_goErrCount   = atomic.Uint64{} // actual errors in go (includes \"connection reset by peer\")\n)\n\nvar _client = &http.Client{\n\tTimeout: 0, // no timeout\n\tTransport: &http.Transport{\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t},\n}\n\nfunc main() {\n\tif _numConcurrent > 0 && _requestInterval > 0 {\n\t\tfmt.Println(\"error: you must set either _numConcurrent or _requestInterval > 0, but not both\")\n\t\tos.Exit(1)\n\t}\n\n\tif _numConcurrent == 0 && _requestInterval == 0 {\n\t\tfmt.Println(\"error: you must set either _numConcurrent or _requestInterval > 0\")\n\t\tos.Exit(1)\n\t}\n\n\turl, jsonPathOrString := mustExtractArgs()\n\tvar jsonBytes []byte\n\tif jsonPathOrString == \"\" {\n\t\tjsonBytes = nil\n\t} else if strings.HasPrefix(jsonPathOrString, \"{\") {\n\t\tjsonBytes = []byte(jsonPathOrString)\n\t} else {\n\t\tjsonBytes = mustReadJSONBytes(jsonPathOrString)\n\t}\n\n\tif _numConcurrent > 0 {\n\t\trunConstantInFlight(url, jsonBytes)\n\t}\n\n\tif _requestInterval > 0 {\n\t\trunConstantRequestsPerSecond(url, jsonBytes)\n\t}\n}\n\nfunc runConstantRequestsPerSecond(url string, jsonBytes []byte) {\n\tinFlightCount := Counter{}\n\tticker := time.NewTicker(_requestInterval)\n\tdone := make(chan bool)\n\n\tc := make(chan os.Signal, 1)\n\tsignal.Notify(c, os.Interrupt, syscall.SIGTERM)\n\tgo func() {\n\t\t<-c\n\t\tdone <- true\n\t}()\n\n\tstart := time.Now()\n\nFOR_LOOP:\n\tfor {\n\t\tselect {\n\t\tcase <-done:\n\t\t\tbreak FOR_LOOP\n\t\tcase <-ticker.C:\n\t\t\tgo runConstantRequestsPerSecondIteration(url, jsonBytes, &inFlightCount, done)\n\t\t}\n\t}\n\n\telapsed := time.Since(start)\n\trequestRate := float64(_requestCount.Load()) / elapsed.Seconds()\n\tfmt.Printf(\"\\nelapsed time: %s | %d requests @ %f req/s | %d succeeded | %d http errors | %d go errors\\n\", elapsed, _requestCount.Load(), requestRate, _successCount.Load(), _httpErrCount.Load(), _goErrCount.Load())\n}\n\nfunc runConstantRequestsPerSecondIteration(url string, jsonBytes []byte, inFlightCount *Counter, done chan bool) {\n\tif _maxInFlight > 0 {\n\t\tinFlightCount.Lock()\n\t\tif inFlightCount.count >= _maxInFlight {\n\t\t\tinFlightCount.Unlock()\n\t\t\tfmt.Printf(\"\\nreached max in-flight (%d)\\n\", _maxInFlight)\n\t\t\treturn\n\t\t}\n\t\tinFlightCount.count++\n\t\tinFlightCount.Unlock()\n\t}\n\n\tmakeRequest(url, jsonBytes)\n\n\tif _numRequests > 0 && _requestCount.Load() >= _numRequests {\n\t\tdone <- true\n\t}\n\n\tif _maxInFlight > 0 {\n\t\tinFlightCount.Lock()\n\t\tinFlightCount.count--\n\t\tinFlightCount.Unlock()\n\t}\n}\n\nfunc runConstantInFlight(url string, jsonBytes []byte) {\n\tif _numRequestsPerThread > 0 {\n\t\tfmt.Printf(\"spawning %d threads, %d requests each, %s delay on each\\n\", _numConcurrent, _numRequestsPerThread, _requestDelay.String())\n\t} else {\n\t\tfmt.Printf(\"spawning %d infinite threads, %s delay on each\\n\", _numConcurrent, _requestDelay.String())\n\t}\n\n\tvar summedRequestCount uint64\n\tvar summedSuccessCount uint64\n\tvar summedHTTPErrCount uint64\n\tvar summedGoErrCount uint64\n\n\tstart := time.Now()\n\tloopNum := 1\n\tfor {\n\t\twasKilled := runConstantInFlightIteration(url, jsonBytes, loopNum)\n\n\t\tsummedRequestCount += _requestCount.Load()\n\t\tsummedSuccessCount += _successCount.Load()\n\t\tsummedHTTPErrCount += _httpErrCount.Load()\n\t\tsummedGoErrCount += _goErrCount.Load()\n\n\t\t_requestCount.Store(0)\n\t\t_successCount.Store(0)\n\t\t_httpErrCount.Store(0)\n\t\t_goErrCount.Store(0)\n\n\t\tif loopNum >= _numMainLoops || wasKilled {\n\t\t\tbreak\n\t\t}\n\t\tloopNum++\n\t}\n\n\tif _numMainLoops > 1 {\n\t\telapsed := time.Since(start)\n\t\trequestRate := float64(summedRequestCount) / elapsed.Seconds()\n\t\tfmt.Printf(\"\\ntotal elapsed time: %s | %d requests @ %f req/s | %d succeeded | %d http errors | %d go errors\\n\", elapsed, summedRequestCount, requestRate, summedSuccessCount, summedHTTPErrCount, summedGoErrCount)\n\t}\n}\n\nfunc runConstantInFlightIteration(url string, jsonBytes []byte, loopNum int) bool {\n\tstart := time.Now()\n\n\twasKilled := false\n\tkilled := make(chan bool)\n\tc := make(chan os.Signal, 1)\n\tsignal.Notify(c, os.Interrupt, syscall.SIGTERM)\n\tgo func() {\n\t\t<-c\n\t\tkilled <- true\n\t}()\n\n\tdoneChans := make([]chan struct{}, _numConcurrent)\n\tfor i := range doneChans {\n\t\tdoneChans[i] = make(chan struct{})\n\t}\n\n\tfor i := range doneChans {\n\t\tdoneChan := doneChans[i]\n\n\t\tgo func() {\n\t\t\tmakeRequestLoop(url, jsonBytes)\n\t\t\tdoneChan <- struct{}{}\n\t\t}()\n\t}\n\nLOOP:\n\tfor _, doneChan := range doneChans {\n\t\tselect {\n\t\tcase <-killed:\n\t\t\twasKilled = true\n\t\t\tbreak LOOP\n\t\tcase <-doneChan:\n\t\t\tcontinue\n\t\t}\n\t}\n\n\telapsed := time.Now().Sub(start)\n\trequestRate := float64(_requestCount.Load()) / elapsed.Seconds()\n\tfmt.Printf(\"\\nelapsed time: %s | %d requests @ %f req/s | %d succeeded | %d http errors | %d go errors\\n\", elapsed, _requestCount.Load(), requestRate, _successCount.Load(), _httpErrCount.Load(), _goErrCount.Load())\n\n\treturn wasKilled\n}\n\nfunc makeRequestLoop(url string, jsonBytes []byte) {\n\tvar i int\n\tisFirstIteration := true\n\tfor true {\n\t\tif !isFirstIteration && _requestDelay != 0 {\n\t\t\ttime.Sleep(_requestDelay)\n\t\t}\n\t\tisFirstIteration = false\n\n\t\tif _numRequestsPerThread > 0 {\n\t\t\tif i >= _numRequestsPerThread {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ti++\n\t\t}\n\n\t\tmakeRequest(url, jsonBytes)\n\t}\n}\n\nfunc makeRequest(url string, jsonBytes []byte) {\n\trequest, err := http.NewRequest(\"POST\", url, bytes.NewBuffer(jsonBytes))\n\tif err != nil {\n\t\tfmt.Print(\"\\n\" + debug.Sppg(err))\n\t\treturn\n\t}\n\n\trequest.Header.Set(\"Content-Type\", \"application/json\")\n\n\t_requestCount.Inc()\n\n\tresponse, err := _client.Do(request)\n\tif err != nil {\n\t\t_goErrCount.Inc()\n\t\tif _printGoErrors {\n\t\t\tfmt.Print(\"\\n\" + debug.Sppg(err))\n\t\t}\n\t\treturn\n\t}\n\n\tbody, bodyReadErr := ioutil.ReadAll(response.Body)\n\tresponse.Body.Close()\n\n\tif response.StatusCode == 200 {\n\t\t_successCount.Inc()\n\t} else {\n\t\t_httpErrCount.Inc()\n\t\tif _printHTTPErrors {\n\t\t\tif bodyReadErr == nil {\n\t\t\t\tfmt.Printf(\"\\nstatus code: %d; body: %s\\n\", response.StatusCode, string(body))\n\t\t\t} else {\n\t\t\t\tfmt.Printf(\"\\nstatus code: %d; error reading body: %s\\n\", response.StatusCode, bodyReadErr.Error())\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\n\tif _printSuccessDots {\n\t\tfmt.Print(\".\")\n\t}\n\tif _printBody {\n\t\tbodyStr := string(body)\n\t\tif bodyStr == \"\" {\n\t\t\tbodyStr = \"(no body)\"\n\t\t}\n\t\tfmt.Print(bodyStr)\n\t}\n}\n\nfunc mustReadJSONBytes(jsonPath string) []byte {\n\tjsonBytes, err := files.ReadFileBytes(jsonPath)\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t\tos.Exit(1)\n\t}\n\n\treturn jsonBytes\n}\n\nfunc mustExtractArgs() (string, string) {\n\tif len(os.Args) != 3 {\n\t\tfmt.Println(\"usage: go run load.go <url> <sample.json OR sample json string>\")\n\t\tos.Exit(1)\n\t}\n\n\turl := os.Args[1]\n\tjsonPathOrString := os.Args[2]\n\n\treturn url, jsonPathOrString\n}\n"
  },
  {
    "path": "dev/minimum_aws_policy.json",
    "content": "{\n    \"Version\": \"2012-10-17\",\n    \"Statement\": [\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"iam:CreateServiceLinkedRole\",\n            \"Resource\": \"*\",\n            \"Condition\": {\n                \"StringEquals\": {\n                    \"iam:AWSServiceName\": [\n                        \"autoscaling.amazonaws.com\",\n                        \"ec2scheduled.amazonaws.com\",\n                        \"elasticloadbalancing.amazonaws.com\",\n                        \"spot.amazonaws.com\",\n                        \"spotfleet.amazonaws.com\",\n                        \"transitgateway.amazonaws.com\"\n                    ]\n                }\n            }\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"iam:CreateServiceLinkedRole\",\n            \"Resource\": \"*\",\n            \"Condition\": {\n                \"StringEquals\": {\n                    \"iam:AWSServiceName\": [\n                        \"eks.amazonaws.com\",\n                        \"eks-nodegroup.amazonaws.com\",\n                        \"eks-fargate.amazonaws.com\"\n                    ]\n                }\n            }\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"logs:ListTagsLogGroup\",\n                \"iam:GetRole\",\n                \"logs:TagLogGroup\",\n                \"ssm:GetParameters\",\n                \"ssm:GetParameter\",\n                \"logs:CreateLogGroup\"\n            ],\n            \"Resource\": [\n                \"arn:*:ssm:*:$CORTEX_ACCOUNT_ID:parameter/aws/*\",\n                \"arn:*:ssm:*::parameter/aws/*\",\n                \"arn:*:logs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:log-group:$CORTEX_CLUSTER_NAME\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:role/*\"\n            ]\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"iam:CreateInstanceProfile\",\n                \"logs:ListTagsLogGroup\",\n                \"logs:DescribeLogStreams\",\n                \"iam:TagRole\",\n                \"iam:GetPolicy\",\n                \"iam:CreatePolicy\",\n                \"iam:DeletePolicy\",\n                \"iam:ListPolicyVersions\",\n                \"iam:RemoveRoleFromInstanceProfile\",\n                \"iam:CreateRole\",\n                \"iam:AttachRolePolicy\",\n                \"iam:PutRolePolicy\",\n                \"iam:AddRoleToInstanceProfile\",\n                \"iam:ListInstanceProfilesForRole\",\n                \"iam:PassRole\",\n                \"logs:CreateLogStream\",\n                \"iam:DetachRolePolicy\",\n                \"logs:TagLogGroup\",\n                \"iam:ListAttachedRolePolicies\",\n                \"iam:DeleteRolePolicy\",\n                \"iam:DeleteOpenIDConnectProvider\",\n                \"iam:TagOpenIDConnectProvider\",\n                \"iam:DeleteInstanceProfile\",\n                \"iam:GetRole\",\n                \"iam:GetInstanceProfile\",\n                \"iam:DeleteRole\",\n                \"iam:ListInstanceProfiles\",\n                \"logs:CreateLogGroup\",\n                \"logs:PutLogEvents\",\n                \"logs:DeleteLogGroup\",\n                \"iam:CreateOpenIDConnectProvider\",\n                \"iam:GetOpenIDConnectProvider\",\n                \"iam:GetRolePolicy\"\n            ],\n            \"Resource\": [\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:instance-profile/eksctl-*\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-*\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:policy/eksctl-*\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:role/aws-service-role/eks-nodegroup.amazonaws.com/AWSServiceRoleForAmazonEKSNodegroup\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-managed-*\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:oidc-provider/*\",\n                \"arn:*:logs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:log-group:$CORTEX_CLUSTER_NAME:*\"\n            ]\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"iam:CreatePolicy\",\n                \"iam:GetPolicyVersion\",\n                \"iam:ListPolicyVersions\",\n                \"iam:DeletePolicy\",\n                \"iam:CreatePolicyVersion\",\n                \"iam:DeletePolicyVersion\"\n            ],\n            \"Resource\": \"arn:*:iam::$CORTEX_ACCOUNT_ID:policy/cortex-*\"\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"sqs:ListQueues\",\n                \"iam:GetPolicy\",\n                \"ecr:GetAuthorizationToken\",\n                \"cloudformation:*\",\n                \"elasticloadbalancing:*\",\n                \"autoscaling:*\",\n                \"cloudwatch:*\",\n                \"ecr:BatchGetImage\",\n                \"kms:DescribeKey\",\n                \"ec2:*\",\n                \"sts:GetCallerIdentity\",\n                \"eks:*\",\n                \"kms:CreateGrant\",\n                \"acm:DescribeCertificate\",\n                \"servicequotas:ListServiceQuotas\",\n                \"logs:PutRetentionPolicy\"\n            ],\n            \"Resource\": \"*\"\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"sqs:*\",\n            \"Resource\": \"arn:*:sqs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:cx-*\"\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"s3:*\",\n            \"Resource\": \"arn:*:s3:::$CORTEX_CLUSTER_NAME*\"\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"s3:*\",\n            \"Resource\": \"arn:*:s3:::$CORTEX_CLUSTER_NAME*/*\"\n        }\n    ]\n}\n"
  },
  {
    "path": "dev/operator_local.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -euo pipefail\n\noperator_only=\"false\"\ndebug=\"false\"\npositional_args=()\nwhile [[ $# -gt 0 ]]; do\n  key=\"$1\"\n  case $key in\n    --operator-only)\n    operator_only=\"true\"\n    shift\n    ;;\n    --debug)\n    debug=\"true\"\n    shift\n    ;;\n    *)\n    positional_args+=(\"$1\")\n    shift\n    ;;\n  esac\ndone\nset -- \"${positional_args[@]}\"\npositional_args=()\nfor i in \"$@\"; do\n  case $i in\n    *)\n    positional_args+=(\"$1\")\n    shift\n    ;;\n  esac\ndone\nset -- \"${positional_args[@]}\"\nfor arg in \"$@\"; do\n  if [[ \"$arg\" == -* ]]; then\n    echo \"unknown flag: $arg\"\n    exit 1\n  fi\ndone\n\nif [ \"$operator_only\" = \"true\" ] && [ \"$debug\" = \"true\" ]; then\n  echo \"error: --operator-only and --debug cannot both be set\"\n  exit 1\nfi\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\neval $(python3 $ROOT/manager/cluster_config_env.py \"$ROOT/dev/config/cluster.yaml\")\n\nexport CORTEX_DEV_DEFAULT_IMAGE_REGISTRY=\"$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY\"\n\npython3 $ROOT/dev/update_cli_config.py \"$HOME/.cortex/cli.yaml\" \"${CORTEX_CLUSTER_NAME}\" \"http://localhost:8888\"\n\ncp -r $ROOT/dev/config/cluster.yaml ~/.cortex/cluster-dev.yaml\n\nif grep -qiP '^telemetry:\\s*false\\s*$' ~/.cortex/cli.yaml; then\n  echo \"telemetry: false\" >> ~/.cortex/cluster-dev.yaml\nfi\n\nexport CORTEX_OPERATOR_IN_CLUSTER=false\nexport CORTEX_CLUSTER_CONFIG_PATH=~/.cortex/cluster-dev.yaml\nexport CORTEX_DISABLE_JSON_LOGGING=true\nexport CORTEX_LOG_LEVEL=debug\nexport CORTEX_PROMETHEUS_URL=\"http://localhost:9090\"\n\nportForwardCMD=\"kubectl port-forward -n default prometheus-prometheus-0 9090\"\nkill $(pgrep -f \"${portForwardCMD}\") >/dev/null 2>&1 || true\n\necho \"Port-forwarding Prometheus to localhost:9090\"\neval \"${portForwardCMD}\" >/dev/null 2>&1 &\n\nmkdir -p $ROOT/bin\n\nif [ \"$operator_only\" = \"true\" ]; then\n  kill $(pgrep -f rerun) >/dev/null 2>&1 || true\n  rerun -watch $ROOT/pkg $ROOT/dev/config $ROOT/cmd/operator -run sh -c \\\n  \"clear && echo 'building operator...' && go build -o $ROOT/bin/operator $ROOT/cmd/operator && echo 'starting local operator...' && $ROOT/bin/operator\"\nelif [ \"$debug\" = \"true\" ]; then\n  DEBUG_CMD=\"dlv --listen=:2345 --headless=true --api-version=2 debug $ROOT/cmd/operator --output ${ROOT}/bin/__debug_bin\"\n  kill $(pgrep -f \"${DEBUG_CMD}\") >/dev/null 2>&1 || true\n  kill $(pgrep -f __debug_bin) >/dev/null 2>&1 || true\n  echo 'starting local operator in debug mode...' && eval \"${DEBUG_CMD}\"\nelse\n  kill $(pgrep -f rerun) >/dev/null 2>&1 || true\n  rerun -watch $ROOT/pkg $ROOT/cli $ROOT/dev/config $ROOT/cmd/operator -run sh -c \\\n  \"clear && echo 'building cli...' && go build -o $ROOT/bin/cortex $ROOT/cli && echo 'building operator...' && go build -o $ROOT/bin/operator $ROOT/cmd/operator && echo 'starting local operator...' && $ROOT/bin/operator\"\nfi\n\n# go run -race $ROOT/cmd/operator # Check for race conditions. Doesn't seem to catch them all?\n"
  },
  {
    "path": "dev/prometheus.md",
    "content": "# Metrics\n\n## Updating metrics\n\nWhen new metrics/labels/exporters are added to be scraped by prometheus, make sure the following list **is updated** as well to keep track of what metrics/labels are needed or not.\n\nThe following is a list of metrics that are currently in use.\n\n#### Cortex metrics\n\n1. cortex_in_flight_requests with the following labels:\n    1. api_name\n1. cortex_async_request_count with the following labels:\n    1. api_name\n    1. api_kind\n    1. status_code\n1. cortex_async_active with the following labels:\n    1. api_name\n    1. api_kind\n1. cortex_async_queued with the following labels:\n    1. api_name\n    1. api_kind\n1. cortex_async_in_flight with the following labels:\n    1. api_name\n    1. api_kind\n1. cortex_async_latency_bucket with the following labels:\n    1. api_name\n    1. api_kind\n1. cortex_batch_succeeded with the following labels:\n    1. api_name\n1. cortex_batch_failed with the following labels:\n    1. api_name\n1. cortex_time_per_batch_sum with the following labels:\n    1. api_name\n1. cortex_time_per_batch_count with the following labels:\n    1. api_name\n\n#### Istio metrics\n\n1. istio_requests_total with the following labels:\n    1. destination_service\n    1. response_code\n1. istio_request_duration_milliseconds_bucket with the following labels:\n    1. destination_service\n    1. le\n1. istio_request_duration_milliseconds_sum with the following labels:\n    1. destination_service\n1. istio_request_duration_milliseconds_count with the following labels:\n    1. destination_service\n\n#### Kubelet metrics\n1. container_cpu_usage_seconds_total with the following labels:\n    1. pod\n    1. container\n    1. name\n1. container_memory_working_set_bytes with the following labels:\n    1. pod\n    1. name\n    1. container\n\n#### Kube-state-metrics metrics\n\n1. kube_pod_container_resource_requests with the following labels:\n    1. exported_pod\n    1. resource\n    1. exported_container (required for not dropping the values for each container of each pod)\n1. kube_pod_info with the following labels:\n    1. exported_pod\n1. kube_deployment_status_replicas_available with the following labels:\n    1. deployment\n1. kube_job_status_active with the following labels:\n    1. job_name\n\n#### DCGM metrics\n\n1. DCGM_FI_DEV_GPU_UTIL with the following labels:\n    1. exported_pod\n1. DCGM_FI_DEV_FB_USED with the following labels:\n    1. exported_pod\n1. DCGM_FI_DEV_FB_FREE with the following labels:\n    1. exported_pod\n\n#### Node metrics\n\n1. node_cpu_seconds_total with the following labels:\n    1. job\n    1. mode\n    1. instance\n    1. cpu\n1. node_load1 with the following labels:\n    1. job\n    1. instance\n1. node_load5 with the following labels:\n    1. job\n    1. instance\n1. node_load15 with the following labels:\n    1. job\n    1. instance\n1. node_exporter_build_info with the following labels:\n    1. job\n    1. instance\n1. node_memory_MemTotal_bytes with the following labels:\n    1. job\n    1. instance\n1. node_memory_MemFree_bytes with the following labels:\n    1. job\n    1. instance\n1. node_memory_Buffers_bytes with the following labels:\n    1. job\n    1. instance\n1. node_memory_Cached_bytes with the following labels:\n    1. job\n    1. instance\n1. node_memory_MemAvailable_bytes with the following labels:\n    1. job\n    1. instance\n1. node_disk_read_bytes_total with the following labels:\n    1. job\n    1. instance\n    1. device\n1. node_disk_written_bytes_total with the following labels:\n    1. job\n    1. instance\n    1. device\n1. node_disk_io_time_seconds_total with the following labels:\n    1. job\n    1. instance\n    1. device\n1. node_filesystem_size_bytes with the following labels:\n    1. job\n    1. instance\n    1. fstype\n    1. mountpoint\n    1. device\n1. node_filesystem_avail_bytes with the following labels:\n    1. job\n    1. instance\n    1. fstype\n    1. device\n1. node_network_receive_bytes_total with the following labels:\n    1. job\n    1. instance\n    1. device\n1. node_network_transmit_bytes_total with the following labels:\n    1. job\n    1. instance\n    1. device\n\n##### Prometheus rules for the node exporter\n\n1. instance:node_cpu_utilisation:rate1m from the following metrics:\n    1. node_cpu_seconds_total with the following labels:\n        1. job\n        1. mode\n1. instance:node_num_cpu:sum from the following metrics:\n    1. node_cpu_seconds_total with the following labels:\n        1. job\n1. instance:node_load1_per_cpu:ratio from the following metrics:\n    1. node_load1 with the following labels:\n        1. job\n1. instance:node_memory_utilisation:ratio from the following metrics:\n    1. node_memory_MemTotal_bytes with the following labels:\n        1. job\n    1. node_memory_MemAvailable_bytes with the following labels:\n        1. job\n1. instance:node_vmstat_pgmajfault:rate1m with the following metrics:\n    1. node_vmstat_pgmajfault with the following labels:\n        1. job\n1. instance_device:node_disk_io_time_seconds:rate1m with the following metrics:\n    1. node_disk_io_time_seconds_total with the following labels:\n        1. job\n        1. device\n1. instance_device:node_disk_io_time_weighted_seconds:rate1m with the following metrics:\n    1. node_disk_io_time_weighted_seconds with the following labels:\n        1. job\n        1. device\n1. instance:node_network_receive_bytes_excluding_lo:rate1m with the following metrics:\n    1. node_network_receive_bytes_total with the following labels:\n        1. job\n        1. device\n1. instance:node_network_transmit_bytes_excluding_lo:rate1m with the following metrics:\n    1. node_network_transmit_bytes_total with the following labels:\n        1. job\n        1. device\n1. instance:node_network_receive_drop_excluding_lo:rate1m with the following metrics:\n    1. node_network_receive_drop_total with the following labels:\n        1. job\n        1. device\n1. instance:node_network_transmit_drop_excluding_lo:rate1m with the following metrics:\n    1. node_network_transmit_drop_total with the following labels:\n        1. job\n        1. device\n1. cluster:cpu_utilization:ratio with the following metrics:\n    1. instance:node_cpu_utilisation:rate1m\n    1. instance:node_num_cpu:sum\n1. cluster:load1:ratio with the following metrics:\n    1. instance:node_load1_per_cpu:ratio\n1. cluster:memory_utilization:ratio with the following metrics:\n    1. instance:node_memory_utilisation:ratio\n1. cluster:vmstat_pgmajfault:rate1m with the following metrics:\n    1. instance:node_vmstat_pgmajfault:rate1m\n1. cluster:network_receive_bytes_excluding_low:rate1m with the following metrics:\n    1. instance:node_network_receive_bytes_excluding_lo:rate1m\n1. cluster:network_transmit_bytes_excluding_lo:rate1m with the following metrics:\n    1. instance:node_network_transmit_bytes_excluding_lo:rate1m\n1. cluster:network_receive_drop_excluding_lo:rate1m with the following metrics:\n    1. instance:node_network_receive_drop_excluding_lo:rate1m\n1. cluster:network_transmit_drop_excluding_lo:rate1m with the following metrics:\n    1. instance:node_network_transmit_drop_excluding_lo:rate1m\n1. cluster:disk_io_utilization:ratio with the following metrics:\n    1. instance_device:node_disk_io_time_seconds:rate1m\n1. cluster:disk_io_saturation:ratio with the following metrics:\n    1. instance_device:node_disk_io_time_weighted_seconds:rate1m\n1. cluster:disk_space_utilization:ratio with the following metrics:\n    1. node_filesystem_size_bytes with the following labels:\n        1. job\n        1. fstype\n        1. mountpoint\n    1. node_filesystem_avail_bytes with the following labels:\n        1. job\n        1. fstype\n        1. mountpoint\n\n## Re-introducing dropped metrics/labels\n\nIf you need to add some metrics/labels back for some particular use case, comment out every `metricRelabelings:` section (except the one from the `prometheus-operator.yaml` file), determine which metrics/labels you want to add back (i.e. by using the explorer from Grafana) and then re-edit the appropriate `metricRelabelings:` sections to account for the un-dropped metrics/labels.\n\n## Prometheus Analysis\n\n### Go Pprof\n\nTo analyse the memory allocations of prometheus, run `kubectl port-forward prometheus-prometheus-0 9090:9090`, and then run `go tool pprof -symbolize=remote -inuse_space localhost:9090/debug/pprof/heap`. Once you get the interpreter, you can run `top` or `dot` for a more detailed hierarchy of the memory usage.\n\n### TSDB\n\nTo analyse the TSDB of prometheus, exec into the `prometheus-prometheus-0` pod, `cd` into `/tmp`, and run the following code-block:\n\n```bash\nwget https://github.com/prometheus/prometheus/releases/download/v1.7.3/prometheus-1.7.3.linux-amd64.tar.gz\ntar -xzf prometheus-*\ncd prometheus-*\n./tsdb analyze /prometheus | less\n```\n\n*Useful link: https://www.robustperception.io/using-tsdb-analyze-to-investigate-churn-and-cardinality*\n\nOr you can go to `localhost:9090` -> `Status` -> `TSDB Status`, but it's not as complete as running a binary analysis.\n"
  },
  {
    "path": "dev/registry.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nCORTEX_VERSION=master\n\nset -eo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/.. >/dev/null && pwd)\"\n\nsource $ROOT/build/images.sh\nsource $ROOT/dev/util.sh\n\nimages_that_can_run_locally=\"operator manager\"\n\nif [ -f \"$ROOT/dev/config/env.sh\" ]; then\n  source $ROOT/dev/config/env.sh\nfi\n\nAWS_ACCOUNT_ID=${AWS_ACCOUNT_ID:-}\nAWS_REGION=${AWS_REGION:-}\n\nskip_push=\"false\"\ninclude_arm64_arch=\"false\"\npositional_args=()\nwhile [[ $# -gt 0 ]]; do\n  key=\"$1\"\n  case $key in\n    --skip-push)\n    skip_push=\"true\"\n    shift\n    ;;\n    --include-arm64-arch)\n    include_arm64_arch=\"true\"\n    shift\n    ;;\n    *)\n    positional_args+=(\"$1\")\n    shift\n    ;;\n  esac\ndone\nset -- \"${positional_args[@]}\"\npositional_args=()\nfor i in \"$@\"; do\n  case $i in\n    *)\n    positional_args+=(\"$1\")\n    shift\n    ;;\n  esac\ndone\nset -- \"${positional_args[@]}\"\nfor arg in \"$@\"; do\n  if [[ \"$arg\" == -* ]]; then\n    echo \"unknown flag: $arg\"\n    exit 1\n  fi\ndone\n\ncmd=${1:-\"\"}\nsub_cmd=${2:-\"\"}\n\nregistry_push_url=\"\"\nif [ \"$skip_push\" != \"true\" ]; then\n  registry_push_url=\"$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com\"\nfi\n\nis_registry_logged_in=\"false\"\n\nfunction registry_login() {\n  if [ \"$is_registry_logged_in\" = \"false\" ]; then\n    blue_echo \"Logging in to ECR...\"\n    aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $registry_push_url\n    is_registry_logged_in=\"true\"\n    green_echo \"Success\\n\"\n  fi\n}\n\nfunction create_ecr_repository() {\n  for image in \"${all_images[@]}\"; do\n    aws ecr create-repository --repository-name=cortexlabs/$image --region=$AWS_REGION || true\n  done\n}\n\n### HELPERS ###\n\nfunction build_and_push() {\n  local image=$1\n  local include_arm64_arch=$2\n  local dir=\"${ROOT}/images/${image}\"\n\n  set -euo pipefail\n\n  if [[ ! \" ${multi_arch_images[*]} \" =~ \" $image \" ]]; then\n    include_arm64_arch=\"false\"\n  fi\n\n  if [ ! -n \"$AWS_ACCOUNT_ID\" ] || [ ! -n \"$AWS_REGION\" ]; then\n    echo \"AWS_ACCOUNT_ID or AWS_REGION env vars not found\"\n    exit 1\n  fi\n\n  tag=$CORTEX_VERSION\n\n  push_or_not_flag=\"\"\n  running_operation=\"Building\"\n  finished_operation=\"Built\"\n  if [ \"$skip_push\" = \"false\" ]; then\n    push_or_not_flag=\"--push\"\n    running_operation+=\" and pushing\"\n    finished_operation+=\" and pushed\"\n    registry_login\n  fi\n\n  if [ \"$include_arm64_arch\" = \"true\" ]; then\n    blue_echo \"$running_operation $image:$tag (amd64 and arm64)...\"\n  else\n    blue_echo \"$running_operation $image:$tag (amd64)...\"\n  fi\n\n  platforms=\"linux/amd64\"\n  if [ \"$include_arm64_arch\" = \"true\" ]; then\n    platforms+=\",linux/arm64\"\n  fi\n\n  docker buildx build $ROOT -f $dir/Dockerfile -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag --platform $platforms $push_or_not_flag\n\n  if [ \"$include_arm64_arch\" = \"true\" ]; then\n    green_echo \"$finished_operation $image:$tag (amd64 and arm64)\"\n  else\n    green_echo \"$finished_operation $image:$tag (amd64)\"\n  fi\n\n  if [[ \" ${images_that_can_run_locally[*]} \" =~ \" $image \" ]] && [[ \"$include_arm64_arch\" == \"false\" ]]; then\n    blue_echo \"Exporting $image:$tag to local docker...\"\n    docker buildx build $ROOT -f $dir/Dockerfile -t cortexlabs/$image:$tag -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/cortexlabs/$image:$tag --platform $platforms --load\n    green_echo \"Exported $image:$tag to local docker\"\n  fi\n}\n\nfunction cleanup_local() {\n  echo \"cleaning local repositories...\"\n  docker container prune -f\n  docker image prune -f\n  docker buildx prune -f\n}\n\nfunction cleanup_ecr() {\n  echo \"cleaning ECR repositories...\"\n  repos=$(aws ecr describe-repositories --output text | awk '{print $6}' | grep -P \"\\S\")\n  echo \"$repos\" |\n  while IFS= read -r repo; do\n    imageIDs=$(aws ecr list-images --repository-name \"$repo\" --filter tagStatus=UNTAGGED --query \"imageIds[*]\" --output text)\n    echo \"$imageIDs\" |\n    while IFS= read -r imageId; do\n      if [ ! -z \"$imageId\" ]; then\n        echo \"Removing from ECR: $repo/$imageId\"\n        aws ecr batch-delete-image --repository-name \"$repo\" --image-ids imageDigest=\"$imageId\" >/dev/null;\n      fi\n    done\n  done\n}\n\nfunction delete_ecr() {\n  echo \"deleting ECR repositories...\"\n  repos=$(aws ecr describe-repositories --output text | awk '{print $6}' | grep -P \"\\S\")\n  echo \"$repos\" |\n  while IFS= read -r repo; do\n    imageIDs=$(aws ecr delete-repository --force --repository-name \"$repo\")\n    echo \"deleted: $repo\"\n  done\n}\n\nfunction validate_env() {\n  if [ \"$skip_push\" != \"true\" ]; then\n    if [ -z ${AWS_REGION} ] || [ -z ${AWS_ACCOUNT_ID} ]; then\n      echo \"error: environment variables AWS_REGION and AWS_ACCOUNT_ID should be exported in dev/config/env.sh\"\n      exit 1\n    fi\n  fi\n}\n\n# validate environment is correctly set on env.sh\nvalidate_env\n\n# usage: registry.sh clean\nif [ \"$cmd\" = \"clean\" ]; then\n  delete_ecr\n  create_ecr_repository\n\n# usage: registry.sh create\nelif [ \"$cmd\" = \"create\" ]; then\n  create_ecr_repository\n\n# usage: registry.sh update-single IMAGE\nelif [ \"$cmd\" = \"update-single\" ]; then\n  image=$sub_cmd\n  build_and_push $image $include_arm64_arch\n\n# usage: registry.sh update all|dev|api\nelif [ \"$cmd\" = \"update\" ]; then\n  images_to_build=()\n\n  if [ \"$sub_cmd\" == \"all\" ]; then\n    images_to_build+=( \"${non_dev_images[@]}\" )\n  fi\n\n  if [[ \"$sub_cmd\" == \"all\" || \"$sub_cmd\" == \"dev\" ]]; then\n    images_to_build+=( \"${dev_images[@]}\" )\n  fi\n\n  for image in \"${images_to_build[@]}\"; do\n    build_and_push $image $include_arm64_arch\n  done\n\n# usage: registry.sh clean-cache\nelif [ \"$cmd\" = \"clean-cache\" ]; then\n  cleanup_local\n\nelse\n  echo \"unknown command: $cmd\"\n  exit 1\nfi\n"
  },
  {
    "path": "dev/update_cli_config.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport yaml\nimport os\n\n\ndef update_cli_config(cli_config_file_path, env_name, operator_endpoint):\n    new_env = {\n        \"name\": env_name,\n        \"operator_endpoint\": operator_endpoint,\n    }\n\n    try:\n        with open(cli_config_file_path, \"r\") as f:\n            cli_config = yaml.safe_load(f)\n            if cli_config is None:\n                raise Exception(\"blank cli config file\")\n    except:\n        cli_config = {\"environments\": [new_env]}\n        with open(cli_config_file_path, \"w\") as f:\n            yaml.dump(cli_config, f, default_flow_style=False)\n        return\n\n    if len(cli_config.get(\"environments\", [])) == 0:\n        cli_config[\"environments\"] = [new_env]\n        with open(cli_config_file_path, \"w\") as f:\n            yaml.dump(cli_config, f, default_flow_style=False)\n        return\n\n    replaced = False\n    for i, prev_env in enumerate(cli_config[\"environments\"]):\n        if prev_env.get(\"name\") == env_name:\n            cli_config[\"environments\"][i] = new_env\n            replaced = True\n            break\n\n    if not replaced:\n        cli_config[\"environments\"].append(new_env)\n\n    with open(cli_config_file_path, \"w\") as f:\n        yaml.dump(cli_config, f, default_flow_style=False)\n\n\nif __name__ == \"__main__\":\n    update_cli_config(\n        cli_config_file_path=sys.argv[1],\n        env_name=sys.argv[2],\n        operator_endpoint=sys.argv[3],\n    )\n"
  },
  {
    "path": "dev/util.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nfunction blue_echo() {\n  echo -e \"\\033[1;34m$1\\033[0m\"\n}\n\nfunction green_echo() {\n  echo -e \"\\033[1;32m$1\\033[0m\"\n}\n\nfunction error_echo() {\n  echo -e \"\\033[1;31mERROR: \\033[0m$1\"\n}\n"
  },
  {
    "path": "dev/versions.md",
    "content": "# Upgrade notes\n\n## eksctl\n\n1. Find the latest release on [GitHub](https://github.com/weaveworks/eksctl/releases) and check the changelog\n1. Search the code base for the old version to find where to update it (e.g. `manager/Dockerfile`)\n1. Update `generate_eks.py` if necessary\n1. Check that `eksctl utils write-kubeconfig` log filter still behaves as desired, and logs in `cortex cluster up` look good.\n1. Update eksctl on your dev\n   machine: `curl --location \"https://github.com/weaveworks/eksctl/releases/download/v0.107.0/eksctl_$(uname -s)_amd64.tar.gz\" | tar xz -C /tmp && sudo mv -f /tmp/eksctl /usr/local/bin`\n1. Check if eksctl iam polices changed by comparing the previous version of the eksctl policy docs to the new version's and update `./dev/minimum_aws_policy.json` and `docs/clusters/management/auth.md` accordingly. https://github.com/weaveworks/eksctl/blob/v0.107.0/userdocs/src/usage/minimum-iam-policies.md\n\n## Kubernetes\n\n1. Find the latest version of Kubernetes supported by\n   eksctl ([source code](https://github.com/weaveworks/eksctl/blob/master/pkg/apis/eksctl.io/v1alpha5/types.go))\n1. Update the version in `generate_eks.py`\n1. Update `ami.json` (see release checklist for instructions)\n1. See instructions for upgrading the Kubernetes client below\n\n## kube-proxy (IPVS mode)\n\n1. Before spinning up a Cortex cluster with the new eksctl/kubernetes/eks updates, make sure to have the `setup_ipvs` functional call commented out in the manager.\n1. Once the cluster is up, run the `cat /var/lib/kube-proxy-config/config` command on any of the kube-proxy pods of the cluster. Compare the output of that with what the `upgrade_kube_proxy_mode.py` script is applying and make sure it's still applicable, if not, check out the spec of the [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/) and upgrade `upgrade_kube_proxy_mode.py`.\n1. Compare the spec of the `kube-proxy.patch.yaml` patch with the current spec of the kube-proxy daemoset and make sure it's still applicable. You can either inspect the `kube-proxy` command helper by exec-ing into the pod or by looking at the [kube-proxy](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) documentation for the respective version of Kubernetes.\n1. Once both config map and the daemonset are updated and the kube-proxy pod(s) has/have started, make sure you notice the `Using ipvs Proxier` log.\n\n## aws-iam-authenticator\n\n1. Find the latest release [here](https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html)\n1. Update the version in `images/manager/Dockerfile`\n\n## kubectl\n\n1. Find the latest release [here](https://storage.googleapis.com/kubernetes-release/release/stable.txt)\n1. Update the version in `images/manager/Dockerfile` and `images/operator/Dockerfile`\n1. Update your local version and alert developers\n    * Linux:\n\n     ```shell\n        mkdir -p $HOME/temp && \\\n        cd $HOME/temp && \\\n        curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \\\n        chmod +x ./kubectl && \\\n        sudo mv -f ./kubectl /usr/local/bin/kubectl && \\\n        if [ -f $HOME/.bash_profile ]; then source $HOME/.bash_profile; else source $HOME/.bashrc; fi && \\\n        cd - && \\\n        kubectl version\n     ```\n\n    * Mac:\n        1. `brew upgrade kubernetes-cli`\n        1. refresh shell\n        1. `kubectl version`\n\n## Istio\n\n1. Find the latest [release](https://istio.io/latest/news/releases) and check the release notes (here are\n   the [latest IstioOperator Options](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/))\n1. Update the version in `images/manager/Dockerfile`.\n1. Update the version in all `images/istio-*` Dockerfiles.\n1. Update `istio.yaml.j2`, `apis.yaml.j2`, `operator.yaml.j2`, and `pkg/lib/k8s` as necessary.\n1. Update `install.sh` as necessary.\n\n## AWS CNI\n\n1. Update the CNI version in `generate_eks.py` ([CNI releases](https://github.com/aws/amazon-vpc-cni-k8s/releases))\n1. Update the go module version (see `Go > Non-versioned modules` section below)\n1. Check if new instance types were added by running the script below (update the two env vars at the top).\n   1. If there are new instance types, check if any changes need to be made to `servicequotas.go` or `validateInstanceType()`.\n\n```bash\nPREV_RELEASE=1.10.1\nNEW_RELEASE=1.11.0\nwget -q -O cni_supported_instances_prev.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${PREV_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; wget -q -O cni_supported_instances_new.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${NEW_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; git diff --no-index cni_supported_instances_prev.txt cni_supported_instances_new.txt; rm -rf cni_supported_instances_prev.txt; rm -rf cni_supported_instances_new.txt\n```\n\n## Go\n\n1. Find the latest release on Golang's [release page](https://golang.org/doc/devel/release.html) (\n   or [downloads page](https://golang.org/dl/)) and check the changelog\n1. Search the codebase for the current minor version (e.g. `1.17`), update versions as appropriate\n1. Update your local version and alert developers:\n    * Linux:\n\n     ```shell\n        mkdir -p $HOME/temp\n        cd $HOME/temp\n        wget https://dl.google.com/go/go1.17.3.linux-amd64.tar.gz && \\\n        tar -xvf go1.17.3.linux-amd64.tar.gz && \\\n        sudo rm -rf /usr/local/go && \\\n        sudo mv -f go /usr/local && \\\n        rm go1.17.3.linux-amd64.tar.gz && \\\n        if [ -f $HOME/.bash_profile ]; then source $HOME/.bash_profile; else source $HOME/.bashrc; fi && \\\n        cd - && \\\n        go version\n     ```\n\n    * Mac:\n        1. `brew upgrade go` or `brew install go@1.17`\n        1. refresh shell\n        1. `go version`\n1. Update go modules as necessary\n\n## Go modules\n\n### Kubernetes client\n\n1. Find the latest patch [release](https://github.com/kubernetes/client-go) for the minor kubernetes version that we use (e.g. for k8s 1.21, use `client-go` version `v0.21.X`, where `X` is the latest available patch release)\n1. Follow the \"Update non-versioned modules\" instructions using the updated version for `k8s.io/client-go`\n\n### Istio client\n\n1. Find the version of istio that we use in `images/manager/Dockerfile`\n1. Follow the \"Update non-versioned modules\" instructions using the updated version for `istio.io/client-go`\n\n### docker/engine/client\n\n1. Find the latest tag from [here](https://github.com/docker/engine/tags)\n1. Follow the \"Update non-versioned modules\" instructions using the updated version for `docker/engine`\n\n_note: docker client installation may be able to be improved,\nsee https://github.com/moby/moby/issues/39302#issuecomment-639687466_\n\n### cortexlabs/yaml\n\n1. Check [go-yaml/yaml](https://github.com/go-yaml/yaml/commits/v2) to see if there were new releases\n   since [cortexlabs/yaml](https://github.com/cortexlabs/yaml/commits/v2)\n1. `git clone git@github.com:cortexlabs/yaml.git && cd yaml`\n1. `git remote add upstream https://github.com/go-yaml/yaml && git fetch upstream`\n1. `git merge upstream/v2`\n1. `git push origin v2`\n1. Follow the \"Update non-versioned modules\" instructions using the desired commit sha for `cortexlabs/yaml`\n\n### cortexlabs/go-input\n\n1. Check [tcnksm/go-input](https://github.com/tcnksm/go-input/commits/master) to see if there were new releases\n   since [cortexlabs/go-input](https://github.com/cortexlabs/go-input/commits/master)\n1. `git clone git@github.com:cortexlabs/go-input.git && cd go-input`\n1. `git remote add upstream https://github.com/tcnksm/go-input && git fetch upstream`\n1. `git merge upstream/master`\n1. `git push origin master`\n1. Follow the \"Update non-versioned modules\" instructions using the desired commit sha for `cortexlabs/go-input`\n\n### Non-versioned modules\n\n1. `rm -rf go.mod go.sum && go mod init && go clean -modcache`\n1. `go get k8s.io/client-go@v0.20.15 && go get k8s.io/apimachinery@v0.20.15 && go get k8s.io/api@v0.20.15`\n1. `go get istio.io/client-go@v1.11.8 && go get istio.io/api@1.11.8`\n1. `go get github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils@v1.11.0`\n1. `go get github.com/cortexlabs/yaml@31e52ba8433b683c471ef92cf1711fe67671dac5`\n1. `go get github.com/cortexlabs/go-input@8b67a7a7b28d1c45f5c588171b3b50148462b247`\n1. `go get github.com/xlab/treeprint@v1.0.0`\n1. `go get -u sigs.k8s.io/controller-runtime@v0.8.3`\n1. `echo -e '\\nreplace github.com/docker/docker => github.com/docker/engine v19.03.13' >> go.mod`\n1. `go get -u github.com/docker/distribution`\n1. `go mod tidy`\n1. Potentially skip these steps\n   1. For every non-indirect, non-hardcoded dependency in go.mod, update with `go get -u <path>`\n   1. `go mod tidy`\n   1. Re-run the relevant hardcoded `go get` commands above\n   1. `go mod tidy`\n   1. `make test`\n   1. `go mod tidy`\n1. Check that the diff in `go.mod` is reasonable\n\n## Nvidia device plugin\n\n1. Update the version in `images/nvidia-device-plugin/Dockerfile` ([releases](https://github.com/NVIDIA/k8s-device-plugin/releases)\n   , [Dockerhub](https://hub.docker.com/r/nvidia/k8s-device-plugin))\n1. In the [GitHub Repo](https://github.com/NVIDIA/k8s-device-plugin), find the latest release and go to this file (\n   replacing the version number): <https://github.com/NVIDIA/k8s-device-plugin/blob/v0.6.0/nvidia-device-plugin.yml>\n1. Copy the contents to `manager/manifests/nvidia.yaml`\n    1. Update the link at the top of the file to the URL you copied from\n    1. Check that your diff is reasonable (and put back any of our modifications, e.g. the image path, rolling update\n       strategy, resource requests, tolerations, node selector, priority class, etc)\n\n## Neuron device plugin and scheduler\n\n1. Update `images/neuron-device-plugin/Dockerfile` if necessary (see [here](https://gallery.ecr.aws/neuron/neuron-device-plugin) for the latest tag)\n1. Update `images/neuron-scheduler/Dockerfile` if necessary (see [here](https://gallery.ecr.aws/neuron/neuron-scheduler) for the latest tag)\n1. Copy the contents of `k8s-neuron-*` in [this folder](https://github.com/aws/aws-neuron-sdk/tree/master/src/k8) into `manager/manifests/inferentia.yaml`\n    1. Update the link at the top of the file to the URL you copied from\n    1. Check that your diff is reasonable (and put back any of our modifications)\n\n## Metrics server\n\n1. Find the latest release on [GitHub](https://github.com/kubernetes-incubator/metrics-server/releases) and check the\n   changelog\n1. Update the version in `images/metrics-server/Dockerfile`\n1. Download the manifest referenced in the latest release in changelog\n1. Copy the contents of the manifest into `manager/manifests/metrics-server.yaml`\n    1. Update accordingly (e.g. image, pull policy, resource request, etc):\n    1. Check that your diff is reasonable\n1. You can confirm the metric server is running by showing the logs of the metrics-server pod, or\n   via `kubectl get deployment metrics-server -n kube-system`\n   and `kubectl get apiservice v1beta1.metrics.k8s.io -o yaml`\n\n## Cluster autoscaler\n\n1. Find the latest patch release for our current version of k8s (e.g. k8s v1.17 -> cluster-autocluster v1.17.3)\n   on [GitHub](https://github.com/kubernetes/autoscaler/releases) and check the changelog\n1. In the [GitHub Repo](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws),\n   set the tree to the tag for the chosen release, and open `cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml`\n   (e.g. <https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.20.0/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml>)\n1. Resolve merge conflicts with the template in `manager/manifests/cluster-autoscaler.yaml.j2`.\n1. Clone our fork: `git clone git@github.com:cortexlabs/autoscaler.git`\n1. Checkout our updated branch: `git checkout cluster-autoscaler-1.21.1-cortex`\n1. List the most recent commit: `git log`\n1. Reset the latest commit (use the SHA of the last non-cortex commit): `git reset <SHA>`\n1. `git add *`\n1. `git stash`\n1. `git remote add upstream https://github.com/kubernetes/autoscaler.git`\n1. `git fetch upstream`\n1. Checkout the appropriate version tag, e.g. `git checkout cluster-autoscaler-1.22.2 -b cluster-autoscaler-1.22.2-cortex`\n1. `git stash pop`\n1. Resolve any merge conflicts\n1. Unstage and check the diff\n1. `git add *; git commit -am \"Add rate limiter\"`\n1. `git push origin cluster-autoscaler-1.22.2-cortex`\n1. Update `images/cluster-autoscaler/Dockerfile` to use the new branch name (e.g. \"cluster-autoscaler-1.22.2\") in the `-b` flag's value from `git clone`.\n1. Match the Go version of the builder in `images/cluster-autoscaler/Dockerfile` with that of the [cluster autoscaler's Dockerfile](https://github.com/kubernetes/autoscaler/blob/master/builder/Dockerfile).\n\n## FluentBit\n\n1. Find the latest release\n   on [Docker Hub](https://hub.docker.com/r/amazon/aws-for-fluent-bit/tags?page=1&ordering=last_updated)\n1. Update the base image version in `images/fluent-bit/Dockerfile`\n1. Update `fluent-bit.yaml` as necessary (make sure to maintain all Cortex environment variables)\n\n## Prometheus Operator / Prometheus Config Reloader\n\n1. Find the latest release in the [GitHub Repo](https://github.com/prometheus-operator/prometheus-operator).\n1. Copy the `bundle.yaml` file contents into `prometheus-operator.yaml`.\n1. Replace the image in the Deployment resource with a cortex env var.\n1. Update the base image versions in `images/prometheus-operator/Dockerfile`\n   and `images/prometheus-config-reloader/Dockerfile`.\n\n## Prometheus\n\n1. Find the latest release on [Docker Hub](https://hub.docker.com/r/prom/prometheus/tags?page=1&ordering=last_updated),\n   compatible to the current version of Prometheus Operator.\n1. Update the base image version in `images/prometheus/Dockerfile`.\n1. Update `prometheus-monitoring.yaml` as necessary, if that's the case.\n\n## Prometheus StatsD Exporter\n\n1. Find the latest release\n   on [Docker Hub](https://registry.hub.docker.com/r/prom/statsd-exporter/tags?page=1&ordering=last_updated).\n1. Update the base image version in `images/prometheus-statsd-exporter/Dockerfile`.\n1. Update `prometheus-statsd-exporter.yaml` as necessary, if that's the case.\n\n## Prometheus DCGM Exporter\n\n1. Run `helm template` on the DCGM charts https://github.com/NVIDIA/gpu-monitoring-tools/tree/master/deployment/dcgm-exporter and save the output somewhere temporarily.\n1. Update the base image version in `images/prometheus-dcgm-exporter/Dockerfile`.\n1. Update `prometheus-dcgm-exporter.yaml` as necessary, if that's the case. Keep in mind that in our k8s template, the `ServiceMonitor` was changed to a `PodMonitor`. Remove any unnecessary labels.\n\n## Prometheus kube-state-metrics Exporter\n\n1. Run `helm template` on the kube-state-metrics charts from https://github.com/kubernetes/kube-state-metrics#helm-chart and save the output somewhere temporarily.\n1. Update the base image version in `images/prometheus-kube-state-metrics/Dockerfile`.\n1. Update `prometheus-kube-state-metrics.yaml` as necessary, if that's the case. Keep in mind that in our k8s template, the `ServiceMonitor` was changed to a `PodMonitor`. Remove any unnecessary labels. The update can also include adjusting the resource requests.\n\n## Prometheus Kubelet Exporter\n\n1. Check if https://github.com/prometheus-operator/kube-prometheus/blob/main/manifests/kubernetes-serviceMonitorKubelet.yaml has changed when compared to `manager/manifests/prometheus-kubelet-exporter`.\n\n## Prometheus Node Exporter\n\n1. Find the latest release in the Kube Prometheus [GitHub Repo](https://github.com/prometheus-operator/kube-prometheus/blob/main/manifests/).\n1. Copy the `node-exporter-*.yaml` files contents into `prometheus-node-exporter.yaml`, but keep the prometheus rules resource.\n1. Replace the image in the Deployment resource with a cortex env var.\n1. Update the base image version in `images/prometheus-node-exporter/Dockerfile`\n1. Update the base branch version in `images/kube-rbac-proxy/Dockerfile` (as well as the rest of the contents if necessary).\n\n## Grafana\n\n1. Find the latest release\n   on [Docker Hub](https://registry.hub.docker.com/r/grafana/grafana/tags?page=1&ordering=last_updated).\n1. Update the base image version in `images/grafana/Dockerfile`.\n1. Update `grafana.yaml` as necessary, if that's the case.\n\n## Event Exporter\n\n1. Find the latest release on [GitHub](https://github.com/opsgenie/kubernetes-event-exporter) / [GitHub Container Registry](https://github.com/opsgenie/kubernetes-event-exporter/pkgs/container/kubernetes-event-exporter).\n1. Update the base image version in `images/event-exporter/Dockerfile`.\n1. Update `event-exporter.yaml` as necessary, if that's the case.\n\n## Alpine base images\n\n1. Find the latest release on [Dockerhub](https://hub.docker.com/_/alpine)\n1. Search the codebase for `alpine` and update accordingly\n\n## Python client dependencies\n\n1. Update package versions in `install_requires` in `python/client/setup.py`.\n"
  },
  {
    "path": "docs/README.md",
    "content": "**Please read our documentation at [docs.cortexlabs.com](https://docs.cortexlabs.com)**\n"
  },
  {
    "path": "docs/clients/cli.md",
    "content": "# CLI commands\n\n## deploy\n\n```text\ncreate or update apis\n\nUsage:\n  cortex deploy [CONFIG_FILE] [flags]\n\nFlags:\n  -e, --env string      environment to use\n  -f, --force           override the in-progress api update\n  -y, --yes             skip prompts\n  -o, --output string   output format: one of pretty|json (default \"pretty\")\n  -h, --help            help for deploy\n```\n\n## get\n\n```text\nget information about apis or jobs\n\nUsage:\n  cortex get [API_NAME] [JOB_ID] [flags]\n\nFlags:\n  -e, --env string      environment to use\n  -w, --watch           re-run the command every 2 seconds\n  -o, --output string   output format: one of pretty|json (default \"pretty\")\n  -v, --verbose         show additional information (only applies to pretty output format)\n  -h, --help            help for get\n```\n\n## describe\n\n```text\ndescribe an api\n\nUsage:\n  cortex describe [API_NAME] [flags]\n\nFlags:\n  -e, --env string   environment to use\n  -w, --watch        re-run the command every 2 seconds\n  -h, --help         help for describe\n```\n\n## logs\n\n```text\nget the logs for a workload\n\nUsage:\n  cortex logs API_NAME [JOB_ID] [flags]\n\nFlags:\n  -e, --env string   environment to use\n  -y, --yes          skip prompts\n      --random-pod   stream logs from a random pod\n  -h, --help         help for logs\n```\n\n## refresh\n\n```text\nrestart all replicas for an api (without downtime)\n\nUsage:\n  cortex refresh API_NAME [flags]\n\nFlags:\n  -e, --env string      environment to use\n  -f, --force           override the in-progress api update\n  -o, --output string   output format: one of pretty|json (default \"pretty\")\n  -h, --help            help for refresh\n```\n\n## delete\n\n```text\ndelete an api or stop a job\n\nUsage:\n  cortex delete API_NAME [JOB_ID] [flags]\n\nFlags:\n  -e, --env string      environment to use\n  -f, --force           delete the api without confirmation\n  -c, --keep-cache      keep cached data for the api\n  -o, --output string   output format: one of pretty|json (default \"pretty\")\n  -h, --help            help for delete\n```\n\n## cluster up\n\n```text\nspin up a cluster on aws\n\nUsage:\n  cortex cluster up CLUSTER_CONFIG_FILE [flags]\n\nFlags:\n  -e, --configure-env string   name of environment to configure (default: the name of your cluster)\n  -y, --yes                    skip prompts\n  -h, --help                   help for up\n```\n\n## cluster info\n\n```text\nget information about a cluster\n\nUsage:\n  cortex cluster info [flags]\n\nFlags:\n  -c, --config string          path to a cluster configuration file\n  -n, --name string            name of the cluster\n  -r, --region string          aws region of the cluster\n  -o, --output string          output format: one of pretty|json|yaml (default \"pretty\")\n  -e, --configure-env string   name of environment to configure\n  -d, --debug                  save the current cluster state to a file\n      --print-config           print the cluster config\n  -y, --yes                    skip prompts\n  -h, --help                   help for info\n```\n\n## cluster configure\n\n```text\nupdate the cluster's configuration\n\nUsage:\n  cortex cluster configure CLUSTER_CONFIG_FILE [flags]\n\nFlags:\n  -y, --yes    skip prompts\n  -h, --help   help for configure\n```\n\n## cluster down\n\n```text\nspin down a cluster\n\nUsage:\n  cortex cluster down [flags]\n\nFlags:\n  -c, --config string        path to a cluster configuration file\n  -n, --name string          name of the cluster\n  -r, --region string        aws region of the cluster\n  -y, --yes                  skip prompts\n      --keep-aws-resources   skip deletion of resources that cortex provisioned on aws (bucket contents, ebs volumes, log group)\n  -h, --help                 help for down\n```\n\n## cluster export\n\n```text\ndownload the configurations for all APIs\n\nUsage:\n  cortex cluster export [flags]\n\nFlags:\n  -c, --config string   path to a cluster configuration file\n  -n, --name string     name of the cluster\n  -r, --region string   aws region of the cluster\n  -h, --help            help for export\n```\n\n## cluster health\n\n```text\ninspect the health of components in the cluster\n\nUsage:\n  cortex cluster health [flags]\n\nFlags:\n  -c, --config string   path to a cluster configuration file\n  -n, --name string     name of the cluster\n  -r, --region string   aws region of the cluster\n  -o, --output string   output format: one of pretty|json (default \"pretty\")\n  -h, --help            help for health\n```\n\n## env configure\n\n```text\nconfigure an environment\n\nUsage:\n  cortex env configure [ENVIRONMENT_NAME] [flags]\n\nFlags:\n  -o, --operator-endpoint string   set the operator endpoint without prompting\n  -h, --help                       help for configure\n```\n\n## env list\n\n```text\nlist all configured environments\n\nUsage:\n  cortex env list [flags]\n\nFlags:\n  -o, --output string   output format: one of pretty|json (default \"pretty\")\n  -h, --help            help for list\n```\n\n## env default\n\n```text\nset the default environment\n\nUsage:\n  cortex env default [ENVIRONMENT_NAME] [flags]\n\nFlags:\n  -h, --help   help for default\n```\n\n## env rename\n\n```text\nrename an environment\n\nUsage:\n  cortex env rename EXISTING_NAME NEW_NAME [flags]\n\nFlags:\n  -h, --help   help for rename\n```\n\n## env delete\n\n```text\ndelete an environment configuration\n\nUsage:\n  cortex env delete [ENVIRONMENT_NAME] [flags]\n\nFlags:\n  -h, --help   help for delete\n```\n\n## version\n\n```text\nprint the cli and cluster versions\n\nUsage:\n  cortex version [flags]\n\nFlags:\n  -e, --env string   environment to use\n  -h, --help         help for version\n```\n\n## completion\n\n```text\ngenerate shell completion scripts\n\nto enable cortex shell completion:\n    bash:\n        add this to ~/.bash_profile (mac) or ~/.bashrc (linux):\n            source <(cortex completion bash)\n\n        note: bash-completion must be installed on your system; example installation instructions:\n            mac:\n                1) install bash completion:\n                   brew install bash-completion\n                2) add this to your ~/.bash_profile:\n                   source $(brew --prefix)/etc/bash_completion\n                3) log out and back in, or close your terminal window and reopen it\n            ubuntu:\n                1) install bash completion:\n                   apt update && apt install -y bash-completion  # you may need sudo\n                2) open ~/.bashrc and uncomment the bash completion section, or add this:\n                   if [ -f /etc/bash_completion ] && ! shopt -oq posix; then . /etc/bash_completion; fi\n                3) log out and back in, or close your terminal window and reopen it\n\n    zsh:\n        option 1:\n            add this to ~/.zshrc:\n                source <(cortex completion zsh)\n            if that failed, you can try adding this line (above the source command you just added):\n                autoload -Uz compinit && compinit\n        option 2:\n            create a _cortex file in your fpath, for example:\n                cortex completion zsh > /usr/local/share/zsh/site-functions/_cortex\n\nNote: this will also add the \"cx\" alias for cortex for convenience\n\nUsage:\n  cortex completion SHELL [flags]\n\nFlags:\n  -h, --help   help for completion\n```\n"
  },
  {
    "path": "docs/clients/install.md",
    "content": "# Install\n\n## Install the CLI\n\n<!-- CORTEX_VERSION_README x2 -->\n```bash\n# download CLI version 0.42.1 (Note the \"v\"):\nbash -c \"$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)\"\n```\n\nBy default, the Cortex CLI is installed at `/usr/local/bin/cortex`. To install the executable elsewhere, export the `CORTEX_INSTALL_PATH` environment variable to your desired location before running the command above.\n\n## Install the CLI and Python client via pip\n\nTo install the latest version:\n\n```bash\npip install cortex\n```\n\n<!-- CORTEX_VERSION_README x2 -->\nTo install or upgrade to a specific version (e.g. v0.42.1):\n\n```bash\npip install cortex==0.42.1\n```\n\nTo upgrade to the latest version:\n\n```bash\npip install --upgrade cortex\n```\n\n## Changing the CLI/client configuration directory\n\nBy default, the CLI/client creates a directory at `~/.cortex/` and uses it to store environment configuration. To use a different directory, export the `CORTEX_CLI_CONFIG_DIR` environment variable before running any `cortex` commands.\n"
  },
  {
    "path": "docs/clients/python.md",
    "content": "# Python client\n\n* [cortex](#cortex)\n  * [client](#client)\n  * [new\\_client](#new_client)\n  * [env\\_list](#env_list)\n  * [env\\_delete](#env_delete)\n* [cortex.client.Client](#cortex-client-client)\n  * [deploy](#deploy)\n  * [deploy\\_from\\_file](#deploy_from_file)\n  * [get\\_api](#get_api)\n  * [list\\_apis](#list_apis)\n  * [get\\_job](#get_job)\n  * [refresh](#refresh)\n  * [delete](#delete)\n  * [stop\\_job](#stop_job)\n\n# cortex\n\n## client\n\n```python\nclient(env_name: Optional[str] = None) -> Client\n```\n\nInitialize a client based on the specified environment. If no environment is specified, it will attempt to use the default environment.\n\n**Arguments**:\n\n- `env_name` - Name of the environment to use.\n\n\n**Returns**:\n\n  Cortex client that can be used to deploy and manage APIs in the specified environment.\n\n## new\\_client\n\n```python\nnew_client(env_name: str, operator_endpoint: str) -> Client\n```\n\nCreate a new environment to connect to an existing cluster, and initialize a client to deploy and manage APIs on that cluster.\n\n**Arguments**:\n\n- `env_name` - Name of the environment to create.\n- `operator_endpoint` - The endpoint for the operator of your Cortex cluster. You can get this endpoint by running the CLI command `cortex cluster info`.\n\n\n**Returns**:\n\n  Cortex client that can be used to deploy and manage APIs on a cluster.\n\n## env\\_list\n\n```python\nenv_list() -> List\n```\n\nList all environments configured on this machine.\n\n## env\\_delete\n\n```python\nenv_delete(name: str)\n```\n\nDelete an environment configured on this machine.\n\n**Arguments**:\n\n- `name` - Name of the environment to delete.\n\n# cortex.client.Client\n\n## deploy\n\n<!-- CORTEX_VERSION_MINOR -->\n\n```python\n | deploy(api_spec: Dict[str, Any], force: bool = True, wait: bool = False)\n```\n\nDeploy or update an API.\n\n**Arguments**:\n\n- `api_spec` - A dictionary defining a single Cortex API. See https://docs.cortexlabs.com/v/master/ for schema.\n- `force` - Override any in-progress api updates.\n- `wait` - Block until the API is ready.\n\n\n**Returns**:\n\n  Deployment status, API specification, and endpoint for each API.\n\n## deploy\\_from\\_file\n\n<!-- CORTEX_VERSION_MINOR -->\n\n```python\n | deploy_from_file(config_file: str, force: bool = False, wait: bool = False) -> Dict\n```\n\nDeploy or update APIs specified in a configuration file.\n\n**Arguments**:\n\n- `config_file` - Local path to a yaml file defining Cortex API(s). See https://docs.cortexlabs.com/v/master/ for schema.\n- `force` - Override any in-progress api updates.\n- `wait` - Block until the API is ready.\n\n\n**Returns**:\n\n  Deployment status, API specification, and endpoint for each API.\n\n## get\\_api\n\n```python\n | get_api(api_name: str) -> Dict\n```\n\nGet information about an API.\n\n**Arguments**:\n\n- `api_name` - Name of the API.\n\n\n**Returns**:\n\n  Information about the API, including the API specification, endpoint, status, and metrics (if applicable).\n\n## list\\_apis\n\n```python\n | list_apis() -> List\n```\n\nList all APIs in the environment.\n\n**Returns**:\n\n  List of APIs, including information such as the API specification, endpoint, status, and metrics (if applicable).\n\n## get\\_job\n\n```python\n | get_job(api_name: str, job_id: str) -> Dict\n```\n\nGet information about a submitted job.\n\n**Arguments**:\n\n- `api_name` - Name of the Batch/Task API.\n- `job_id` - Job ID.\n\n\n**Returns**:\n\n  Information about the job, including the job status, worker status, and job progress.\n\n## refresh\n\n```python\n | refresh(api_name: str, force: bool = False)\n```\n\nRestart all of the replicas for a Realtime API without downtime.\n\n**Arguments**:\n\n- `api_name` - Name of the API to refresh.\n- `force` - Override an already in-progress API update.\n\n## delete\n\n```python\n | delete(api_name: str, keep_cache: bool = False)\n```\n\nDelete an API.\n\n**Arguments**:\n\n- `api_name` - Name of the API to delete.\n- `keep_cache` - Whether to retain the cached data for this API.\n\n## stop\\_job\n\n```python\n | stop_job(api_name: str, job_id: str, keep_cache: bool = False)\n```\n\nStop a running job.\n\n**Arguments**:\n\n- `api_name` - Name of the Batch/Task API.\n- `job_id` - ID of the Job to stop.\n"
  },
  {
    "path": "docs/clients/uninstall.md",
    "content": "# Uninstall\n\n## Uninstall (if installed with pip)\n\n```bash\npip uninstall cortex\nrm -rf ~/.cortex\n```\n\n## Uninstall (if installed without pip)\n\n```bash\nrm /usr/local/bin/cortex\nrm -rf ~/.cortex\n```\n"
  },
  {
    "path": "docs/clusters/advanced/kubectl.md",
    "content": "# Setting up `kubectl`\n\n## Install `kubectl`\n\nFollow these [instructions](https://kubernetes.io/docs/tasks/tools/install-kubectl).\n\n## Install the AWS CLI\n\nFollow these [instructions](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html).\n\n## Configure the AWS CLI\n\n```bash\naws --version  # should be >= 1.16\n\naws configure\n```\n\n## Update `kubeconfig`\n\n```bash\naws eks update-kubeconfig --name=<cluster_name> --region=<region>\n```\n\n## Test `kubectl`\n\n```bash\nkubectl get pods\n```\n"
  },
  {
    "path": "docs/clusters/advanced/registry.md",
    "content": "# Private Docker registry\n\n## Configuring `kubectl`\n\nFollow the instructions [here](kubectl.md).\n\n## Setting credentials\n\n```bash\nDOCKER_USERNAME=***\nDOCKER_PASSWORD=***\n\nkubectl create secret docker-registry registry-credentials \\\n    --namespace default \\\n    --docker-username=$DOCKER_USERNAME \\\n    --docker-password=$DOCKER_PASSWORD\n\nkubectl patch serviceaccount default --namespace default \\\n    -p \"{\\\"imagePullSecrets\\\": [{\\\"name\\\": \\\"registry-credentials\\\"}]}\"\n```\n\n## Deleting credentials\n\n```bash\nkubectl delete secret --namespace default registry-credentials\n\nkubectl patch serviceaccount default --namespace default \\\n    -p \"{\\\"imagePullSecrets\\\": []}\"\n```\n"
  },
  {
    "path": "docs/clusters/advanced/self-hosted-images.md",
    "content": "# Self-hosted Docker images\n\nSelf-hosting the Cortex cluster's system Docker images can be useful for reducing the ingress costs, for accelerating image pulls, or for eliminating the dependency on Cortex's public container registry.\n\nIn this guide, we'll use [ECR](https://aws.amazon.com/ecr/) as the destination container registry. When an ECR repository resides in the same region as your Cortex cluster, there are no costs incurred when pulling images.\n\n## Step 1\n\nMake sure you have the [aws](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv1.html), [docker](https://docs.docker.com/get-docker/), and [skopeo](https://github.com/containers/skopeo/blob/master/install.md) utilities installed.\n\n## Step 2\n\nExport the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables in your current shell, or run `aws configure`. These credentials must have access to push to ECR.\n\n## Step 3\n\nClone the Cortex repo using the release tag corresponding to your version (which you can check by running `cortex version`):\n\n<!-- CORTEX_VERSION_README -->\n\n```bash\nexport CORTEX_VERSION=0.42.1\ngit clone --depth 1 --branch v$CORTEX_VERSION https://github.com/cortexlabs/cortex.git\n```\n\n## Step 4\n\nRun the script below to export images to ECR in the same region and account as your cluster.\n\nThe script will automatically create ECR Repositories with prefix `cortexlabs` if they don't already exist.\n\nFeel free to modify the script if you would like to export the images to a different registry such as a private docker hub.\n\n```bash\n./cortex/dev/export_images.sh <AWS_REGION> <AWS_ACCOUNT_ID>\n```\n\nYou can now configure Cortex to use your images when creating a cluster (see [here](../management/create.md) for instructions).\n\n## Cleanup\n\nYou can delete your ECR images from the [AWS ECR dashboard](https://console.aws.amazon.com/ecr/repositories) (set your region in the upper right corner). Make sure all of your Cortex clusters have been deleted before deleting any ECR images.\n"
  },
  {
    "path": "docs/clusters/instances/multi.md",
    "content": "# Multi-instance type clusters\n\nCortex can be configured to provision different instance types to improve workload performance and reduce cloud infrastructure spend.\n\n## Best practices\n\n1. Spot node groups should have a higher priority than on-demand node groups.\n1. CPU node groups should have higher priorities than GPU/Inferentia node groups.\n1. Node groups with small instance types should have higher priorities than node groups with large instance types.\n\n## Examples\n\n### CPU spot cluster, with on-demand backup\n\n```yaml\n# cluster.yaml\n\nnode_groups:\n  - name: cpu-spot\n    instance_type: m5.large\n    min_instances: 0\n    max_instances: 5\n    priority: 100\n    spot: true\n    spot_config:\n      instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large]\n  - name: cpu-on-demand\n    instance_type: m5.large\n    min_instances: 0\n    max_instances: 5\n```\n\n### On-demand cluster supporting CPU, GPU, and Inferentia\n\n```yaml\n# cluster.yaml\n\nnode_groups:\n  - name: cpu\n    instance_type: m5.large\n    min_instances: 0\n    max_instances: 5\n    priority: 100\n  - name: gpu\n    instance_type: g4dn.xlarge\n    min_instances: 0\n    max_instances: 5\n  - name: inf\n    instance_type: inf.xlarge\n    min_instances: 0\n    max_instances: 5\n```\n\n### Spot cluster supporting CPU and GPU (with on-demand backup)\n\n```yaml\n# cluster.yaml\n\nnode_groups:\n  - name: cpu-spot\n    instance_type: m5.large\n    min_instances: 0\n    max_instances: 5\n    priority: 100\n    spot: true\n    spot_config:\n      instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large]\n  - name: cpu-on-demand\n    instance_type: m5.large\n    min_instances: 0\n    max_instances: 5\n    priority: 50\n  - name: gpu-spot\n    instance_type: g4dn.xlarge\n    min_instances: 0\n    max_instances: 5\n    priority: 20\n    spot: true\n  - name: gpu-on-demand\n    instance_type: g4dn.xlarge\n    min_instances: 0\n    max_instances: 5\n```\n\n### CPU spot cluster with multiple instance types and on-demand backup\n\n```yaml\n# cluster.yaml\n\nnode_groups:\n  - name: cpu-1\n    instance_type: t3.medium\n    min_instances: 0\n    max_instances: 5\n    priority: 100\n    spot: true\n  - name: cpu-2\n    instance_type: m5.2xlarge\n    min_instances: 0\n    max_instances: 5\n    priority: 70\n    spot: true\n  - name: cpu-3\n    instance_type: m5.8xlarge\n    min_instances: 0\n    max_instances: 5\n    priority: 30\n    spot: true\n  - name: cpu-4\n    instance_type: m5.24xlarge\n    min_instances: 0\n    max_instances: 5\n```\n"
  },
  {
    "path": "docs/clusters/instances/spot.md",
    "content": "# Spot instances\n\n```yaml\n# cluster.yaml\n\nnode_groups:\n  - name: node-group-1\n\n    # whether to use spot instances for this node group (default: false)\n    spot: false  # this must be set to true to use spot instances\n\n    spot_config:\n      # additional instance types with identical or better specs than the primary cluster instance type (defaults to only the primary instance type)\n      instance_distribution: # [similar_instance_type_1, similar_instance_type_2]\n\n      # minimum number of on demand instances (default: 0)\n      on_demand_base_capacity: 0\n\n      # percentage of on demand instances to use after the on demand base capacity has been met [0, 100] (default: 50)\n      # note: setting this to 0 may hinder cluster scale-up when spot instances are not available\n      on_demand_percentage_above_base_capacity: 0\n\n      # max price for spot instances (default: the on-demand price of the primary instance type)\n      max_price: # <float>\n\n      # number of spot instance pools across which to allocate spot instances [1, 20] (default: number of instances in instance distribution)\n      instance_pools: 3\n```\n\nSpot instances are not guaranteed to be available. The chances of getting spot instances can be improved by providing `instance_distribution`, a list of alternative instance types to the primary `instance_type` you specified. If left blank, Cortex will only include the primary instance type in the `instance_distribution`. When using `instance_distribution`, use the instance type with the fewest compute resources as your primary `instance_type`. Note that the default value for `max_price` is the on-demand price of the primary instance type, but you may wish to set this to the on-demand price of the most expensive instance type in your `instance_distribution`.\n\nSpot instances can be mixed with on-demand instances in a single node group by configuring `on_demand_base_capacity` and `on_demand_percentage_above_base_capacity`. `on_demand_base_capacity` enforces the minimum number of nodes that will be fulfilled by on-demand instances as your cluster is scaling up. `on_demand_percentage_above_base_capacity` defines the percentage of instances that will be on-demand after the base capacity has been fulfilled (the rest being spot instances). `instance_pools` is the number of pools per availability zone to allocate your instances from. See [here](https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_InstancesDistribution.html) for more details.\n\nEven if multiple instances are specified in your `instance_distribution`, it is still possible that AWS will not be able to provision a spot instance when requested. One possibility is that AWS has exhausted all of the available spot instances of your requested type(s) in your availability zones. Another possibility is that the current price of your requested instance type(s) is higher than your `max_price`. To mitigate this, you may add a second node group to your cluster configuration which is configured to use on-demand instances as a backup. When doing this, it is important to position the on-demand node group after the spot node group in the `node_groups` list (since node groups with lower indices have higher priority). See [here](multi.md) for docs and examples.\n\nThere is a spot instance limit associated with your AWS account for each instance family in each region. You can check your current limit and request an increase [here](https://console.aws.amazon.com/servicequotas/home?#!/services/ec2/quotas) (set the region in the upper right corner to your desired region, type \"spot\" in the search bar, and click on the quota that matches your instance type). Note that the quota values indicate the number of vCPUs available, not the number of instances; different instances have a different numbers of vCPUs, which can be seen [here](https://aws.amazon.com/ec2/instance-types/).\n\n## Example spot configuration\n\n### Only spot instances\n\n```yaml\nnode_groups:\n  - name: cpu-spot\n    instance_type: m5.large\n    min_instances: 0\n    max_instances: 5\n    spot: true\n    spot_config:\n      instance_distribution: [m5a.large, m5d.large, m5n.large, m5ad.large, m5dn.large, m4.large, t3.large, t3a.large, t2.large]\n```\n\n### 3 on-demand base capacity with 0% on-demand above base capacity\n\n```yaml\n\nnode_groups:\n  - name: gpu-spot\n    instance_type: g4dn.xlarge\n    min_instances: 0\n    max_instances: 5\n    spot: true\n    spot_config:\n      on_demand_base_capacity: 3\n      on_demand_percentage_above_base_capacity: 0\n\n# instance 1-3: on-demand\n# instance 4-5: spot\n```\n\n### 0 on-demand base capacity with 50% on-demand above base capacity\n\n```yaml\nnode_groups:\n  - name: gpu-spot\n    instance_type: g4dn.xlarge\n    min_instances: 0\n    max_instances: 4\n    spot: true\n    spot_config:\n      on_demand_base_capacity: 0\n      on_demand_percentage_above_base_capacity: 50\n\n# instance 1: on-demand\n# instance 2: spot\n# instance 3: on-demand\n# instance 4: spot\n```\n"
  },
  {
    "path": "docs/clusters/management/auth.md",
    "content": "# Auth\n\n## Client\n\nThe Cortex CLI and Python client use the default credential provider chain to get credentials for cluster and api management. Credentials will be read in the following order of precedence:\n\n- environment variables\n- the name of the profile specified by `AWS_PROFILE` environment variable\n- `default` profile from `~/.aws/credentials`\n\n### Cluster management\n\nIt is recommended that your AWS credentials have AdministratorAccess when running `cortex cluster *` commands. If you are unable to use AdministratorAccess, see the [minimum IAM policy](#minimum-iam-policy) below for the minimum permissions required to run `cortex cluster *` commands.\n\nAfter spinning up a cluster using `cortex cluster up`, the IAM user or role that created the cluster is automatically granted `system:masters` permission to the cluster's RBAC. Make sure to keep track of which IAM entity originally created the cluster.\n\n#### Running `cortex cluster` commands from different IAM users\n\nBy default, the `cortex cluster *` commands can only be executed by the IAM user who created the cluster. To grant access to additional IAM users, follow these steps:\n\n1. Install `eksctl` by following these [instructions](https://eksctl.io/introduction/#installation).\n\n1. Determine the ARN of the IAM user that you would like to grant access to. You can get the ARN via the [IAM dashboard](https://console.aws.amazon.com/iam/home#/users), or by running `aws iam get-user` on a machine that is authenticated as the IAM user (or `AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** aws iam get-user` on any machine, using the credentials of the IAM user). The ARN should look similar to `arn:aws:iam::764403040417:user/my-username`.\n\n1. Set the following environment variables:\n\n    ```bash\n    CREATOR_AWS_ACCESS_KEY_ID=***      # access key ID for the IAM user that created the cluster\n    CREATOR_AWS_SECRET_ACCESS_KEY=***  # secret access key for the IAM user that created the cluster\n    NEW_USER_ARN=***                   # ARN of the IAM user to grant access to\n    CLUSTER_NAME=***                   # the name of your cortex cluster (will be \"cortex\" unless you specified a different name in your cluster configuration file)\n    CLUSTER_REGION=***                 # the region of your cortex cluster\n    ```\n\n1. Run the following command:\n\n    ```bash\n    AWS_ACCESS_KEY_ID=$CREATOR_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$CREATOR_AWS_SECRET_ACCESS_KEY eksctl create iamidentitymapping --region $CLUSTER_REGION --cluster $CLUSTER_NAME --arn $NEW_USER_ARN --group system:masters --username $NEW_USER_ARN\n    ```\n\n1. To revoke access in the future, run:\n\n    ```bash\n    AWS_ACCESS_KEY_ID=$CREATOR_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$CREATOR_AWS_SECRET_ACCESS_KEY eksctl delete iamidentitymapping --region $CLUSTER_REGION --cluster $CLUSTER_NAME --arn $NEW_USER_ARN --all\n    ```\n\n### API management\n\nThe Cortex CLI and Python client rely on AWS IAM to authenticate requests to a cluster on AWS (e.g. `cortex deploy`, `cortex get`). AWS credentials required to authenticate Cortex client requests to the operator don't require any specific permissions; they must only be valid credentials within the same AWS account as the Cortex cluster. However, managing the cluster (i.e. running `cortex cluster *` commands) does require permissions.\n\n## Authorizing your APIs\n\nWhen spinning up a cortex cluster, you can provide additional policies to authorize your APIs to access AWS resources by creating a policy and adding it to the `iam_policy_arns` list in your cluster configuration file.\n\nIf you already have a cluster running and would like to add additional permissions, you can update the policy that is created automatically during `cortex cluster up`. In the [IAM console](https://console.aws.amazon.com/iam/home?policies#/policies), search for `cortex-<cluster_name>-<region>` to find the policy that has been attached to your cluster. Adding more permissions to this policy will automatically give more access to all of your Cortex APIs.\n\n_NOTE: The policy created during `cortex cluster up` will automatically be deleted during `cortex cluster down`. It is recommended to create your own policies that can be specified in `iam_policy_arns` field in cluster configuration. The precreated policy should only be updated for development and testing purposes._\n\n## Minimum IAM Policy\n\nThe policy shown below contains the minimum permissions required to manage a Cortex cluster (i.e. via `cortex cluster *` commands).\n\nReplace the following placeholders with their respective values in the policy template below: `$CORTEX_CLUSTER_NAME`, `$CORTEX_ACCOUNT_ID`, `$CORTEX_REGION`.\n\n```json\n{\n    \"Version\": \"2012-10-17\",\n    \"Statement\": [\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"iam:CreateServiceLinkedRole\",\n            \"Resource\": \"*\",\n            \"Condition\": {\n                \"StringEquals\": {\n                    \"iam:AWSServiceName\": [\n                        \"autoscaling.amazonaws.com\",\n                        \"ec2scheduled.amazonaws.com\",\n                        \"elasticloadbalancing.amazonaws.com\",\n                        \"spot.amazonaws.com\",\n                        \"spotfleet.amazonaws.com\",\n                        \"transitgateway.amazonaws.com\"\n                    ]\n                }\n            }\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"iam:CreateServiceLinkedRole\",\n            \"Resource\": \"*\",\n            \"Condition\": {\n                \"StringEquals\": {\n                    \"iam:AWSServiceName\": [\n                        \"eks.amazonaws.com\",\n                        \"eks-nodegroup.amazonaws.com\",\n                        \"eks-fargate.amazonaws.com\"\n                    ]\n                }\n            }\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"logs:ListTagsLogGroup\",\n                \"iam:GetRole\",\n                \"logs:TagLogGroup\",\n                \"ssm:GetParameters\",\n                \"ssm:GetParameter\",\n                \"logs:CreateLogGroup\"\n            ],\n            \"Resource\": [\n                \"arn:*:ssm:*:$CORTEX_ACCOUNT_ID:parameter/aws/*\",\n                \"arn:*:ssm:*::parameter/aws/*\",\n                \"arn:*:logs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:log-group:$CORTEX_CLUSTER_NAME\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:role/*\"\n            ]\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"iam:CreateInstanceProfile\",\n                \"logs:ListTagsLogGroup\",\n                \"logs:DescribeLogStreams\",\n                \"iam:TagRole\",\n                \"iam:GetPolicy\",\n                \"iam:CreatePolicy\",\n                \"iam:DeletePolicy\",\n                \"iam:ListPolicyVersions\",\n                \"iam:RemoveRoleFromInstanceProfile\",\n                \"iam:CreateRole\",\n                \"iam:AttachRolePolicy\",\n                \"iam:PutRolePolicy\",\n                \"iam:AddRoleToInstanceProfile\",\n                \"iam:ListInstanceProfilesForRole\",\n                \"iam:PassRole\",\n                \"logs:CreateLogStream\",\n                \"iam:DetachRolePolicy\",\n                \"logs:TagLogGroup\",\n                \"iam:ListAttachedRolePolicies\",\n                \"iam:DeleteRolePolicy\",\n                \"iam:DeleteOpenIDConnectProvider\",\n                \"iam:TagOpenIDConnectProvider\",\n                \"iam:DeleteInstanceProfile\",\n                \"iam:GetRole\",\n                \"iam:GetInstanceProfile\",\n                \"iam:DeleteRole\",\n                \"iam:ListInstanceProfiles\",\n                \"logs:CreateLogGroup\",\n                \"logs:PutLogEvents\",\n                \"logs:DeleteLogGroup\",\n                \"iam:CreateOpenIDConnectProvider\",\n                \"iam:GetOpenIDConnectProvider\",\n                \"iam:GetRolePolicy\"\n            ],\n            \"Resource\": [\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:instance-profile/eksctl-*\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-*\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:policy/eksctl-*\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:role/aws-service-role/eks-nodegroup.amazonaws.com/AWSServiceRoleForAmazonEKSNodegroup\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-managed-*\",\n                \"arn:*:iam::$CORTEX_ACCOUNT_ID:oidc-provider/*\",\n                \"arn:*:logs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:log-group:$CORTEX_CLUSTER_NAME:*\"\n            ]\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"iam:CreatePolicy\",\n                \"iam:GetPolicyVersion\",\n                \"iam:ListPolicyVersions\",\n                \"iam:DeletePolicy\",\n                \"iam:CreatePolicyVersion\",\n                \"iam:DeletePolicyVersion\"\n            ],\n            \"Resource\": \"arn:*:iam::$CORTEX_ACCOUNT_ID:policy/cortex-*\"\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": [\n                \"sqs:ListQueues\",\n                \"iam:GetPolicy\",\n                \"ecr:GetAuthorizationToken\",\n                \"cloudformation:*\",\n                \"elasticloadbalancing:*\",\n                \"autoscaling:*\",\n                \"cloudwatch:*\",\n                \"ecr:BatchGetImage\",\n                \"kms:DescribeKey\",\n                \"ec2:*\",\n                \"sts:GetCallerIdentity\",\n                \"eks:*\",\n                \"kms:CreateGrant\",\n                \"acm:DescribeCertificate\",\n                \"servicequotas:ListServiceQuotas\",\n                \"logs:PutRetentionPolicy\"\n            ],\n            \"Resource\": \"*\"\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"sqs:*\",\n            \"Resource\": \"arn:*:sqs:$CORTEX_REGION:$CORTEX_ACCOUNT_ID:cx-*\"\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"s3:*\",\n            \"Resource\": \"arn:*:s3:::$CORTEX_CLUSTER_NAME*\"\n        },\n        {\n            \"Effect\": \"Allow\",\n            \"Action\": \"s3:*\",\n            \"Resource\": \"arn:*:s3:::$CORTEX_CLUSTER_NAME*/*\"\n        }\n    ]\n}\n```\n"
  },
  {
    "path": "docs/clusters/management/create.md",
    "content": "# Install\n\n## Prerequisites\n\n1. Install and run [Docker](https://docs.docker.com/install) on your machine.\n1. Subscribe to the [AMI with GPU support](https://aws.amazon.com/marketplace/pp/B07GRHFXGM) (for GPU clusters).\n1. Create an IAM user with `AdministratorAccess` and programmatic access.\n1. You may need to [request limit increases](https://console.aws.amazon.com/servicequotas/home?#!/services/ec2/quotas) for your desired instance types.\n\n## Create a cluster on your AWS account\n\n<!-- CORTEX_VERSION_README -->\n```bash\n# install the cortex CLI\nbash -c \"$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)\"\n\n# create a cluster\ncortex cluster up cluster.yaml\n```\n\n## `cluster.yaml`\n\n```yaml\n# cluster name\ncluster_name: cortex\n\n# AWS region\nregion: us-east-1\n\n# list of availability zones for your region\navailability_zones:  # default: 3 random availability zones in your region, e.g. [us-east-1a, us-east-1b, us-east-1c]\n\n# list of cluster node groups;\nnode_groups:\n  - name: ng-cpu # name of the node group\n    instance_type: m5.large # instance type\n    min_instances: 1 # minimum number of instances\n    max_instances: 5 # maximum number of instances\n    priority: 1 # priority of the node group; the higher the value, the higher the priority [1-100]\n    instance_volume_size: 50 # disk storage size per instance (GB)\n    instance_volume_type: gp3 # instance volume type [gp2 | gp3 | io1 | st1 | sc1]\n    # instance_volume_iops: 3000 # instance volume iops (only applicable to io1/gp3)\n    # instance_volume_throughput: 125 # instance volume throughput (only applicable to gp3)\n    spot: false # whether to use spot instances\n\n  - name: ng-gpu\n    instance_type: g4dn.xlarge\n    min_instances: 1\n    max_instances: 5\n    instance_volume_size: 50\n    instance_volume_type: gp3\n    spot: false\n  # ...\n\n# subnet visibility for instances [public (instances will have public IPs) | private (instances will not have public IPs)]\n# when using private subnets, you may wish to enable VPC endpoints (via the AWS console) for S3 and ECR to avoid extra NAT Gateway charges\nsubnet_visibility: public\n\n# NAT gateway (required when using private subnets) [none | single | highly_available (a NAT gateway per availability zone)]\nnat_gateway: none\n\n# API load balancer type [nlb | elb]\napi_load_balancer_type: nlb\n\n# API load balancer scheme [internet-facing | internal]\napi_load_balancer_scheme: internet-facing\n\n# operator load balancer scheme [internet-facing | internal]\n# note: if using \"internal\", you must configure VPC Peering to connect your CLI to your cluster operator\noperator_load_balancer_scheme: internet-facing\n\n# to install Cortex in an existing VPC, you can provide a list of subnets for your cluster to use\n# subnet_visibility (specified above in this file) must match your subnets' visibility\n# this is an advanced feature (not recommended for first-time users) and requires your VPC to be configured correctly; see https://eksctl.io/usage/vpc-networking/#use-existing-vpc-other-custom-configuration\n# here is an example:\n# subnets:\n#   - availability_zone: us-west-2a\n#     subnet_id: subnet-060f3961c876872ae\n#   - availability_zone: us-west-2b\n#     subnet_id: subnet-0faed05adf6042ab7\n\n# restrict access to APIs by cidr blocks/ip address ranges\napi_load_balancer_cidr_white_list: [0.0.0.0/0]\n\n# restrict access to the Operator by cidr blocks/ip address ranges\noperator_load_balancer_cidr_white_list: [0.0.0.0/0]\n\n# additional tags to assign to AWS resources (all resources will automatically be tagged with cortex.dev/cluster-name: <cluster_name>)\ntags:  # <string>: <string> map of key/value pairs\n\n# SSL certificate ARN (only necessary when using a custom domain)\nssl_certificate_arn:\n\n# list of IAM policies to attach to your Cortex APIs\niam_policy_arns: [\"arn:aws:iam::aws:policy/AmazonS3FullAccess\"]\n\n# primary CIDR block for the cluster's VPC\nvpc_cidr: 192.168.0.0/16\n\n# instance type for prometheus (use an instance with more memory for clusters exceeding 300 nodes or 300 pods)\nprometheus_instance_type: \"t3.medium\"\n```\n\nThe docker images used by the cluster can also be overridden. They can be configured by adding any of these keys to your cluster configuration file (default values are shown):\n\n<!-- CORTEX_VERSION_BRANCH_STABLE -->\n```yaml\nimage_manager: quay.io/cortexlabs/manager:master\nimage_operator: quay.io/cortexlabs/operator:master\nimage_controller_manager: quay.io/cortexlabs/controller-manager:master\nimage_autoscaler: quay.io/cortexlabs/autoscaler:master\nimage_proxy: quay.io/cortexlabs/proxy:master\nimage_async_gateway: quay.io/cortexlabs/async-gateway:master\nimage_activator: quay.io/cortexlabs/activator:master\nimage_enqueuer: quay.io/cortexlabs/enqueuer:master\nimage_dequeuer: quay.io/cortexlabs/dequeuer:master\nimage_cluster_autoscaler: quay.io/cortexlabs/cluster-autoscaler:master\nimage_metrics_server: quay.io/cortexlabs/metrics-server:master\nimage_nvidia_device_plugin: quay.io/cortexlabs/nvidia-device-plugin:master\nimage_neuron_device_plugin: quay.io/cortexlabs/neuron-device-plugin:master\nimage_neuron_scheduler: quay.io/cortexlabs/neuron-scheduler:master\nimage_fluent_bit: quay.io/cortexlabs/fluent-bit:master\nimage_istio_proxy: quay.io/cortexlabs/istio-proxy:master\nimage_istio_pilot: quay.io/cortexlabs/istio-pilot:master\nimage_prometheus: quay.io/cortexlabs/prometheus:master\nimage_prometheus_config_reloader: quay.io/cortexlabs/prometheus-config-reloader:master\nimage_prometheus_operator: quay.io/cortexlabs/prometheus-operator:master\nimage_prometheus_statsd_exporter: quay.io/cortexlabs/prometheus-statsd-exporter:master\nimage_prometheus_dcgm_exporter: quay.io/cortexlabs/prometheus-dcgm-exporter:master\nimage_prometheus_kube_state_metrics: quay.io/cortexlabs/prometheus-kube-state-metrics:master\nimage_prometheus_node_exporter: quay.io/cortexlabs/prometheus-node-exporter:master\nimage_kube_rbac_proxy: quay.io/cortexlabs/kube-rbac-proxy:master\nimage_grafana: quay.io/cortexlabs/grafana:master\nimage_event_exporter: quay.io/cortexlabs/event-exporter:master\nimage_kubexit: quay.io/cortexlabs/kubexit:master\n```\n"
  },
  {
    "path": "docs/clusters/management/delete.md",
    "content": "# Uninstall\n\n```bash\ncortex cluster down\n```\n\n## Bucket Contents\n\nWhen a Cortex cluster is created, an S3 bucket is created for its internal use. When running `cortex cluster down`, a lifecycle rule is applied to the bucket such that its entire contents are removed within the next 24 hours. You can safely delete the bucket at any time after `cortex cluster down` has finished running.\n\n## Delete SSL Certificate\n\nIf you've set up HTTPS, you can remove the SSL Certificate by following these [instructions](../networking/https.md#cleanup).\n\n## Delete Hosted Zone\n\nIf you've configured a custom domain for your APIs, follow these [instructions](../networking/custom-domain.md#cleanup) to delete the Hosted Zone.\n\n## Keep Cortex Resources\n\nThe contents of Cortex's S3 bucket, the EBS volumes (used by Cortex's Prometheus and Grafana instances), and the log group are deleted by default when running `cortex cluster down`. If you want to keep these resources, you can pass the `--keep-aws-resources` flag to the `cortex cluster down` command.\n\n## Troubleshooting\n\nOn rare occasions, `cortex cluster down` may not be able to spin down your Cortex cluster. When this happens, follow\nthese steps:\n\n1. If you've manually created any AWS networking resources that are pointed to the cluster or its VPC (e.g. API Gateway\n   VPC links, custom domains, etc), delete them from the AWS console.\n\n1. Replace \"<region>\" and \"<cluster_name>\" in the following URL, and open it in your\n   browser: `https://console.aws.amazon.com/cloudformation/home?region=<region>#/stacks?filteringText=eksctl-<cluster_name>-`\n\n   ![image](https://user-images.githubusercontent.com/808475/97790394-963b4880-1b85-11eb-8e27-ba5a551606b3.png)\n\n1. For each CloudFormation stack which contains the word \"nodegroup\", select the stack and click \"Delete\".\n\n1. Select the final stack (the one that ends in \"-cluster\") and click \"Delete\".\n\n   If deleting the stack fails, navigate to the EC2 dashboard in the AWS console, delete the load balancers that are\n   associated with the cluster, and try again (you can determine which load balancers are associated with the cluster by\n   setting the correct region in the console and checking the `cortex.dev/cluster-name` tag on all load balancers). If\n   the problem still persists, delete any other AWS resources that are blocking the stack deletion and try again.\n\n1. In rare cases, you may need to delete other AWS resources associated with your Cortex cluster. For each the following\n   resources, go to the appropriate AWS Dashboard (in the region that your cluster was in), and confirm that there are\n   no resources left behind by the cluster: CloudWatch Dashboard, SQS Queues, S3 Bucket, and CloudWatch LogGroups (the\n   Cortex bucket and log groups are not deleted by `cluster down` in order to preserve your data).\n"
  },
  {
    "path": "docs/clusters/management/environments.md",
    "content": "# Environments\n\nWhen you create a cluster with `cortex cluster up`, an environment with the same name as your cluster is automatically created to point to your cluster and is configured to be the default environment. You can name the environment something else via the `--configure-env` flag, e.g. `cortex cluster up --configure-env prod`. You can also use the `--configure-env` flag with `cortex cluster info` to create / update the specified environment.\n\nYou can list your environments with `cortex env list`, change the default environment with `cortex env default`, rename an environment with `cortex env rename`, delete an environment with `cortex env delete`, and create/update an environment with `cortex env configure`.\n\n## Multiple clusters\n\n```bash\ncortex cluster up cluster1.yaml --configure-env cluster1  # configures the cluster1 env\ncortex cluster up cluster2.yaml --configure-env cluster2  # configures the cluster2 env\n\ncortex deploy --env cluster1\ncortex delete my-api --env cluster1\n\ncortex deploy --env cluster2\ncortex delete my-api --env cluster2\n```\n\n## Multiple clusters, if you omitted the `--configure-env` on `cortex cluster up`\n\n```bash\ncortex cluster info cluster1.yaml --configure-env cluster1  # configures the cluster1 env\ncortex cluster info cluster2.yaml --configure-env cluster2  # configures the cluster2 env\n\ncortex deploy --env cluster1\ncortex delete my-api --env cluster1\n\ncortex deploy --env cluster2\ncortex delete my-api --env cluster2\n```\n\n## Configure `cortex` CLI to connect to an existing cluster\n\nIf you are installing the `cortex` CLI on a new machine, you can configure it to access an existing Cortex cluster.\n\nOn the machine which already has the CLI configured, run:\n\n```bash\ncortex env list\n```\n\nTake note of the environment name and operator endpoint of the desired environment.\n\nOn your new machine, run:\n\n```bash\ncortex env configure\n```\n"
  },
  {
    "path": "docs/clusters/management/production.md",
    "content": "# Production guide\n\nAs you take Cortex from development to production, here are a few pointers that might be useful.\n\n## Use images from a colocated ECR\n\nConfigure your cluster and APIs to use images from ECR in the same region as your cluster to accelerate scale-ups, reduce ingress costs, and remove the dependency on Cortex's public quay.io registry.\n\nYou can find instructions for mirroring Cortex images [here](../advanced/self-hosted-images.md)\n\n## Handling Cortex updates/upgrades\n\nUse a Route 53 hosted zone as a proxy in front of your Cortex cluster. Every new Cortex cluster provisions a new API load balancer with a unique endpoint. Using a Route 53 hosted zone configured with a subdomain will expose your Cortex cluster API endpoint as a static endpoint (e.g. `cortex.your-company.com`). You will be able to upgrade Cortex versions without downtime, and you will avoid the need to updated your client code every time you migrate to a new cluster. You can find instructions for setting up a custom domain with a Route 53 hosted zone [here](../networking/custom-domain.md), and instructions for updating/upgrading your cluster [here](update.md).\n\n## Production cluster configuration\n\n### Securing your cluster\n\nThe following configuration will improve security by preventing your cluster's nodes from being publicly accessible.\n\n```yaml\nsubnet_visibility: private\n\nnat_gateway: single  # use \"highly_available\" for large clusters making requests to services outside of the cluster\n```\n\nYou can make your load balancer private to prevent your APIs from being publicly accessed. In order to access your APIs, you will need to set up VPC peering between the Cortex cluster's VPC and the VPC containing the consumers of the Cortex APIs. See the [VPC peering guide](../networking/vpc-peering.md) for more details.\n\n```yaml\napi_load_balancer_scheme: internal\n```\n\nYou can also restrict access to your load balancers by IP address:\n\n```yaml\napi_load_balancer_cidr_white_list: [0.0.0.0/0]\n```\n\nThese two fields are also available for the operator load balancer. Keep in mind that if you make the operator load balancer private, you'll need to configure VPC peering to use the `cortex` CLI or Python client.\n\n```yaml\noperator_load_balancer_scheme: internal\noperator_load_balancer_cidr_white_list: [0.0.0.0/0]\n```\n\nSee [here](../networking/load-balancers.md) for more information about the load balancers.\n\n### Workload load-balancing\n\nDepending on your application's requirements, you might have different needs from the cluster's api load balancer. By default, the api load balancer is a [Network load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) (NLB). In some situations, a [Classic load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/introduction.html) (ELB) may be preferred, and can be selected in your cluster config by setting `api_load_balancer_type: elb`. This selection can only be made before creating your cluster.\n\n### Ensure node provisioning\n\nYou can take advantage of the cost savings of spot instances and the reliability of on-demand instances by utilizing the `priority` field in node groups. You can deploy two node groups, one that is spot and another that is on-demand. Set the priority of the spot node group to be higher than the priority of the on-demand node group. This encourages the cluster-autoscaler to try to spin up instances from the spot node group first. If there are no more spot instances available, the on-demand node group will be used instead.\n\n```yaml\nnode_groups:\n  - name: gpu-spot\n    instance_type: g4dn.xlarge\n    min_instances: 0\n    max_instances: 5\n    spot: true\n    priority: 100\n  - name: gpu-on-demand\n    instance_type: g4dn.xlarge\n    min_instances: 0\n    max_instances: 5\n    priority: 1\n```\n\n### Considerations for large clusters\n\nIf you plan on scaling your Cortex cluster past 300 nodes or 300 pods, it is recommended to set `prometheus_instance_type` to an instance type with more memory (the default is `t3.medium`, which has 4gb).\n\n## API Spec\n\n### Container design\n\nConfigure your health checks to be as accurate as possible to prevent requests from being routed to pods that aren't ready to handle traffic.\n\n### Pods section\n\nMake sure that `max_concurrency` is set to match the concurrency supported by your container.\n\nTune `max_queue_length` to lower values if you would like to more aggressively redistribute requests to newer pods as your API scales up rather than allowing requests to linger in queues. This would mean that the clients consuming your APIs should implement retry logic with a delay (such as exponential backoff).\n\n### Compute section\n\nMake sure to specify all of the relevant compute resources (especially cpu and memory) to ensure that your pods aren't starved for resources.\n\n### Autoscaling\n\nRevisit the autoscaling docs for [Realtime APIs](../../workloads/realtime/autoscaling.md) and/or [Async APIs](../../workloads/async/autoscaling.md) to effectively handle production traffic by tuning the scaling rate, sensitivity, and over-provisioning.\n"
  },
  {
    "path": "docs/clusters/management/update.md",
    "content": "# Update\n\n## Modify existing cluster\n\nYou can add or remove node groups, resize existing node groups, and update some configuration fields of a running cluster.\n\nFetch the current cluster configuration:\n\n```bash\ncortex cluster info --print-config --name CLUSTER_NAME --region REGION > cluster.yaml\n```\n\nMake your desired changes, and then apply them:\n\n```bash\ncortex cluster configure cluster.yaml\n```\n\nCortex will calculate the difference and you will be prompted with the update plan.\n\nIf you would like to update fields that cannot be modified on a running cluster, you must create a new cluster with your desired configuration.\n\n## Upgrade to a new version\n\nUpdating an existing Cortex cluster is not supported at the moment. Please spin down the previous version of the cluster, install the latest version of the Cortex CLI, and use it to spin up a new Cortex cluster. See the next section for how to do this without downtime.\n\n## Update or upgrade without downtime\n\nIt is possible to update to a new version Cortex or to migrate from one cluster to another without downtime.\n\nNote: it is important to not spin down your previous cluster until after your new cluster is receiving traffic.\n\n### Set up a subdomain using a Route 53 hosted zone\n\nIf you've already set up a subdomain with a Route 53 hosted zone pointing to your cluster, skip this step.\n\nSetting up a Route 53 hosted zone allows you to transfer traffic seamlessly from from an existing cluster to a new cluster, thereby avoiding downtime. You can find the instructions for setting up a subdomain [here](../networking/custom-domain.md). You will need to update any clients interacting with your Cortex APIs to point to the new subdomain.\n\n### Export all APIs from your previous cluster\n\nThe `cluster export` command can be used to get the YAML specifications of all APIs deployed in your cluster:\n\n```bash\ncortex cluster export --name <previous_cluster_name> --region <region>\n```\n\n### Spin up a new cortex cluster\n\nIf you are creating a new cluster with the same Cortex version:\n\n```bash\ncortex cluster up new-cluster.yaml --configure-env cortex2\n```\n\nThis will create a CLI environment named `cortex2` for accessing the new cluster.\n\nIf you are spinning a up a new cluster with a different Cortex version, first install the cortex CLI matching the desired cluster version:\n\n<!-- CORTEX_VERSION_README x2 -->\n\n```bash\n# download the desired CLI version, replace 0.42.1 with the desired version (Note the \"v\"):\nbash -c \"$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)\"\n\n# confirm Cortex CLI version\ncortex version\n\n# spin up your cluster using the new CLI version\ncortex cluster up cluster.yaml --configure-env cortex2\n```\n\nYou can use different Cortex CLIs to interact with the different versioned clusters; here is an example:\n\n<!-- CORTEX_VERSION_README x4 -->\n\n```bash\n# download the desired CLI version, replace 0.42.1 with the desired version (Note the \"v\"):\nCORTEX_INSTALL_PATH=$(pwd)/cortex0.42.1 bash -c \"$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)\"\n\n# confirm cortex CLI version\n./cortex0.42.1 version\n```\n\n### Deploy the APIs to your new cluster\n\nPlease read the [changelogs](https://github.com/cortexlabs/cortex/releases) and the latest documentation to identify any features and breaking changes in the new version. You may need to make modifications to your cluster and/or API configuration files.\n\n```bash\ncortex deploy -e cortex2 <api_spec_file>\n```\n\nAfter you've updated the API specifications and images if necessary, you can deploy them onto your new cluster.\n\n### Point your custom domain to your new cluster\n\nVerify that all of the APIs in your new cluster are working as expected by accessing via the cluster's API load balancer URL.\n\nGet the cluster's API load balancer URL:\n\n```bash\ncortex cluster info --name <new_cluster_name> --region <region>\n```\n\nOnce the APIs on the new cluster have been verified as working properly, it is recommended to update `min_replicas` of your APIs on the new cluster to match the current values in your previous cluster. This will avoid large sudden scale-up events as traffic is shifted to the new cluster.\n\nThen, navigate to the A record in your custom domains's Route 53 hosted zone and update the Alias to point the new cluster's API load balancer URL. Rather than suddenly routing all of your traffic from the previous cluster to the new cluster, you can use [weighted records](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-policy.html#routing-policy-weighted) to incrementally route more traffic to your new cluster.\n\nIf you increased `min_replicas` for your APIs in the new cluster during the transition, you may reduce `min_replicas` back to your desired level once all traffic has been shifted.\n\n### Spin down the previous cluster\n\nAfter confirming that your previous cluster has completed servicing all existing traffic and is not receiving any new traffic, spin down your previous cluster:\n\n```bash\n# Note: it is recommended to install the Cortex CLI matching the previous cluster's version to ensure proper deletion.\n\ncortex cluster down --name <previous_cluster_name> --region <region>\n```\n"
  },
  {
    "path": "docs/clusters/networking/api-gateway.md",
    "content": "# API Gateway\n\nThis guide shows how set up AWS API Gateway for your Cortex APIs, which is the simplest way to enable HTTPS if a custom domain is not required. See [here](https.md) for how to set up a custom domain with SSL certificates instead.\n\nPlease note that one limitation of API Gateway is that there is a 30-second time limit for all requests.\n\nIf your API load balancer is internet-facing (which is the default, or you set `api_load_balancer_scheme: internet-facing` in your cluster configuration file before creating your cluster), use the [first section](#internet-facing-load-balancer) of this guide.\n\nIf your API load balancer is internal (i.e. you set `api_load_balancer_scheme: internal` in your cluster configuration file before creating your cluster), use the [second section](#internal-load-balancer) of this guide.\n\n## Internet-facing load balancer\n\n_This section applies if your API load balancer is internet-facing (which is the default, or you set `api_load_balancer_scheme: internet-facing` in your cluster configuration file before creating your cluster). If your API load balancer is internal, see the [internal load balancer](#internal-load-balancer) section below._\n\nThere are two types of API Gateways that can be used when your load balancer is internet-facing: HTTP and REST. HTTP APIs are faster and less expensive, while REST APIs have more features an allow for more control. See the following section for creating an [HTTP API](#http-api), or skip to the next section for creating a [REST API](#rest-api).\n\n### HTTP API\n\n#### Create an API Gateway\n\nGo to the [API Gateway console](https://console.aws.amazon.com/apigateway/home), select \"HTTP API\" under \"Choose an API type\", and click \"Build\".\n\n![](https://user-images.githubusercontent.com/808475/125668597-35ad8d8e-8b5f-4274-bbb3-62ff47e5d544.png)\n\nClick \"Add integration\".\n\n![](https://user-images.githubusercontent.com/808475/125668635-f92df672-f516-45e0-a152-538aff901c0d.png)\n\nClick the drop-down menu, and select \"HTTP\".\n\n![](https://user-images.githubusercontent.com/808475/125668655-78754a51-c77a-4a03-a548-78ab991d1486.png)\n\nSet the \"URL endpoint\" to `http://API_LOAD_BALANCER_ENDPOINT/{proxy}`. You can get your API load balancer endpoint via `cortex cluster info`; make sure to prepend `http://` and append `/{proxy}`. For example, mine is: `http://aa9f5fdabfa6446aca53a526f59bc3c5-18cd00a628421fa3.elb.us-east-1.amazonaws.com/{proxy}`.\n\nChoose a name for your API (e.g. \"cortex\"), and click \"Next\".\n\n![](https://user-images.githubusercontent.com/808475/125668735-b57976c7-f68e-4a28-ab2a-92cfad5b49bb.png)\n\nOn the next page, update the pre-populated route's resource path to `/{proxy+}`, and click \"Next\".\n\n![](https://user-images.githubusercontent.com/808475/125668752-21aef8dd-1e75-41a5-bfc5-1e7157efbe01.png)\n\nClick \"Next\" again.\n\n![](https://user-images.githubusercontent.com/808475/125668761-8a177f6e-1977-48a2-be84-4b0df0105283.png)\n\nClick \"Create\".\n\n![](https://user-images.githubusercontent.com/808475/125668770-69fd3fbe-24e4-4a8a-8413-d50c779392c5.png)\n\nCopy your \"Invoke URL\" for the `$default` stage\n\n![](https://user-images.githubusercontent.com/808475/125668795-b18b564c-7091-432c-a4fd-a5a8bb157946.png)\n\n#### Use your new endpoint\n\nYou may now use the \"Invoke URL\" in place of your API load balancer endpoint in your client. For example, this curl request:\n\n```bash\ncurl -X POST http://aa9f5fdabfa6446aca53a526f59bc3c5-18cd00a628421fa3.elb.us-east-1.amazonaws.com/hello-world\n```\n\nWould become:\n\n```bash\ncurl -X POST https://nj3f5l96oe.execute-api.us-east-1.amazonaws.com/hello-world\n```\n\n#### Cleanup\n\nDelete the API Gateway before spinning down your Cortex cluster:\n\n![](https://user-images.githubusercontent.com/808475/125668816-83bce6bf-cf0e-4835-8d18-d641eb6cdb29.png)\n\n### REST API\n\n#### Create an API Gateway\n\nGo to the [API Gateway console](https://console.aws.amazon.com/apigateway/home), select \"REST API\" under \"Choose an API type\", and click \"Build\".\n\n![](https://user-images.githubusercontent.com/808475/78293216-18269e80-74dd-11ea-9e68-86922c2cbc7c.png)\n\nSelect \"REST\" and \"New API\", name your API (e.g. \"cortex\"), select either \"Regional\" or \"Edge optimized\" (depending on your preference), and click \"Create API\".\n\n![](https://user-images.githubusercontent.com/808475/78293434-66d43880-74dd-11ea-92d6-692158171a3f.png)\n\nSelect \"Actions\" > \"Create Resource\":\n\n![](https://user-images.githubusercontent.com/808475/80154502-8b6b7f80-8574-11ea-9c78-7d9f277bf55b.png)\n\nSelect \"Configure as proxy resource\" and \"Enable API Gateway CORS\", and click \"Create Resource\"\n\n![](https://user-images.githubusercontent.com/808475/80154565-ad650200-8574-11ea-8753-808cd35902e2.png)\n\nSelect \"HTTP Proxy\" and set \"Endpoint URL\" to `http://API_LOAD_BALANCER_ENDPOINT/{proxy}`. You can get your API load balancer endpoint via `cortex cluster info`; make sure to prepend `http://` and append `/{proxy}`. For example, mine is: `http://a9eaf69fd125947abb1065f62de59047-81cdebc0275f7d96.elb.us-west-2.amazonaws.com/{proxy}`.\n\nLeave \"Content Handling\" set to \"Passthrough\" and Click \"Save\".\n\n![](https://user-images.githubusercontent.com/808475/80154735-13ea2000-8575-11ea-83ca-58f182df83c6.png)\n\nSelect \"Actions\" > \"Deploy API\"\n\n![](https://user-images.githubusercontent.com/808475/80154802-2c5a3a80-8575-11ea-9ab3-de89885fd658.png)\n\nCreate a new stage (e.g. \"dev\") and click \"Deploy\"\n\n![](https://user-images.githubusercontent.com/808475/80154859-4431be80-8575-11ea-9305-50384b1f9847.png)\n\nCopy your \"Invoke URL\"\n\n![](https://user-images.githubusercontent.com/808475/80154911-5dd30600-8575-11ea-9682-1a7328783011.png)\n\n#### Use your new endpoint\n\nYou may now use the \"Invoke URL\" in place of your API load balancer endpoint in your client. For example, this curl request:\n\n```bash\ncurl -X POST http://a9eaf69fd125947abb1065f62de59047-81cdebc0275f7d96.elb.us-west-2.amazonaws.com/hello-world\n```\n\nWould become:\n\n```bash\ncurl -X POST https://31qjv48rs6.execute-api.us-west-2.amazonaws.com/dev/hello-world\n```\n\n#### Cleanup\n\nDelete the API Gateway before spinning down your Cortex cluster:\n\n![](https://user-images.githubusercontent.com/808475/80155073-bdc9ac80-8575-11ea-99a1-95c0579da79e.png)\n\n## Internal load balancer\n\n_This section applies if your API load balancer is internal (i.e. you set `api_load_balancer_scheme: internal` in your cluster configuration file before creating your cluster). If your API load balancer is internet-facing, see the [internet-facing load balancer](#internet-facing-load-balancer) section above._\n\n### Create a VPC Link\n\nNavigate to AWS's EC2 Load Balancer dashboard and locate the Cortex API load balancer. You can determine which is the API load balancer by inspecting the `kubernetes.io/service-name` tag:\n\n![](https://user-images.githubusercontent.com/808475/80142777-961c1980-8560-11ea-9202-40964dbff5e9.png)\n\nTake note of the load balancer's name.\n\nGo to the [API Gateway console](https://console.aws.amazon.com/apigateway/home), click \"VPC Links\" on the left sidebar, and click \"Create\"\n\n![](https://user-images.githubusercontent.com/808475/80142466-0c6c4c00-8560-11ea-8293-eb5e5572b797.png)\n\nSelect \"VPC link for REST APIs\", name your VPC link (e.g. \"cortex\"), select the API load balancer, and click \"Create\".\n\n![](https://user-images.githubusercontent.com/808475/80143027-03c84580-8561-11ea-92de-9ed0a5dfa593.png)\n\nWait for the VPC link to be created (it will take a few minutes)\n\n![](https://user-images.githubusercontent.com/808475/80144088-bbaa2280-8562-11ea-901b-8520eb253df7.png)\n\n### Create an API Gateway\n\nGo to the [API Gateway console](https://console.aws.amazon.com/apigateway/home), select \"REST API\" under \"Choose an API type\", and click \"Build\"\n\n![](https://user-images.githubusercontent.com/808475/78293216-18269e80-74dd-11ea-9e68-86922c2cbc7c.png)\n\nSelect \"REST\" and \"New API\", name your API (e.g. \"cortex\"), select either \"Regional\" or \"Edge optimized\" (depending on your preference), and click \"Create API\"\n\n![](https://user-images.githubusercontent.com/808475/78293434-66d43880-74dd-11ea-92d6-692158171a3f.png)\n\nSelect \"Actions\" > \"Create Resource\"\n\n![](https://user-images.githubusercontent.com/808475/80141938-3cffb600-855f-11ea-9c1c-132ca4503b7a.png)\n\nSelect \"Configure as proxy resource\" and \"Enable API Gateway CORS\", and click \"Create Resource\"\n\n![](https://user-images.githubusercontent.com/808475/80142124-80f2bb00-855f-11ea-8e4e-9413146e0815.png)\n\nSelect \"VPC Link\", select \"Use Proxy Integration\", choose your newly-created VPC Link, and set \"Endpoint URL\" to `http://API_LOAD_BALANCER_ENDPOINT/{proxy}`. You can get your API load balancer endpoint via `cortex cluster info`; make sure to prepend `http://` and append `/{proxy}`. For example, mine is: `http://a5044e34a352d44b0945adcd455c7fa3-32fa161d3e5bcbf9.elb.us-west-2.amazonaws.com/{proxy}`. Click \"Save\"\n\n![](https://user-images.githubusercontent.com/808475/80147407-4f322200-8568-11ea-8ef5-df5164c1375f.png)\n\nSelect \"Actions\" > \"Deploy API\"\n\n![](https://user-images.githubusercontent.com/808475/80147555-86083800-8568-11ea-86af-1b1e38c9d322.png)\n\nCreate a new stage (e.g. \"dev\") and click \"Deploy\"\n\n![](https://user-images.githubusercontent.com/808475/80147631-a7692400-8568-11ea-8a09-13dbd50b17b9.png)\n\nCopy your \"Invoke URL\"\n\n![](https://user-images.githubusercontent.com/808475/80147716-c798e300-8568-11ea-9aef-7dd6fdf4a68a.png)\n\n### Use your new endpoint\n\nYou may now use the \"Invoke URL\" in place of your API load balancer endpoint in your client. For example, this curl request:\n\n```bash\ncurl -X POST http://a5044e34a352d44b0945adcd455c7fa3-32fa161d3e5bcbf9.elb.us-west-2.amazonaws.com/hello-world\n```\n\nWould become:\n\n```bash\ncurl -X POST https://lrivodooqh.execute-api.us-west-2.amazonaws.com/dev/hello-world\n```\n\n### Cleanup\n\nDelete the API Gateway and VPC Link before spinning down your Cortex cluster:\n\n![](https://user-images.githubusercontent.com/808475/80149163-05970680-856b-11ea-9f82-61f4061a3321.png)\n\n![](https://user-images.githubusercontent.com/808475/80149204-1ba4c700-856b-11ea-83f7-9741c78b6b95.png)\n"
  },
  {
    "path": "docs/clusters/networking/custom-domain.md",
    "content": "# Custom domain\n\nYou can set up DNS to use a custom domain for your Cortex APIs. For example, you can make your API accessible via `api.example.com/hello-world`.\n\nThis guide will demonstrate how to create a dedicated subdomain in AWS Route 53. After completing this guide, if you want to enable HTTPS with your custom subdomain, see [these instructions](https.md).\n\n## Configure DNS\n\nDecide on a subdomain that you want to dedicate to Cortex APIs. For example if your domain is `example.com`, a valid subdomain can be `api.example.com`. This guide will use `cortexlabs.dev` as the domain and `api.cortexlabs.dev` as the subdomain.\n\nWe will set up a hosted zone on Route 53 to manage the DNS records for the subdomain. Go to the [Route 53 console](https://console.aws.amazon.com/route53/home) and click \"Hosted Zones\".\n\n![](https://user-images.githubusercontent.com/4365343/82210754-a6b07d00-98dd-11ea-9cec-9f6b07282aa8.png)\n\nClick \"Create Hosted Zone\" and then enter your subdomain as the domain name for your hosted zone and click \"Create\".\n\n![](https://user-images.githubusercontent.com/4365343/82211091-4968fb80-98de-11ea-8ec4-8d26d1aea77a.png)\n\nTake note of the values in the NS record.\n\n![](https://user-images.githubusercontent.com/4365343/82211656-386cba00-98df-11ea-8c86-4961082b5f49.png)\n\nNavigate to your root DNS service provider (e.g. Google Domains, AWS Route 53, Go Daddy). Your root DNS service provider is typically the registrar where you purchased your domain (unless you have transferred DNS management elsewhere). The procedure for adding DNS records may vary based on your service provider.\n\nWe are going to add an NS (name server) record that specifies that any traffic to your subdomain should use the name servers of your hosted zone in Route 53 for DNS resolution.\n\n`cortexlabs.dev` is managed by Google Domains. The image below is a screenshot for adding a DNS record in Google Domains (your UI may differ based on your DNS service provider).\n\n![](https://user-images.githubusercontent.com/808475/109039458-abcb0580-7681-11eb-8644-76436328687e.png)\n\n## Add DNS record\n\nNavigate to your [EC2 Load Balancer console](https://us-west-2.console.aws.amazon.com/ec2/v2/home#LoadBalancers:sort=loadBalancerName) and locate the Cortex API load balancer. You can determine which is the API load balancer by inspecting the `kubernetes.io/service-name` tag.\n\nTake note of the load balancer's name.\n\n![](https://user-images.githubusercontent.com/808475/80142777-961c1980-8560-11ea-9202-40964dbff5e9.png)\n\nGo back to the [Route 53 console](https://console.aws.amazon.com/route53/home#hosted-zones:) and select the hosted zone you created earlier. Click \"Create Record Set\", and add an Alias record that routes traffic to your Cortex cluster's API load balancer (leave \"Name\" blank).\n\n![](https://user-images.githubusercontent.com/808475/84083422-6ac97e80-a996-11ea-9679-be37268a2133.png)\n\n## Debugging connectivity issues\n\nYou could run into connectivity issues if you make a request to your API without waiting long enough for your DNS records to propagate after creating them (it usually takes 5-10 minutes). If you are updating existing DNS records, it could take anywhere from a few minutes to 48 hours for the DNS cache to expire (until then, your previous DNS configuration will be used).\n\nTo test connectivity, try the following steps:\n\n1. Deploy an api.\n1. Make a request to your api (e.g. `curl http://api.cortexlabs.dev/hello-world` or paste the url into your browser if your API supports GET requests).\n1. If you run into an error such as `curl: (6) Could not resolve host: api.cortexlabs.dev` wait a few minutes and make the request from another device that hasn't made a request to that url in a while.\n\n## Cleanup\n\nSpin down your Cortex cluster.\n\nDelete the hosted zone for your subdomain in the [Route 53 console](https://console.aws.amazon.com/route53/home#hosted-zones:):\n\n![](https://user-images.githubusercontent.com/4365343/82228729-81306d00-98f7-11ea-8570-e9de15f5267f.png)\n"
  },
  {
    "path": "docs/clusters/networking/https.md",
    "content": "# Setting up HTTPS\n\nThis guide shows how to support HTTPS traffic to Cortex APIs via a custom domain. It is also possible to use AWS API Gateway to enable HTTPS without using your own domain (see [here](api-gateway.md) for instructions).\n\nIn order to create a valid SSL certificate for your domain, you must have the ability to configure DNS to satisfy the DNS challenges which prove that you own the domain. This guide assumes that you are using a Route 53 hosted zone to manage a subdomain. Follow this [guide](./custom-domain.md) to set up a subdomain managed by a Route 53 hosted zone.\n\n## Generate an SSL certificate\n\nTo create an SSL certificate, go to the [ACM console](https://us-west-2.console.aws.amazon.com/acm/home) and click \"Get Started\" under the \"Provision certificates\" section.\n\n![](https://user-images.githubusercontent.com/4365343/82202340-c04ac800-98cf-11ea-9472-89dd6d67eb0d.png)\n\nSelect \"Request a public certificate\" and then \"Request a certificate\".\n\n![](https://user-images.githubusercontent.com/4365343/82202654-3e0ed380-98d0-11ea-8c57-025f0b69c54f.png)\n\nEnter your subdomain and then click \"Next\".\n\n![](https://user-images.githubusercontent.com/4365343/82224652-1cbedf00-98f2-11ea-912b-466cee2f6e25.png)\n\nSelect \"DNS validation\" and then click \"Next\".\n\n![](https://user-images.githubusercontent.com/4365343/82205311-66003600-98d4-11ea-90e3-da7e8b0b2b9c.png)\n\nAdd tags for searchability (optional) then click \"Review\".\n\n![](https://user-images.githubusercontent.com/4365343/82206485-52ee6580-98d6-11ea-95a9-1d0ebafc178a.png)\n\nClick \"Confirm and request\".\n\n![](https://user-images.githubusercontent.com/4365343/82206602-84ffc780-98d6-11ea-9f2f-ce383404ec67.png)\n\nClick \"Create record in Route 53\". A popup will appear indicating that a Record is going to be added to Route 53. Click \"Create\" to automatically add the DNS record to your subdomain's hosted zone. Then click \"Continue\".\n\n![](https://user-images.githubusercontent.com/4365343/82223539-c8ffc600-98f0-11ea-93a2-044aa0c9670d.png)\n\nWait for the Certificate Status to be \"issued\". This might take a few minutes.\n\n![](https://user-images.githubusercontent.com/4365343/82209663-a616e700-98db-11ea-95cb-c6efedadb942.png)\n\nTake note of the certificate's ARN. The certificate is ineligible for renewal because it is currently not being used. It will be eligible for renewal once it's used in Cortex.\n\n![](https://user-images.githubusercontent.com/4365343/82222684-9e613d80-98ef-11ea-98c0-5a20b457f062.png)\n\n## Create or update your cluster\n\nAdd the following field to your cluster configuration:\n\n```yaml\n# cluster.yaml\n\n...\n\nssl_certificate_arn: <ARN of your certificate>\n```\n\nCreate a cluster:\n\n```bash\ncortex cluster up cluster.yaml\n```\n\nOr update an existing cluster:\n\n```bash\ncortex cluster configure cluster.yaml\n```\n\n## Use your new endpoint\n\nWait a few minutes to allow the DNS changes to propagate. You may now use your subdomain in place of your API load balancer endpoint in your client. For example, this curl request:\n\n```bash\ncurl http://a5044e34a352d44b0945adcd455c7fa3-32fa161d3e5bcbf9.elb.us-west-2.amazonaws.com/hello-world -X POST -H \"Content-Type: application/json\" -d @sample.json\n```\n\nWould become:\n\n```bash\n# add the `-k` flag or use http:// instead of https:// if you didn't configure an SSL certificate\ncurl https://api.cortexlabs.dev/hello-world -X POST -H \"Content-Type: application/json\" -d @sample.json\n```\n\n## Cleanup\n\nSpin down your Cortex cluster.\n\nIf you created an SSL certificate, delete it from the [ACM console](https://us-west-2.console.aws.amazon.com/acm/home):\n\n![](https://user-images.githubusercontent.com/4365343/82228835-a624e000-98f7-11ea-92e2-cb4fb0f591e2.png)\n"
  },
  {
    "path": "docs/clusters/networking/load-balancers.md",
    "content": "# Load balancers\n\n![api architecture diagram](https://user-images.githubusercontent.com/808475/103417256-dd6e9700-4b3e-11eb-901e-90425f1f8fd4.png)\n\nAll APIs share a single API load balancer. By default, the API load balancer is public. You can configure your API load balancer to be private by setting `api_load_balancer_scheme: internal` in your cluster configuration file (before creating your cluster). This will make your API only accessible through [VPC Peering](vpc-peering.md). You can enforce that incoming requests to APIs must originate from specific ip address ranges by specifying `api_load_balancer_cidr_white_list: [<CIDR list>]` in your cluster configuration.\n\nThe SSL certificate on the API load balancer is autogenerated during installation using `localhost` as the Common Name (CN). Therefore, clients will need to skip certificate verification when making HTTPS requests to your APIs (e.g. `curl -k https://***`), or make HTTP requests instead (e.g. `curl http://***`). Alternatively, you can enable HTTPS by using a [custom domain](custom-domain.md) and setting up [https](https.md) or by [creating an API Gateway](api-gateway.md) to forward requests to your API load balancer.\n\nThere is a separate load balancer for the Cortex operator. By default, the operator load balancer is public. You can configure your operator load balancer to be private by setting `operator_load_balancer_scheme: internal` in your cluster configuration file (before creating your cluster). You can use [VPC Peering](vpc-peering.md) to enable your Cortex CLI to connect to your cluster operator from another VPC. You can enforce that incoming requests to the Cortex operator must originate from specific ip address ranges by specifying `operator_load_balancer_cidr_white_list: [<CIDR list>]` in your cluster configuration.\n\nBy default, the API load balancer and Operator load balancer are both [Network load balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) (NLB). The api load balancer can be configured as a [Classic load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/introduction.html) (ELB) instead if desired. The API load balancer type must be specified before creating your cluster.\n"
  },
  {
    "path": "docs/clusters/networking/vpc-peering.md",
    "content": "# VPC peering\n\nIf you are using an internal operator load balancer (i.e. you set `operator_load_balancer_scheme: internal` in your cluster configuration file before creating your cluster), you can use VPC Peering to enable your Cortex CLI to connect to your cluster operator from another VPC so that you may run `cortex` commands.\n\nIf you are using an internal API load balancer (i.e. you set `api_load_balancer_scheme: internal` in your cluster configuration file before creating your cluster), you can use VPC Peering to make requests from another VPC.\n\nThis guide illustrates how to create a VPC Peering connection between a VPC of your choice and the Cortex load balancers.\n\n## Gather Cortex's VPC information\n\nNavigate to AWS's EC2 Load Balancer dashboard and locate the Cortex operator's load balancer. You can determine which is the operator load balancer by inspecting the `kubernetes.io/service-name` tag:\n\n![](https://user-images.githubusercontent.com/808475/80126132-804e2a80-8547-11ea-8ce4-57d3fd96e2c4.png)\n\nClick back to the \"Description\" tab and note the VPC ID of the load balancer and the ID of each of the subnets associated with the load balancer:\n\n![](https://user-images.githubusercontent.com/808475/80127144-c2c43700-8548-11ea-95b4-ce9d1df024cc.png)\n\nNavigate to AWS's VPC dashboard and identify the ID and CIDR block of Cortex's VPC:\n\n![](https://user-images.githubusercontent.com/808475/80125554-af17d100-8546-11ea-96ec-00e2aaee7100.png)\n\nThe VPC ID here should match that of the load balancer.\n\n## Create peering connection\n\nIdentify the ID and CIDR block of the VPC from which you'd like to connect to the Cortex VPC.\n\nIn my case, I have a VPC in the same AWS account and region, and I can locate its ID and CIDR block from AWS's VPC dashboard:\n\n![](https://user-images.githubusercontent.com/808475/80125729-eb4b3180-8546-11ea-8d20-6bc2478747ae.png)\n\nFrom AWS's VPC dashboard, navigate to the \"Peering Connections\" page, and click \"Create Peering Connection\":\n\n![](https://user-images.githubusercontent.com/808475/80127600-67df0f80-8549-11ea-9e10-765a6e273b54.png)\n\nName your new VPC Peering Connection (I used \"cortex-operator\", but \"cortex\" or \"cortex-api\" may make more sense depending on your use case). Then configure the connection such that the \"Requester\" is the VPC from which you'll connect to the Cortex VPC, and the \"Accepter\" is Cortex's VPC.\n\n![](https://user-images.githubusercontent.com/808475/80131545-3f5a1400-854f-11ea-9ca0-c51433d3fa3d.png)\n\nClick \"Create Peering Connection\", navigate back to the Peering Connections dashboard, select the newly created peering connection, and click \"Actions\" > \"Accept Request\":\n\n![](https://user-images.githubusercontent.com/808475/80132168-21d97a00-8550-11ea-8c22-79c65710d369.png)\n\n![](https://user-images.githubusercontent.com/808475/80132179-26059780-8550-11ea-80fc-6670fcab7026.png)\n\n## Update route tables\n\nNavigate to the VPC Route Tables page. Select the route table for the VPC from which you'd like to connect to the Cortex cluster (in my case, I just have one route table for this VPC). Select the \"Routes\" tab, and click \"Edit routes\":\n\n![](https://user-images.githubusercontent.com/808475/80135180-b940cc00-8554-11ea-8162-c7409090897b.png)\n\nAdd a route where the \"Destination\" is the CIDR block for Cortex's VPC, and the \"Target\" is the newly-created Peering Connection:\n\n![](https://user-images.githubusercontent.com/808475/80137033-78968200-8557-11ea-9d84-9221b772f0fc.png)\n\nDo not create new route tables or change subnet associations.\n\nNavigate back to the VPC Route Tables page. There will be a route table for each of the subnets associated with the Cortex operator load balancer:\n\n![](https://user-images.githubusercontent.com/808475/80138244-5dc50d00-8559-11ea-9248-fc201d011530.png)\n\nFor each of these route tables, click \"Edit routes\" and add a new route where the \"Destination\" is the CIDR block for the VPC from which you will be connecting to the Cortex cluster, and the \"Target\" is the newly-created Peering Connection:\n\n![](https://user-images.githubusercontent.com/808475/80138653-f78cba00-8559-11ea-8444-406e218c3bab.png)\n\nRepeat adding this route for each route table associated with the Cortex operator's subnets; in my case there were three. Do not create new route tables or change subnet associations.\n\nYou should now be able to use the Cortex CLI and make requests from your VPC.\n\n## Cleanup\n\nDelete the VPC Peering connection before spinning down your Cortex cluster:\n\n![](https://user-images.githubusercontent.com/808475/80138851-57836080-855a-11ea-92f1-06d501932a41.png)\n"
  },
  {
    "path": "docs/clusters/observability/alerting.md",
    "content": "# Alerting\n\nCortex supports setting alerts for your APIs out-of-the-box. Alerts are an effective way of identifying problems in your system as they occur.\n\nThe following dashboards can be configured with alerts:\n\n- RealtimeAPI\n- BatchAPI\n- Cluster resources\n- Node resources\n\nThis page demonstrates the process for configuring alerts for a realtime API. The same principles apply to the others as well. Alerts can be configured for a variety of notification channels such as for Slack, Discord, Microsoft Teams, PagerDuty, Telegram and traditional webhooks. In this example, we'll use Slack.\n\nIf you don't know how to access the Grafana dashboard for your API, make sure you check out [this page](metrics.md) first.\n\n## Create a Slack channel\n\nCreate a slack channel on your team's Slack workspace. We'll name ours \"cortex-alerts\".\n\nAdd an _Incoming Webhook_ to your channel and retrieve the webhook URL. It will look like something like `https://hooks.slack.com/services/<XXX>/<YYY>/<ZZZ>`.\n\n## Create a Grafana notification channel\n\nGo to Grafana and on the left-hand side panel, hover over the alerting bell and select _\"Notification channels\"_.\n\n![](https://user-images.githubusercontent.com/26958764/114937638-b6667780-9e46-11eb-963a-8a53e5655c3d.png)\n\nClick on _\"Create channel\"_, add the name of your channel, select _Slack_ as the type of the channel, and paste your Slack webhook URL.\n\n![](https://user-images.githubusercontent.com/26958764/114937856-06ddd500-9e47-11eb-8f47-47b043b0bb5c.png)\n\nClick _\"Test\"_ and see if a sample notification is sent to your Slack channel. If the message goes through, click on _\"Save\"_.\n\n![](https://user-images.githubusercontent.com/26958764/114938358-b2872500-9e47-11eb-87aa-ee818aae4cd0.png)\n\n## Create alerts\n\nNow that the notification channel is functioning properly, we can create alerts for our APIs and cluster. For all of our examples, we are using the `mpg-estimator` API as an example.\n\n![](https://user-images.githubusercontent.com/26958764/114939831-a8662600-9e49-11eb-8774-fbac3ce627d9.png)\n\n### API replica threshold alert\n\nLet's create an alert for the _\"Active Replicas\"_ panel. We want to send notifications every time the number of replicas for the given API exceeds a certain threshold.\n\nEdit the _\"Active Replicas\"_ panel.\n\n![](https://user-images.githubusercontent.com/26958764/114941416-d2b8e300-9e4b-11eb-8def-ee64535fc799.png)\n\nCreate a copy of the primary query by clicking on the \"duplicate\" icon.\n\n![](https://user-images.githubusercontent.com/26958764/114941557-fe3bcd80-9e4b-11eb-8f69-b9d43ff8eb28.png)\n\nIn the copied query, replace the `$api_name` variable with the name of the API you want to create the alert for. In our case, it's `mpg-estimator`. Also, click on the eye icon to disable the query from being shown on the graph - otherwise, you'll see duplicates.\n\n![](https://user-images.githubusercontent.com/26958764/114941701-275c5e00-9e4c-11eb-991b-34d660d0d05c.png)\n\nGo to the _\"Alert\"_ tab and click _\"Create Alert\"_.\n\n![](https://user-images.githubusercontent.com/26958764/114941779-40fda580-9e4c-11eb-9aee-514e6b4832ba.png)\n\nConfigure your alert like in the following example and click _\"Apply\"_.\n\n![](https://user-images.githubusercontent.com/26958764/114944749-d7cc6100-9e50-11eb-9b6b-b2c3dabcc78c.png)\n\nThe next time the threshold is exceeded, a notification will be sent to your Slack channel.\n\n![](https://user-images.githubusercontent.com/26958764/114948423-a3a86e80-9e57-11eb-8717-94e456a15298.png)\n\n### In-flight requests spike alert\n\nLet's add an alert on the _\"In-Flight Requests\"_ panel. We want to send an alert if the metric exceeds 50 in-flight requests. For this, follow the same set of instructions as for the previous alert, but this time configure the alert to match the following screenshot:\n\n![](https://user-images.githubusercontent.com/26958764/114949182-1bc36400-9e59-11eb-9c19-0d788872a388.png)\n\nAn alert triggered for this will look like:\n\n![](https://user-images.githubusercontent.com/26958764/114949593-000c8d80-9e5a-11eb-8cb5-b2c9a2b344e8.png)\n\n#### Memory usage alert\n\nLet's add another alert, this time for the _\"Avg Memory Usage\"_ panel. We want to send an alert if the average memory usage per API replica exceeds its memory request. For this, we need to follow the same set of instructions as for the first alert, but this time the hidden query needs to be expressed as the ratio between the memory usage and memory request:\n\n![](https://user-images.githubusercontent.com/26958764/114951903-f1c07080-9e5d-11eb-9aaf-898d46efb7ef.png)\n\nThe memory usage alert can to be defined like in the following screenshot:\n\n![](https://user-images.githubusercontent.com/26958764/114951782-bfaf0e80-9e5d-11eb-834d-e48ab3546d3c.png)\n\nThe resulting alert will look like this:\n\n![](https://user-images.githubusercontent.com/26958764/114952346-bd00e900-9e5e-11eb-879a-5851dab7630b.png)\n\n## Persistent changes\n\nTo save your changes permanently, go back to your dashboard and click on the save icon on the top-right corner.\n\n![](https://user-images.githubusercontent.com/26958764/114953264-af4c6300-9e60-11eb-8095-40e438c125d8.png)\n\nCopy the JSON to your clipboard.\n\n![](https://user-images.githubusercontent.com/26958764/114953338-d6a33000-9e60-11eb-8390-0f24704c5b7d.png)\n\nClick on the settings button on the top-right corner of your dashboard.\n\n![](https://user-images.githubusercontent.com/26958764/114953437-00f4ed80-9e61-11eb-91f6-4b669ffe0c16.png)\n\nGo to the _\"JSON Model\"_ section and replace the JSON with the one you've copied to your clipboard. Then click _\"Save Changes\"_.\n\n![](https://user-images.githubusercontent.com/26958764/114953473-1ec25280-9e61-11eb-8fcc-12615b73067a.png)\n\nYour dashboard now has stored the alert configuration permanently.\n\n## Multiple APIs alerts\n\nDue to how Grafana was built, you'll need to re-do the steps of setting a given alert for each individual API. That's because Grafana doesn't currently support alerts on template or transformation queries.\n\n## Enabling email alerts\n\nIt is possible to manually configure SMTP to enable email alerts (we plan on automating this process, see [#2210](https://github.com/cortexlabs/cortex/issues/2210)).\n\n**Step 1**\n\nInstall [kubectl](../advanced/kubectl.md).\n\n**Step 2**\n\n```bash\nkubectl create secret generic grafana-smtp \\\n    --from-literal=GF_SMTP_ENABLED=true \\\n    --from-literal=GF_SMTP_HOST=<SMTP-HOST> \\\n    --from-literal=GF_SMTP_USER=<EMAIL-ADDRESS> \\\n    --from-literal=GF_SMTP_FROM_ADDRESS=<EMAIL-ADDRESS> \\\n    --from-literal=GF_SMTP_PASSWORD=<EMAIL-PASSWORD>\n```\n\nThe `<SMTP-HOST>` varies from provider to provider (e.g. Gmail's is `smtp.gmail.com:587`).\n\n**Step 3**\n\nEdit Grafana's statefulset by running `kubectl edit statefulset grafana` (this will open a code editor). Inside the container named `grafana` (in the `containers` section), add an `envFrom` section that will mount the SMTP secret. Here is an example of what it looks like after the addition:\n\n<!-- CORTEX_VERSION_README -->\n\n```yaml\n# ...\ncontainers:\n- env:\n  - name: GF_SERVER_ROOT_URL\n    value: '%(protocol)s://%(domain)s:%(http_port)s/dashboard'\n  - name: GF_SERVER_SERVE_FROM_SUB_PATH\n    value: \"true\"\n  - name: GF_USERS_DEFAULT_THEME\n    value: light\n  envFrom:\n    - secretRef:\n        name: grafana-smtp\n  image: quay.io/cortexlabs/grafana:0.42.1\n  imagePullPolicy: IfNotPresent\n  name: grafana\n# ...\n```\n\nSave and close your editor.\n\nIt will take 30-60 seconds for Grafana to restart, after which you can access the dashboard. You can check the logs with `kubectl logs -f grafana-0`.\n"
  },
  {
    "path": "docs/clusters/observability/logging.md",
    "content": "# Logging\n\nLogs are collected with Fluent Bit and are exported to CloudWatch.\n\n## Logs on AWS\n\nLogs will automatically be pushed to CloudWatch and a log group with the same name as your cluster will be created to store your logs. API logs are tagged with labels to help with log aggregation and filtering. Log lines greater than 5 MB in size will be ignored.\n\nYou can use the `cortex logs` command to get a CloudWatch Insights URL of query to fetch logs for your API. Please note that there may be a few minutes of delay from when a message is logged to when it is available in CloudWatch Insights.\n\n**RealtimeAPI:**\n\n```text\nfields @timestamp, message\n| filter cortex.labels.apiName=\"<INSERT API NAME>\"\n| filter cortex.labels.apiKind=\"RealtimeAPI\"\n| sort @timestamp asc\n| limit 1000\n```\n\n**AsyncAPI:**\n\n```text\nfields @timestamp, message\n| filter cortex.labels.apiName=\"<INSERT API NAME>\"\n| filter cortex.labels.apiKind=\"AsyncAPI\"\n| sort @timestamp asc\n| limit 1000\n```\n\n**BatchAPI:**\n\n```text\nfields @timestamp, message\n| filter cortex.labels.apiName=\"<INSERT API NAME>\"\n| filter cortex.labels.jobID=\"<INSERT JOB ID>\"\n| filter cortex.labels.apiKind=\"BatchAPI\"\n| sort @timestamp asc\n| limit 1000\n```\n\n**TaskAPI:**\n\n```text\nfields @timestamp, message\n| filter cortex.labels.apiName=\"<INSERT API NAME>\"\n| filter cortex.labels.jobID=\"<INSERT JOB ID>\"\n| filter cortex.labels.apiKind=\"TaskAPI\"\n| sort @timestamp asc\n| limit 1000\n```\n\n## Streaming logs from the CLI\n\nYou can stream logs directly from a random pod of a running workload to iterate and debug quickly. These logs will not be as comprehensive as the logs that are available in CloudWatch.\n\n```bash\n# RealtimeAPI\ncortex logs --random-pod <api_name>\n\n# BatchAPI or TaskAPI\ncortex logs --random-pod <api_name> <job_id>  # the job must be in a running state\n```\n\n## Structured logging\n\nIf you log JSON strings from your APIs, they will be automatically parsed before pushing to CloudWatch.\n\nIt is recommended to configure your JSON logger to use `message` or `msg` as the key for the log line if you would like the sample queries above to display the messages in your logs.\n\nAvoid using top-level keys that start with \"cortex\" to prevent collisions with Cortex's internal logging.\n\n## Exporting logs\n\nYou can export both the Cortex system logs and your application logs to your desired destination by configuring FluentBit.\n\n### Configure kubectl\n\nFollow these [instructions](../advanced/kubectl.md) to set up kubectl.\n\n### Find supported destinations in FluentBit\n\nVisit FluentBit's [output docs](https://docs.fluentbit.io/manual/concepts/data-pipeline/output) to see a list supported destinations.\n\nMake sure to navigate to the version of FluentBit being used in your cluster. You can find the version of FluentBit by looking at the first view lines of one of the FluentBit pod logs.\n\nGet the FluentBit pods:\n\n```bash\nkubectl get pods --selector app=fluent-bit\n```\n\nFluentBit's version should be in the first few log lines of a FluentBit pod:\n\n```bash\nkubectl logs fluent-bit-kxmzn | head -n 20\n```\n\n### Update FluentBit configuration\n\nDefine `patch.yaml` with your new output configuration:\n\n```yaml\ndata:\n  output.conf: |\n    [OUTPUT]\n        Name              es\n        Match             k8s_container.*\n        Host              https://abc123.us-west-2.es.amazonaws.com\n        Port              443\n        AWS_Region        us-west-2\n        AWS_Auth          On\n        tls               On\n        Logstash_Format   On\n        Logstash_Prefix   my-logs\n```\n\nUpdate FluentBit's configuration:\n\n```bash\nkubectl patch configmap fluent-bit-config --patch-file patch.yaml --type merge\n```\n\n### Restart FluentBit\n\nRestart FluentBit to apply the new configuration:\n\n```bash\nkubectl rollout restart daemonset/fluent-bit\n```\n"
  },
  {
    "path": "docs/clusters/observability/metrics.md",
    "content": "# Metrics\n\nCortex includes Prometheus for metrics collection and Grafana for visualization. You can monitor your APIs with the default Grafana dashboards, or create custom metrics and dashboards.\n\n## Accessing the dashboard\n\nThe dashboard URL is displayed once you run a `cortex get <api_name>` command.\n\nAlternatively, you can access it on `http://<operator_url>/dashboard`. Run the following command to get the operator\nURL:\n\n```bash\ncortex env list\n```\n\nIf your operator load balancer is configured to be internal, there are a few options for accessing the dashboard:\n\n1. Access the dashboard from a machine that has VPC Peering configured to your cluster's VPC, or which is inside of your\n   cluster's VPC.\n1. Run `kubectl port-forward -n default grafana-0 3000:3000` to forward Grafana's port to your local machine, and access\n   the dashboard on [http://localhost:3000](http://localhost:3000) (see instructions for setting up `kubectl` [here](../advanced/kubectl.md)).\n1. Set up VPN access to your cluster's\n   VPC ([docs](https://docs.aws.amazon.com/vpc/latest/userguide/vpn-connections.html)).\n\n### Default credentials\n\nThe dashboard is protected with username / password authentication, which by default are:\n\n- Username: admin\n- Password: admin\n\nYou will be prompted to change the admin user password in the first time you log in.\n\nGrafana allows managing the access of several users and managing teams. For more information on this topic check\nthe [grafana documentation](https://grafana.com/docs/grafana/latest/manage-users).\n\n### Selecting an API\n\nYou can select one or more APIs to visualize in the top left corner of the dashboard.\n\n![](https://user-images.githubusercontent.com/7456627/107375721-57545180-6ae9-11eb-9474-ba58ad7eb0c5.png)\n\n### Selecting a time range\n\nGrafana allows you to select a time range on which the metrics will be visualized. You can do so in the top right corner\nof the dashboard.\n\n![](https://user-images.githubusercontent.com/7456627/107376148-d9dd1100-6ae9-11eb-8c2b-c678b41ade01.png)\n\n**Note: Cortex only retains a maximum of 2 weeks worth of data at any moment in time**\n\n### Available dashboards\n\nThere are more than one dashboard available by default. You can view the available dashboards by accessing the Grafana\nmenu: `Dashboards -> Manage -> Cortex folder`.\n\nThe dashboards that Cortex ships with are the following:\n\n- RealtimeAPI\n- BatchAPI\n- Cluster resources\n- Node resources\n\n### Available metrics\n\nCortex exposes additional metrics with Prometheus. To view all available metrics, navigate to the `Explore` menu in Grafana and click the `Metrics` button.\n\n![](https://user-images.githubusercontent.com/7456627/107377492-515f7000-6aeb-11eb-9b46-909120335060.png)\n\nYou can use any of these metrics to set up your own dashboards.\n\n## Exporting metrics to monitoring solutions\n\nYou can scrape metrics from the in-cluster Prometheus server via the `/federate` endpoint and push them to monitoring solutions such as Datadog.\n\nThe steps for exporting metrics from Prometheus will vary based on your monitoring solution. Here are a few high-level steps to get you started. We will be using Datadog as an example; feel free to reach out to us on [Slack](https://community.cortexlabs.com/) if you need help setting up your monitoring tool.\n\n### Configure kubectl\n\nFollow these [instructions](../advanced/kubectl.md) to set up kubectl.\n\n### Install agent\n\nMonitoring solutions provide Kubernetes agents that are capable of scraping Prometheus metrics. Follow the appropriate instructions to install the agent onto your cluster (here are the [instructions](https://docs.datadoghq.com/agent/kubernetes/?tab=helm#installation) for Datadog).\n\n### Scrape Prometheus\n\nSome agents require a Prometheus endpoint to scrape directly. You can provide `http://prometheus.default:9090/federate?match[]={job=~\".+\"}` as the target url to indicate that all metrics should be scraped.\n\nSome agents look for targets to scrape via annotations. You can update Cortex's Prometheus server with the correct annotations. First, Create a `patch.yaml` file and add the relevant annotations for your monitoring solution. Below is an example for [Datadog](https://docs.datadoghq.com/agent/kubernetes/prometheus/). These annotations instruct the Datadog agent to scrape the Prometheus server at the endpoint `/federate?match[]={job=~\".+\"}` and extract `cortex_in_flight_requests`. Note that Datadog specifically requires the query params in the Prometheus url to be encoded.\n\n```yaml\nspec:\n  podMetadata:\n    annotations:\n      ad.datadoghq.com/prometheus.check_names: |\n         [\"prometheus\"]\n      ad.datadoghq.com/prometheus.init_configs: |\n         [{}]\n      ad.datadoghq.com/prometheus.instances: |\n         [\n            {\n               \"prometheus_url\": \"http://%%host%%:%%port%%/federate?match[]=%7Bjob%3D~%22.%2B%22%7D\",\n               \"namespace\": \"cortex\",\n               \"metrics\": [{\"cortex_in_flight_requests\":\"in_flight_requests\"}]\n            }\n         ]\n```\n\nThen, update Prometheus with your annotations:\n\n```bash\nkubectl patch --namespace prometheus prometheuses.monitoring.coreos.com prometheus --patch-file patch.yaml --type merge\n```\n\n## Long term metric storage\n\nPrometheus can be configured to write metrics to other monitoring solutions or databases for long term storage. You can attach a remote storage adapter to Prometheus that will receive samples from Prometheus and write to your destination. You can find a list of Prometheus remote storage adapters [here](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). Additional remote storage adapters can be found online if yours isn't on the list.\n\nOnce you've found an adapter that works for you, follow the steps below:\n\n### Configure kubectl\n\nFollow these [instructions](../advanced/kubectl.md) to set up kubectl.\n\n### Update Prometheus\n\nDefine a `patch.yaml` file with your changes to the Prometheus server:\n\n```yaml\nspec:\n  containers: # container for your adapter\n    ...\n  remoteWrite:\n    url: \"http://localhost:9201/write\" # http endpoint for your adapter\n```\n\nUpdate Prometheus with your changes:\n\n```bash\nkubectl patch --namespace prometheus prometheuses.monitoring.coreos.com prometheus --patch-file patch.yaml --type merge\n```\n"
  },
  {
    "path": "docs/overview.md",
    "content": "# Overview\n\n## Cluster\n\nThe Cortex cluster runs on an EKS (Kubernetes) cluster in a dedicated VPC on your AWS account.\n\n### Worker node groups\n\nThe Kubernetes cluster uses EC2 autoscaling groups for its worker node groups. Cortex supports most EC2 instance types, and the necessary device drivers are installed to expose GPU and Inferentia hardware to your workloads. Reserved and spot instances can be used to reduce costs.\n\nCortex uses the Kubernetes Cluster Autoscaler to scale the appropriate node groups to satisfy the compute demands of your workloads.\n\n### Networking\n\nBy default, a new dedicated VPC is created for the cluster during installation.\n\nTwo AWS load balancers are created to route traffic to the cluster. One load balancer is dedicated for traffic to your APIs, and the other load balancer is dedicated for API management requests to Cortex from your CLI or Python client. Traffic to the load balancers can be secured and restricted based on your cluster configuration.\n\n### Observability\n\nAll logs from the Cortex cluster are pushed to a CloudWatch log group using FluentBit. An in-cluster Prometheus installation is used to collect metrics for observability and autoscaling purposes. Metrics and dashboards pertaining to your workloads and instance usage can be viewed and modified via Grafana.\n\n## Deploying to the cluster\n\nAfter a successful cluster creation, you can use the CLI or Python Client to deploy different types of workloads. The clients use AWS credentials to authenticate to the cluster.\n\nCortex uses a collection of containers, referred to as a pod, as the atomic unit; scaling and replication occurs at the pod level. The orchestration and scaling of pods is unique to the different types of workloads:\n\n* Realtime\n* Async\n* Batch\n* Task\n\nVisit the workload-specific documentation for more details.\n\n## Architecture Diagram\n\n![](https://user-images.githubusercontent.com/808475/146854233-505fb8c2-513d-4836-920b-5d447da95dee.png)\n"
  },
  {
    "path": "docs/start.md",
    "content": "# Get started\n\n## Create a cluster on your AWS account\n\n<!-- CORTEX_VERSION_README -->\n```bash\n# install the CLI\nbash -c \"$(curl -sS https://raw.githubusercontent.com/cortexlabs/cortex/v0.42.1/get-cli.sh)\"\n\n# create a cluster\ncortex cluster up cluster.yaml\n```\n\n* [Client installation](clients/install.md) - customize your client installation.\n* [Cluster configuration](clusters/management/create.md) - optimize your cluster for your workloads.\n* [Environments](clusters/management/environments.md) - manage multiple clusters.\n\n## Build scalable APIs\n\n```bash\n# deploy APIs\ncortex deploy apis.yaml\n```\n\n* [Realtime](workloads/realtime/example.md) - create APIs that respond to requests in real-time.\n* [Async](workloads/async/example.md) - create APIs that respond to requests asynchronously.\n* [Batch](workloads/batch/example.md) - create APIs that run distributed batch jobs.\n* [Task](workloads/task/example.md) - create APIs that run jobs on-demand.\n"
  },
  {
    "path": "docs/summary.md",
    "content": "# Summary\n\n* [Get started](start.md)\n* [Overview](overview.md)\n\n## Clusters\n\n* Management\n  * [Auth](clusters/management/auth.md)\n  * [Create](clusters/management/create.md)\n  * [Update](clusters/management/update.md)\n  * [Delete](clusters/management/delete.md)\n  * [Environments](clusters/management/environments.md)\n  * [Production Guide](clusters/management/production.md)\n* Instances\n  * [Multi-instance](clusters/instances/multi.md)\n  * [Spot instances](clusters/instances/spot.md)\n* Observability\n  * [Logging](clusters/observability/logging.md)\n  * [Metrics](clusters/observability/metrics.md)\n  * [Alerting](clusters/observability/alerting.md)\n* Networking\n  * [Load balancers](clusters/networking/load-balancers.md)\n  * [Custom domain](clusters/networking/custom-domain.md)\n  * [HTTPS](clusters/networking/https.md)\n  * [HTTPS with API Gateway](clusters/networking/api-gateway.md)\n  * [VPC peering](clusters/networking/vpc-peering.md)\n* Advanced\n  * [Setting up kubectl](clusters/advanced/kubectl.md)\n  * [Private Docker registry](clusters/advanced/registry.md)\n  * [Self hosted images](clusters/advanced/self-hosted-images.md)\n\n## Workloads\n\n* [Realtime](workloads/realtime/realtime.md)\n  * [Example](workloads/realtime/example.md)\n  * [Configuration](workloads/realtime/configuration.md)\n  * [Containers](workloads/realtime/containers.md)\n  * [Autoscaling](workloads/realtime/autoscaling.md)\n  * [Traffic Splitter](workloads/realtime/traffic-splitter.md)\n  * [Metrics](workloads/realtime/metrics.md)\n  * [Statuses](workloads/realtime/statuses.md)\n  * [Troubleshooting](workloads/realtime/troubleshooting.md)\n* [Async](workloads/async/async.md)\n  * [Example](workloads/async/example.md)\n  * [Configuration](workloads/async/configuration.md)\n  * [Containers](workloads/async/containers.md)\n  * [Autoscaling](workloads/async/autoscaling.md)\n  * [Statuses](workloads/async/statuses.md)\n* [Batch](workloads/batch/batch.md)\n  * [Example](workloads/batch/example.md)\n  * [Configuration](workloads/batch/configuration.md)\n  * [Containers](workloads/batch/containers.md)\n  * [Jobs](workloads/batch/jobs.md)\n  * [Statuses](workloads/batch/statuses.md)\n* [Task](workloads/task/task.md)\n  * [Example](workloads/task/example.md)\n  * [Configuration](workloads/task/configuration.md)\n  * [Containers](workloads/task/containers.md)\n  * [Jobs](workloads/task/jobs.md)\n  * [Statuses](workloads/task/statuses.md)\n\n## Clients\n\n* [Install](clients/install.md)\n* [Uninstall](clients/uninstall.md)\n* [CLI commands](clients/cli.md)\n* [Python client](clients/python.md)\n"
  },
  {
    "path": "docs/workloads/async/async.md",
    "content": "# Async\n\nAsync APIs are designed for asynchronous workloads in which the user submits an asynchronous request and retrieves the result later (either by polling or through a webhook).\n\nAsync APIs are a good fit for users who want to submit longer workloads (such as video, audio or document processing), and do not need the result immediately or synchronously.\n\n**Key features**\n\n* asynchronously process requests\n* retrieve status and response via HTTP endpoint\n* autoscale based on queue length\n* avoid cold starts\n* scale to zero\n* perform rolling updates\n* automatically recover from failures and spot instance termination\n\n## How it works\n\nWhen you deploy an AsyncAPI, Cortex creates an SQS queue, a pool of Async Gateway workers, and a pool of worker pods. Each worker pod is running a dequeuer sidecar and your containers.\n\nUpon receiving a request, the Async Gateway will save the request payload to S3, enqueue the request ID onto an SQS FIFO queue, and respond with the request ID.\n\nThe dequeuer sidecar in the worker pod will pull the request from the SQS queue, download the request's payload from S3, and make a POST request to your containers. After the dequeuer receives a response, the corresponding request payload will be deleted from S3 and the response will be saved in S3 for 7 days.\n\nYou can fetch the result by making a GET request to the AsyncAPI endpoint with the request ID. The Async Gateway will respond with the status and the result (if the request has been completed).\n\nThe pool of workers running your containers autoscales based on the average number of messages in the queue and can scale down to 0 (if configured to do so).\n\n![](https://user-images.githubusercontent.com/808475/146854251-fed4235f-3627-4cd0-bc86-066272d7f138.png)\n"
  },
  {
    "path": "docs/workloads/async/autoscaling.md",
    "content": "# Autoscaling\n\nCortex auto-scales AsyncAPIs on a per-API basis based on your configuration.\n\n## Autoscaling replicas\n\n### Relevant pod configuration\n\nIn addition to the autoscaling configuration options (described below), there is one field in the pod configuration which is relevant to replica autoscaling:\n\n**`max_concurrency`** (default: 1): The maximum number of requests that will be concurrently sent into the container by Cortex. If your web server is designed to handle multiple concurrent requests, increasing `max_concurrency` will increase the throughput of a replica (and result in fewer total replicas for a given load).\n\n<br>\n\n### Autoscaling configuration\n\n**`min_replicas`** (default: 1): The lower bound on how many replicas can be running for an API. Scale-to-zero is supported.\n\n<br>\n\n**`max_replicas`** (default: 100): The upper bound on how many replicas can be running for an API.\n\n<br>\n\n**`target_in_flight`** (default: `max_concurrency` in the pod configuration): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. The number of in-flight requests is simply how many requests have been submitted and are not yet finished being processed. Therefore, this number includes requests which are actively being processed as well as requests which are waiting in the queue.\n\nThe autoscaler uses this formula to determine the number of desired replicas:\n\n`desired replicas = total in-flight requests / target_in_flight`\n\nFor example, setting `target_in_flight` to `max_concurrency` (the default) causes the cluster to adjust the number of replicas so that on average, there are no requests waiting in the queue.\n\n<br>\n\n**`window`** (default: 60s): The time over which to average the API's in-flight requests. The longer the window, the slower the autoscaler will react to changes in in-flight requests, since it is averaged over the `window`. An API's in-flight requests is calculated every 10 seconds, so `window` must be a multiple of 10 seconds.\n\n<br>\n\n**`downscale_stabilization_period`** (default: 5m): The API will not scale below the highest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the max of the current recommendation and all recommendations made during the `downscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to decreased traffic, and will reduce thrashing.\n\n<br>\n\n**`upscale_stabilization_period`** (default: 1m): The API will not scale above the lowest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the min of the current recommendation and all recommendations made during the `upscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to increased traffic, and will reduce thrashing.\n\n<br>\n\n**`max_downscale_factor`** (default: 0.75): The maximum factor by which to scale down the API on a single scaling event. For example, if `max_downscale_factor` is 0.5 and there are 10 running replicas, the autoscaler will not recommend fewer than 5 replicas. Increasing this number will allow the cluster to shrink more quickly in response to dramatic dips in traffic.\n\n<br>\n\n**`max_upscale_factor`** (default: 1.5): The maximum factor by which to scale up the API on a single scaling event. For example, if `max_upscale_factor` is 10 and there are 5 running replicas, the autoscaler will not recommend more than 50 replicas. Increasing this number will allow the cluster to grow more quickly in response to dramatic spikes in traffic.\n\n<br>\n\n**`downscale_tolerance`** (default: 0.05): Any recommendation falling within this factor below the current number of replicas will not trigger a scale down event. For example, if `downscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 18 or 19 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.\n\n<br>\n\n**`upscale_tolerance`** (default: 0.05): Any recommendation falling within this factor above the current number of replicas will not trigger a scale-up event. For example, if `upscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 21 or 22 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.\n\n<br>\n\n## Autoscaling instances\n\nCortex spins up and down instances based on the aggregate resource requests of all APIs. The number of instances will be at least `min_instances` and no more than `max_instances` for each node group (configured during installation and modifiable via `cortex cluster configure`).\n\n## Overprovisioning\n\nThe default value for `target_in_flight` is `max_concurrency`, which behaves well in many situations (see above for an explanation of how `target_in_flight` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_in_flight` to a lower value relative to the expected replica's concurrency. The smaller `target_in_flight` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity).\n\nFor example, if you've determined that each replica in your API can efficiently handle 2 concurrent requests, you would typically set `target_in_flight` to 2. In a scenario where your API is receiving 8 concurrent requests on average, the autoscaler would maintain 4 live replicas (8/2 = 4). If you wanted to overprovision by 25%, you could set `target_in_flight` to 1.6, causing the autoscaler maintain 5 live replicas (8/1.6 = 5).\n\n## Autoscaling responsiveness\n\nAssuming that `window` and `upscale_stabilization_period` are set to their default values (1 minute), it could take up to 2 minutes of increased traffic before an extra replica is requested. As soon as the additional replica is requested, the replica request will be visible in the output of `cortex get`, but the replica won't yet be running. If an extra instance is required to schedule the newly requested replica, it could take a few minutes for AWS to provision the instance (depending on the instance type), plus a few minutes for the newly provisioned instance to download your api image and for the api to initialize.\n\nKeep these delays in mind when considering overprovisioning (see above) and when determining appropriate values for `window` and `upscale_stabilization_period`. If you want the autoscaler to react as quickly as possible, set `upscale_stabilization_period` and `window` to their minimum values (0s and 10s respectively).\n"
  },
  {
    "path": "docs/workloads/async/configuration.md",
    "content": "# Configuration\n\n```yaml\n- name: <string>  # name of the API (required)\n  kind: AsyncAPI  # must be \"AsyncAPI\" for async APIs (required)\n  pod:  # pod configuration (required)\n    port: <int>  # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT)\n    max_concurrency: <int>  # maximum number of requests that will be concurrently sent into the container (default: 1, max allowed: 100)\n    containers:  # configurations for the containers to run (at least one constainer must be provided)\n      - name: <string>  # name of the container (required)\n        image: <string>  # docker image to use for the container (required)\n        command: <list[string]>  # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's ENTRYPOINT)\n        args: <list[string]>  # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's CMD)\n        env: <map[string:string]>  # dictionary of environment variables to set in the container (optional)\n        compute:  # compute resource requests (default: see below)\n          cpu: <string|int|float>  # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the \"m\" suffix (default: 200m)\n          gpu: <int>  # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0)\n          inf: <int>  # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0)\n          mem: <string>  # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null)\n          shm: <string>  # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null)\n        readiness_probe:  # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional)\n          http_get:  # specifies an http endpoint which must respond with status code 200 (only one of http_get and tcp_socket may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          tcp_socket:  # specifies a port which must be ready to receive traffic (only one of http_get and tcp_socket may be specified)\n            port: <int|string>  # the port to access on the container (required)\n          initial_delay_seconds: <int>  # number of seconds after the container has started before the probe is initiated (default: 0)\n          timeout_seconds: <int>  # number of seconds until the probe times out (default: 1)\n          period_seconds: <int>  # how often (in seconds) to perform the probe (default: 10)\n          success_threshold: <int>  # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)\n          failure_threshold: <int>  # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)\n        liveness_probe:  # periodic probe of container liveness; container will be restarted if the probe fails (optional)\n          http_get:  # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          tcp_socket:  # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n          exec:  # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified)\n            command: <list[string]>  # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)\n          initial_delay_seconds: <int>  # number of seconds after the container has started before the probe is initiated (default: 0)\n          timeout_seconds: <int>  # number of seconds until the probe times out (default: 1)\n          period_seconds: <int>  # how often (in seconds) to perform the probe (default: 10)\n          success_threshold: <int>  # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)\n          failure_threshold: <int>  # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)\n        pre_stop:  # a pre-stop lifecycle hook for the container; will be executed before container termination (optional)\n          http_get:  # specifies an http endpoint to send a request to (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          exec:  # specifies a command to run (only one of http_get, tcp_socket, and exec may be specified)\n            command: <list[string]>  # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)\n  autoscaling:  # autoscaling configuration (default: see below)\n    min_replicas: <int>  # minimum number of replicas (default: 1; min value: 0)\n    max_replicas: <int>  # maximum number of replicas (default: 100)\n    init_replicas: <int>  # initial number of replicas (default: <min_replicas>)\n    target_in_flight: <float>  # desired number of in-flight requests per replica (including requests actively being processed as well as queued), which the autoscaler tries to maintain (default: <max_concurrency>)\n    window: <duration>  # duration over which to average the API's in-flight requests per replica (default: 60s)\n    downscale_stabilization_period: <duration>  # the API will not scale below the highest recommendation made during this period (default: 5m)\n    upscale_stabilization_period: <duration>  # the API will not scale above the lowest recommendation made during this period (default: 1m)\n    max_downscale_factor: <float>  # maximum factor by which to scale down the API on a single scaling event (default: 0.75)\n    max_upscale_factor: <float>  # maximum factor by which to scale up the API on a single scaling event (default: 1.5)\n    downscale_tolerance: <float>  # any recommendation falling within this factor below the current number of replicas will not trigger a scale down event (default: 0.05)\n    upscale_tolerance: <float>  # any recommendation falling within this factor above the current number of replicas will not trigger a scale-up event (default: 0.05)\n  node_groups: <list[string]>  # a list of node groups on which this API can run (default: all node groups are eligible)\n  update_strategy:  # deployment strategy to use when replacing existing replicas with new ones (default: see below)\n    max_surge: <string|int>  # maximum number of replicas that can be scheduled above the desired number of replicas during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) (set to 0 to disable rolling updates)\n    max_unavailable: <string|int>  # maximum number of replicas that can be unavailable during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%)\n  networking:  # networking configuration (default: see below)\n    endpoint: <string>  # endpoint for the API (default: <api_name>)\n```\n"
  },
  {
    "path": "docs/workloads/async/containers.md",
    "content": "# Containers\n\n## Handling requests\n\nIn order to handle requests to your Async API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080).\n\nRequests will be sent to your web server via HTTP POST requests to the root path (`/`) as they are pulled off of the queue. The payload and the content type header of the HTTP request to your web server will match those of the original request to your Async API. In addition, the request's ID will be passed in via the \"X-Cortex-Request-ID\" header.\n\nYour web server must respond with valid JSON (with the `Content-Type` header set to \"application/json\"). The response will remain queryable for 7 days.\n\n## Readiness checks\n\nIt is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving traffic. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them.\n\nThere are two types of readiness checks which are supported: `http_get` and `tcp_socket` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly:\n\n```yaml\nreadiness_probe:\n  http_get:\n    port: 8080\n    path: /healthz\n```\n\n## Multiple containers\n\nYour API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers).\n\nThe `/mnt` directory is mounted to each container's filesystem, and is shared across all containers.\n\n## Resource requests\n\nEach container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's dequeuer sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory.\n\n## Observability\n\nSee docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md).\n\n## Using the Cortex CLI or client\n\nIt is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`).\n\nNote: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable).\n\n## Chaining APIs\n\nIt is possible to submit requests to Async APIs from any Cortex API within a Cortex cluster. Requests can be made to `http://ingressgateway-apis.istio-system.svc.cluster.local/<api_name>`, where `<api_name>` is the name of the Async API you are making a request to.\n\nFor example, if there is an Async API named `hello-world` running in the cluster, you can make a request to it from a different API in Python by using:\n\n```python\nimport requests\n\n# make a request to an Async API\nresponse = requests.post(\n    \"http://ingressgateway-apis.istio-system.svc.cluster.local/hello-world\",\n    json={\"text\": \"hello world\"},\n)\n\n# retreive a result from an Async API\nresponse = requests.get(\"http://ingressgateway-apis.istio-system.svc.cluster.local/hello-world/<id>\")\n```\n\nTo make requests from your Async API to a Realtime, Batch, or Task API running within the cluster, see the \"Chaining APIs\" docs associated with the target workload type.\n"
  },
  {
    "path": "docs/workloads/async/example.md",
    "content": "# AsyncAPI\n\n### Define an API\n\n```python\n# main.py\n\nfrom fastapi import FastAPI\nfrom pydantic import BaseModel\n\napp = FastAPI()\n\nclass Data(BaseModel):\n    msg: str\n\n@app.post(\"/\")\ndef handle_async(data: Data):\n    return data\n```\n\n### Create a `Dockerfile`\n\n```Dockerfile\nFROM python:3.8-slim\n\nRUN pip install --no-cache-dir fastapi uvicorn\nCOPY main.py /\n\nCMD uvicorn --host 0.0.0.0 --port 8080 main:app\n```\n\n### Build an image\n\n```bash\ndocker build . -t hello-world\n```\n\n### Run a container locally\n\n```bash\ndocker run -p 8080:8080 hello-world\n```\n\n### Make a request\n\n```bash\ncurl -X POST -H \"Content-Type: application/json\" -d '{\"msg\": \"hello world\"}' localhost:8080\n```\n\n### Login to ECR\n\n```bash\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com\n```\n\n### Create a repository\n\n```bash\naws ecr create-repository --repository-name hello-world\n```\n\n### Tag the image\n\n```bash\ndocker tag hello-world <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Push the image\n\n```bash\ndocker push <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Configure a Cortex deployment\n\n```yaml\n# cortex.yaml\n\n- name: hello-world\n  kind: AsyncAPI\n  pod:\n    containers:\n    - name: api\n      image: <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Create a Cortex deployment\n\n```bash\ncortex deploy\n```\n\n### Wait for the API to be ready\n\n```bash\ncortex get --watch\n```\n\n### Get the API endpoint\n\n```bash\ncortex get hello-world\n```\n\n### Make a request\n\n```bash\ncurl -X POST -H \"Content-Type: application/json\" -d '{\"msg\": \"hello world\"}' http://***.amazonaws.com/hello-world\n```\n\n### Get the response\n\n```bash\ncurl http://***.amazonaws.com/hello-world/<REQUEST_ID>\n```\n"
  },
  {
    "path": "docs/workloads/async/statuses.md",
    "content": "# Request statuses\n\n| Status            | Meaning                                                               |\n| :---              | :---                                                                  |\n| in_queue          | Workload is in the queue and is yet to be consumed by the API         |\n| in_progress       | Workload has been pulled by the API and is currently being processed  |\n| completed         | Workload has completed with success                                   |\n| failed            | Workload encountered an error during processing                       |\n\n# Replica states\n\nThe replica states of an API can be inspected by running `cortex describe <api-name>`. Here are the possible states for each replica in an API:\n\n| State | Meaning |\n|:---|:---|\n| Ready | Replica is running and it has passed the readiness checks |\n| ReadyOutOfDate | Replica is running and it has passed the readiness checks (for an out-of-date replica) |\n| NotReady | Replica is running but it's not passing the readiness checks; make sure the server is listening on the designed port of the API |\n| Pending | Replica is in a pending state (waiting to get scheduled onto a node) |\n| Creating | Replica is in the process of having its containers created |\n| ErrImagePull | Replica was not created because one of the specified Docker images was inaccessible at runtime; check that your API's docker images exist and are accessible via your cluster's AWS credentials |\n| Failed | Replica couldn't start due to an error; run `cortex logs <name>` to view the logs |\n| Killed | Replica has had one of its containers killed |\n| KilledOOM | Replica was terminated due to excessive memory usage; try allocating more memory to the API and re-deploy |\n| Stalled | Replica has been in a pending state for more than 15 minutes; see [troubleshooting](../realtime/troubleshooting.md) |\n| Terminating | Replica is currently in the process of being terminated |\n| Unknown | Replica is in an unknown state |\n"
  },
  {
    "path": "docs/workloads/batch/batch.md",
    "content": "# Batch\n\nBatch APIs run distributed and fault-tolerant batch processing jobs on demand.\n\nBatch APIs are a good fit for users who want to break up their workloads and distribute them across a dedicated pool of workers (for example, running inference on a set of images).\n\n**Key features**\n\n* distribute a batch job across multiple workers\n* scale to 0 (when there are no batch jobs)\n* trigger `/on-job-complete` hook once all batches have been processed\n* attempt all batches at least once\n* reroute failed batches to a dead letter queue\n* automatically recover from failures and spot instance termination\n\n## How it works\n\nWhen you deploy a Batch API, Cortex creates an endpoint to receive job submissions. Upon submitting a job, Cortex will respond with a Job ID, and will asynchronously trigger a Batch Job.\n\nA Batch Job begins with the deployment of an enqueuer process which breaks up the data in the job into batches and pushes them onto an SQS FIFO queue.\n\nAfter enqueuing is complete, Cortex initializes the requested number of worker pods and attaches a dequeuer sidecar to each pod. The dequeuer is responsible for retrieving batches from the queue and making an http request to your pod for each batch.\n\nAfter the worker pods have emptied the queue, the job is marked as complete, and Cortex will terminate the worker pods and delete the SQS queue.\n\nYou can make GET requests to the BatchAPI endpoint to get the status of the Job and metrics such as the number of batches completed and failed.\n\n![](https://user-images.githubusercontent.com/808475/146854256-b5b0c9a0-1753-4018-bda2-5ebddd8a6ffa.png)\n"
  },
  {
    "path": "docs/workloads/batch/configuration.md",
    "content": "# Configuration\n\n```yaml\n- name: <string>  # name of the API (required)\n  kind: BatchAPI  # must be \"BatchAPI\" for batch APIs (required)\n  pod:  # pod configuration (required)\n    port: <int>  # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT)\n    containers:  # configurations for the containers to run (at least one constainer must be provided)\n      - name: <string>  # name of the container (required)\n        image: <string>  # docker image to use for the container (required)\n        command: <list[string]>  # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (required)\n        args: <list[string]>  # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: no args)\n        env: <map[string:string]>  # dictionary of environment variables to set in the container (optional)\n        compute:  # compute resource requests (default: see below)\n          cpu: <string|int|float>  # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the \"m\" suffix (default: 200m)\n          gpu: <int>  # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0)\n          inf: <int>  # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0)\n          mem: <string>  # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null)\n          shm: <string>  # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null)\n        readiness_probe:  # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional)\n          http_get:  # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          tcp_socket:  # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n          initial_delay_seconds: <int>  # number of seconds after the container has started before the probe is initiated (default: 0)\n          timeout_seconds: <int>  # number of seconds until the probe times out (default: 1)\n          period_seconds: <int>  # how often (in seconds) to perform the probe (default: 10)\n          success_threshold: <int>  # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)\n          failure_threshold: <int>  # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)\n        liveness_probe:  # periodic probe of container liveness; container will be restarted if the probe fails (optional)\n          http_get:  # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          tcp_socket:  # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n          exec:  # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified)\n            command: <list[string]>  # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)\n          initial_delay_seconds: <int>  # number of seconds after the container has started before the probe is initiated (default: 0)\n          timeout_seconds: <int>  # number of seconds until the probe times out (default: 1)\n          period_seconds: <int>  # how often (in seconds) to perform the probe (default: 10)\n          success_threshold: <int>  # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)\n          failure_threshold: <int>  # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)\n  node_groups: <list[string]>  # a list of node groups on which this API can run (default: all node groups are eligible)\n  networking:  # networking configuration (default: see below)\n    endpoint: <string>  # endpoint for the API (default: <api_name>)\n```\n"
  },
  {
    "path": "docs/workloads/batch/containers.md",
    "content": "# Containers\n\n## Handling requests\n\nIn order to receive batches in your Batch API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080).\n\nBatches will be sent to your web server via HTTP POST requests to the root path (`/`). The payload will be a JSON-encoded array representing one batch, and the `Content-Type` header will be set to \"application/json\". In addition, the job's ID will be passed in via the \"X-Cortex-Job-ID\" header.\n\nYour web server must respond with status code 200 for the batch to be marked as succeeded (the response body will be ignored).\n\nOnce all batches have been processed, one of your workers will receive an HTTP POST request to `/on-job-complete`. It is not necessary for your web server to handle requests to `/on-job-complete` (404 errors will be ignored).\n\n## Job specification\n\nIf you need access to any parameters in the job submission (e.g. `config`), the entire job specification is available at `/cortex/spec/job.json` in your API containers' filesystems.\n\n## Readiness checks\n\nIt is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving batches. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them.\n\nThere are two types of readiness checks which are supported: `http_get` and `tcp_socket` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly:\n\n```yaml\nreadiness_probe:\n  http_get:\n    port: 8080\n    path: /healthz\n```\n\n## Multiple containers\n\nYour API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers).\n\nThe `/mnt` directory is mounted to each container's filesystem, and is shared across all containers.\n\n## Resource requests\n\nEach container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's dequeuer sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory.\n\n## Observability\n\nSee docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md).\n\n## Using the Cortex CLI or client\n\nIt is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`).\n\nNote: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable).\n\n## Chaining APIs\n\nIt is possible to submit Batch jobs from any Cortex API within a Cortex cluster. Jobs can be submitted to `http://ingressgateway-operator.istio-system.svc.cluster.local/batch/<api_name>`, where `<api_name>` is the name of the Batch API you are making a request to.\n\nFor example, if there is a Batch API named `hello-world` running in the cluster, you can make a request to it from a different API in Python by using:\n\n```python\nimport requests\n\njob_spec = {\n    \"workers\": 1,\n    \"item_list\": {\"items\": [...], \"batch_size\": 10},\n    \"config\": {\"my_key\": \"my_value\"},\n}\n\nresponse = requests.post(\n    \"http://ingressgateway-operator.istio-system.svc.cluster.local/batch/hello-world\",\n    json=job_spec,\n)\n```\n\nTo make requests from your Batch API to a Realtime, Task, or Async API running within the cluster, see the \"Chaining APIs\" docs associated with the target workload type.\n"
  },
  {
    "path": "docs/workloads/batch/example.md",
    "content": "# BatchAPI\n\n### Define an API\n\n```python\n# main.py\n\nfrom fastapi import FastAPI\nfrom typing import List\n\napp = FastAPI()\n\n@app.post(\"/\")\ndef handle_batch(batch: List[int]):\n    print(batch)\n\n@app.post(\"/on-job-complete\")\ndef on_job_complete():\n    print(\"done\")\n```\n\n### Create a `Dockerfile`\n\n```Dockerfile\nFROM python:3.8-slim\n\nRUN pip install --no-cache-dir fastapi uvicorn\n\nCOPY main.py /\n\nCMD uvicorn --host 0.0.0.0 --port 8080 main:app\n```\n\n### Build an image\n\n```bash\ndocker build . -t hello-world\n```\n\n### Run a container locally\n\n```bash\ndocker run -p 8080:8080 hello-world\n```\n\n### Make a request\n\n```bash\ncurl -X POST -H \"Content-Type: application/json\" -d '[1,2,3,4]' localhost:8080\n```\n\n### Login to ECR\n\n```bash\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com\n```\n\n### Create a repository\n\n```bash\naws ecr create-repository --repository-name hello-world\n```\n\n### Tag the image\n\n```bash\ndocker tag hello-world <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Push the image\n\n```bash\ndocker push <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Configure a Cortex deployment\n\n```yaml\n# cortex.yaml\n\n- name: hello-world\n  kind: BatchAPI\n  pod:\n    containers:\n    - name: api\n      image: <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n      command: [\"uvicorn\", \"--host\", \"0.0.0.0\", \"--port\", \"8080\", \"main:app\"]\n```\n\n### Create a Cortex deployment\n\n```bash\ncortex deploy\n```\n\n### Get the API endpoint\n\n```bash\ncortex get hello-world\n```\n\n### Make a request\n\n```bash\ncurl -X POST -H \"Content-Type: application/json\" -d '{\"workers\": 2, \"item_list\": {\"items\": [1,2,3,4], \"batch_size\": 2}}' http://***.amazonaws.com/hello-world\n```\n\n### View the logs\n\n```bash\ncortex logs hello-world <JOB_ID>\n```\n"
  },
  {
    "path": "docs/workloads/batch/jobs.md",
    "content": "# BatchAPI jobs\n\n## Get the Batch API's endpoint\n\n```bash\ncortex get <batch_api_name>\n```\n\n## Submit a Job\n\nThere are three options for providing the dataset for your job:\n\n1. [Data in the request](#data-in-the-request)\n1. [List S3 file paths](#s3-file-paths)\n1. [Newline delimited JSON file(s) in S3](#newline-delimited-json-files-in-s3)\n\n### Data in the request\n\nThe input data for your job can be included directly in your job submission request by specifying an `item_list` in your json request payload. Each item can be any type (object, list, string, etc.) and is treated as a single sample. `item_list.batch_size` specifies how many items to include in a single batch.\n\n__Each batch must be smaller than 256 KiB, and the total request size must be less than 10 MiB.__ If you want to submit more data, explore the other job submission methods.\n\nSubmitting data in the request can be useful in the following scenarios:\n\n* the request only has a few items\n* each item in the request is small (e.g. urls to images/videos)\n* you want to avoid using S3 as an intermediate storage layer\n\n```yaml\nPOST <batch_api_endpoint>:\n{\n    \"workers\": <int>,         # the number of workers to allocate for this job (required)\n    \"timeout\": <int>,         # duration in seconds since the submission of a job before it is terminated (optional)\n    \"sqs_dead_letter_queue\": {      # specify a queue to redirect failed batches (optional)\n        \"arn\": <string>,            # arn of dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo\n        \"max_receive_count\": <int>  # number of a times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1)\n    },\n    \"item_list\": {\n        \"items\": [            # a list items that can be of any type (required)\n            <any>,\n            <any>\n        ],\n        \"batch_size\": <int>,  # the number of items per batch (the handle_batch() function is called once per batch) (required)\n    }\n    \"config\": {               # arbitrary input for this specific job (optional)\n        \"string\": <any>\n    }\n}\n\nRESPONSE:\n{\n    \"job_id\": <string>,\n    \"api_name\": <string>,\n    \"kind\": \"BatchAPI\",\n    \"workers\": <int>,\n    \"config\": {<string>: <any>},\n    \"api_id\": <string>,\n    \"sqs_url\": <string>,\n    \"timeout\": <int>,\n    \"sqs_dead_letter_queue\": {\n        \"arn\": <string>,\n        \"max_receive_count\": <int>\n    },\n    \"created_time\": <string>\n}\n```\n\nThe entire job specification is written to `/cortex/spec/job.json` in the API containers.\n\n### S3 file paths\n\nIf your input data is a list of files such as images/videos in an S3 directory, you can define `file_path_lister` in your submission request payload. You can use `file_path_lister.s3_paths` to specify a list of files or prefixes, and `file_path_lister.includes` and/or `file_path_lister.excludes` to remove unwanted files. The S3 file paths will be aggregated into batches of size `file_path_lister.batch_size`. To learn more about fine-grained S3 file filtering see [filtering files](#filtering-files).\n\n__The total size of a batch must be less than 256 KiB.__\n\nThis submission pattern can be useful in the following scenarios:\n\n* you have a list of images/videos in an S3 directory\n* each S3 file represents a single sample or a small number of samples\n\nIf a single S3 file contains a lot of samples/rows, try the next submission strategy.\n\n```yaml\nPOST <batch_api_endpoint>:\n{\n    \"workers\": <int>,               # the number of workers to allocate for this job (required)\n    \"timeout\": <int>,               # duration in seconds since the submission of a job before it is terminated (optional)\n    \"sqs_dead_letter_queue\": {      # specify a queue to redirect failed batches (optional)\n        \"arn\": <string>,            # arn of dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo\n        \"max_receive_count\": <int>  # number of a times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1)\n    },\n    \"file_path_lister\": {\n        \"s3_paths\": [<string>],     # can be S3 prefixes or complete S3 paths (required)\n        \"includes\": [<string>],     # glob patterns (optional)\n        \"excludes\": [<string>],     # glob patterns (optional)\n        \"batch_size\": <int>,        # the number of S3 file paths per batch (the handle_batch() function is called once per batch) (required)\n    }\n    \"config\": {                     # arbitrary input for this specific job (optional)\n        \"string\": <any>\n    }\n}\n\nRESPONSE:\n{\n    \"job_id\": <string>,\n    \"api_name\": <string>,\n    \"kind\": \"BatchAPI\",\n    \"workers\": <int>,\n    \"config\": {<string>: <any>},\n    \"api_id\": <string>,\n    \"sqs_url\": <string>,\n    \"timeout\": <int>,\n    \"sqs_dead_letter_queue\": {\n        \"arn\": <string>,\n        \"max_receive_count\": <int>\n    },\n    \"created_time\": <string>\n}\n```\n\nThe entire job specification is written to `/cortex/spec/job.json` in the API containers.\n\n### Newline delimited JSON files in S3\n\nIf your input dataset is a newline delimited json file in an S3 directory (or a list of them), you can define `delimited_files` in your request payload to break up the contents of the file into batches of size `delimited_files.batch_size`.\n\nUpon receiving `delimited_files`, your Batch API will iterate through the `delimited_files.s3_paths` to generate the set of S3 files to process. You can use `delimited_files.includes` and `delimited_files.excludes` to filter out unwanted files. Each S3 file will be parsed as a newline delimited JSON file. Each line in the file should be a JSON object, which will be treated as a single sample. The S3 file will be broken down into batches of size `delimited_files.batch_size` and submitted to your workers. To learn more about fine-grained S3 file filtering see [filtering files](#filtering-files).\n\n__The total size of a batch must be less than 256 KiB.__\n\nThis submission pattern is useful in the following scenarios:\n\n* one or more S3 files contains a large number of samples and must be broken down into batches\n\n```yaml\nPOST <batch_api_endpoint>:\n{\n    \"workers\": <int>,               # the number of workers to allocate for this job (required)\n    \"timeout\": <int>,               # duration in seconds since the submission of a job before it is terminated (optional)\n    \"sqs_dead_letter_queue\": {      # specify a queue to redirect failed batches (optional)\n        \"arn\": <string>,            # arn of dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo\n        \"max_receive_count\": <int>  # number of a times a batch is allowed to be handled by a worker before it is considered to be failed and transferred to the dead letter queue (must be >= 1)\n    },\n    \"delimited_files\": {\n        \"s3_paths\": [<string>],     # can be S3 prefixes or complete S3 paths (required)\n        \"includes\": [<string>],     # glob patterns (optional)\n        \"excludes\": [<string>],     # glob patterns (optional)\n        \"batch_size\": <int>,        # the number of json objects per batch (the handle_batch() function is called once per batch) (required)\n    }\n    \"config\": {                     # arbitrary input for this specific job (optional)\n        \"string\": <any>\n    }\n}\n\nRESPONSE:\n{\n    \"job_id\": <string>,\n    \"api_name\": <string>,\n    \"kind\": \"BatchAPI\",\n    \"workers\": <int>,\n    \"config\": {<string>: <any>},\n    \"api_id\": <string>,\n    \"sqs_url\": <string>,\n    \"timeout\": <int>,\n    \"sqs_dead_letter_queue\": {\n        \"arn\": <string>,\n        \"max_receive_count\": <int>\n    },\n    \"created_time\": <string>\n}\n```\n\nThe entire job specification is written to `/cortex/spec/job.json` in the API containers.\n\n## Get a job's status\n\n```bash\ncortex get <batch_api_name> <job_id>\n```\n\nOr make a GET request to `<batch_api_endpoint>?jobID=<jobID>`:\n\n```yaml\nGET <batch_api_endpoint>?jobID=<jobID>:\n\nRESPONSE:\n{\n    \"job_status\": {\n        \"job_id\": <string>,\n        \"api_name\": <string>,\n        \"kind\": \"BatchAPI\",\n        \"workers\": <int>,\n        \"config\": {<string>: <any>},\n        \"api_id\": <string>,\n        \"sqs_url\": <string>,\n        \"status\": <string>,\n        \"batches_in_queue\": <int>   # number of batches remaining in the queue\n        \"worker_counts\": {          # worker counts are only available while a job is running\n            \"pending\": <int>,       # number of workers that are waiting for compute resources to be provisioned\n            \"initializing\": <int>,  # number of workers that are initializing\n            \"running\": <int>,       # number of workers that are actively working on batches from the queue\n            \"succeeded\": <int>,     # number of workers that have completed after verifying that the queue is empty\n            \"failed\": <int>,        # number of workers that have failed\n            \"stalled\": <int>,       # number of workers that have been stuck in pending for more than 10 minutes\n        },\n        \"created_time\": <string>\n        \"start_time\": <string>\n        \"end_time\": <string> (optional)\n    },\n    \"endpoint\": <string>\n    \"api_spec\": {\n        ...\n    },\n    \"metrics\": {\n        \"succeeded\": <int>      # number of succeeded batches\n        \"failed\": int           # number of failed attempts\n        \"avg_time_per_batch\": <float> (optional)  # average time spent working on a batch (only considers successful attempts)\n    }\n}\n```\n\n## Stop a job\n\n```bash\ncortex delete <batch_api_name> <job_id>\n```\n\nOr make a DELETE request to `<batch_api_endpoint>?jobID=<jobID>`:\n\n```yaml\nDELETE <batch_api_endpoint>?jobID=<jobID>:\n\nRESPONSE:\n{\"message\":\"stopped job <job_id>\"}\n```\n\n## Additional Information\n\n### Filtering files\n\nWhen submitting a job using `delimited_files` or `file_path_lister`, you can use `s3_paths` in conjunction with `includes` and `excludes` to precisely filter files.\n\nThe Batch API will iterate through each S3 path in `s3_paths`. If the S3 path is a prefix, it iterates through each file in that prefix. For each file, if `includes` is non-empty, it will discard the S3 path if the S3 file doesn't match any of the glob patterns provided in `includes`. After passing the `includes` filter (if specified), if the `excludes` is non-empty, it will discard the S3 path if the S3 files matches any of the glob patterns provided in `excludes`.\n\nIf you aren't sure which files will be processed in your request, specify the `dryRun=true` query parameter in the job submission request to see the target list.\n\nHere are a few examples of filtering for a folder structure like this:\n\n```text\n├── s3://bucket\n    └── images\n        ├── img_1.png\n        ├── img_2.jpg\n        ├── img_3.jpg\n        └── img_4.gif\n```\n\nSelect all files\n\n```yaml\n{\n    \"s3_paths\": [\"s3://bucket/images/\"]\n}\n\n# or\n\n{\n    \"s3_paths\": [\"s3://bucket/images/img\"]\n}\n\n# Would select the following files:\n# s3://bucket/images/img_1.png\n# s3://bucket/images/img_2.jpg\n# s3://bucket/images/img_3.jpg\n# s3://bucket/images/img_4.gif\n```\n\nSelect specific files\n\n```yaml\n{\n    \"s3_paths\": [\n        \"s3://bucket/images/img_1.png\",\n        \"s3://bucket/images/img_2.jpg\"\n    ]\n}\n\n# Would select the following files:\n# s3://bucket/images/img_1.png\n# s3://bucket/images/img_2.jpg\n```\n\nOnly select JPG files\n\n```yaml\n{\n    \"s3_paths\": [\"s3://bucket/images/\"],\n    \"includes\": [\"**.jpg\"]\n}\n\n# Would select the following files:\n# s3://bucket/images/img_2.jpg\n# s3://bucket/images/img_3.jpg\n```\n\nSelect all JPG files except one specific JPG file\n\n```yaml\n{\n    \"s3_paths\": [\"s3://bucket/images/\"],\n    \"includes\": [\"**.jpg\"],\n    \"excludes\": [\"**_3.jpg\"]\n}\n\n# Would select the file:\n# s3://bucket/images/img_2.jpg\n```\n\nSelect all files except GIFs\n\n```yaml\n{\n    \"s3_paths\": [\"s3://bucket/images/\"],\n    \"excludes\": [\"**.gif\"]\n}\n\n# Would select the files:\n# s3://bucket/images/img_1.png\n# s3://bucket/images/img_2.jpg\n# s3://bucket/images/img_3.jpg\n```\n"
  },
  {
    "path": "docs/workloads/batch/statuses.md",
    "content": "# Job statuses\n\n| Status                   | Meaning |\n| :--- | :--- |\n| enqueuing                | Job is being split into batches and placed into a queue |\n| running                  | Workers are retrieving batches from the queue and running inference |\n| succeeded                | Workers completed all items in the queue without any failures |\n| failed while enqueuing   | Failure occurred while enqueuing; check job logs for more details |\n| completed with failures  | Workers completed all items in the queue but some of the batches weren't processed successfully and raised exceptions; check job logs for more details |\n| worker error             | One or more workers experienced an irrecoverable error, causing the job to fail; check job logs for more details |\n| out of memory            | One or more workers ran out of memory, causing the job to fail; check job logs for more details |\n| timed out                | Job was terminated after the specified timeout has elapsed |\n| stopped                  | Job was stopped by the user or the Batch API was deleted |\n"
  },
  {
    "path": "docs/workloads/realtime/autoscaling.md",
    "content": "# Autoscaling\n\nCortex autoscales each API independently based on its configuration.\n\n## Autoscaling replicas\n\n### Relevant pod configuration\n\nIn addition to the autoscaling configuration options (described below), there are two fields in the pod configuration which are relevant to replica autoscaling:\n\n**`max_concurrency`** (default: 1): The maximum number of requests that will be concurrently sent into the container by Cortex. If your web server is designed to handle multiple concurrent requests, increasing `max_concurrency` will increase the throughput of a replica (and result in fewer total replicas for a given load).\n\n<br>\n\n**`max_queue_length`** (default: 100): The maximum number of requests which will be queued by the replica (beyond `max_concurrency`) before requests are rejected with HTTP error code 503. For long-running APIs, decreasing `max_queue_length` and configuring the client to retry when it receives 503 responses will improve queue fairness accross replicas by preventing requests from sitting in long queues.\n\n<br>\n\n### Autoscaling configuration\n\n**`min_replicas`** (default: 1): The lower bound on how many replicas can be running for an API. Scale-to-zero is supported (experimental).\n\n<br>\n\n**`max_replicas`** (default: 100): The upper bound on how many replicas can be running for an API.\n\n<br>\n\n**`target_in_flight`** (default: `max_concurrency` in the pod configuration): This is the desired number of in-flight requests per replica, and is the metric which the autoscaler uses to make scaling decisions. The number of in-flight requests is simply how many requests have been sent to a replica and have not yet been responded to. Therefore, this number includes requests which are actively being processed as well as requests which are waiting in the replica's queue.\n\nThe autoscaler uses this formula to determine the number of desired replicas:\n\n`desired replicas = sum(in-flight requests accross all replicas) / target_in_flight`\n\nFor example, setting `target_in_flight` to `max_concurrency` (the default) causes the cluster to adjust the number of replicas so that on average, requests are immediately processed without waiting in a queue.\n\n<br>\n\n**`window`** (default: 60s): The time over which to average the API's in-flight requests (which is the sum of in-flight requests in each replica). The longer the window, the slower the autoscaler will react to changes in in-flight requests, since it is averaged over the `window`. An API's in-flight requests is calculated every 10 seconds, so `window` must be a multiple of 10 seconds.\n\n<br>\n\n**`downscale_stabilization_period`** (default: 5m): The API will not scale below the highest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the max of the current recommendation and all recommendations made during the `downscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to decreased traffic, and will reduce thrashing.\n\n<br>\n\n**`upscale_stabilization_period`** (default: 1m): The API will not scale above the lowest recommendation made during this period. Every 10 seconds, the autoscaler makes a recommendation based on all of the other configuration parameters described here. It will then take the min of the current recommendation and all recommendations made during the `upscale_stabilization_period`, and use that to determine the final number of replicas to scale to. Increasing this value will cause the cluster to react more slowly to increased traffic, and will reduce thrashing.\n\n<br>\n\n**`max_downscale_factor`** (default: 0.75): The maximum factor by which to scale down the API on a single scaling event. For example, if `max_downscale_factor` is 0.5 and there are 10 running replicas, the autoscaler will not recommend fewer than 5 replicas. Increasing this number will allow the cluster to shrink more quickly in response to dramatic dips in traffic.\n\n<br>\n\n**`max_upscale_factor`** (default: 1.5): The maximum factor by which to scale up the API on a single scaling event. For example, if `max_upscale_factor` is 10 and there are 5 running replicas, the autoscaler will not recommend more than 50 replicas. Increasing this number will allow the cluster to grow more quickly in response to dramatic spikes in traffic.\n\n<br>\n\n**`downscale_tolerance`** (default: 0.05): Any recommendation falling within this factor below the current number of replicas will not trigger a scale down event. For example, if `downscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 18 or 19 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.\n\n<br>\n\n**`upscale_tolerance`** (default: 0.05): Any recommendation falling within this factor above the current number of replicas will not trigger a scale-up event. For example, if `upscale_tolerance` is 0.1 and there are 20 running replicas, a recommendation of 21 or 22 replicas will not be acted on, and the API will remain at 20 replicas. Increasing this value will prevent thrashing, but setting it too high will prevent the cluster from maintaining it's optimal size.\n\n<br>\n\n## Autoscaling instances\n\nCortex spins up and down instances based on the aggregate resource requests of all APIs. The number of instances will be at least `min_instances` and no more than `max_instances` for each node group (configured during installation and modifiable via `cortex cluster configure`).\n\n## Overprovisioning\n\nThe default value for `target_in_flight` is `max_concurrency`, which behaves well in many situations (see above for an explanation of how `target_in_flight` affects autoscaling). However, if your application is sensitive to spikes in traffic or if creating new replicas takes too long (see below), you may find it helpful to maintain extra capacity to handle the increased traffic while new replicas are being created. This can be accomplished by setting `target_in_flight` to a lower value relative to the expected replica's concurrency. The smaller `target_in_flight` is, the more unused capacity your API will have, and the more room it will have to handle sudden increased load. The increased request rate will still trigger the autoscaler, and your API will stabilize again (maintaining the overprovisioned capacity).\n\nFor example, if you've determined that each replica in your API can efficiently handle 2 concurrent requests, you would typically set `target_in_flight` to 2. In a scenario where your API is receiving 8 concurrent requests on average, the autoscaler would maintain 4 live replicas (8/2 = 4). If you wanted to overprovision by 25%, you could set `target_in_flight` to 1.6, causing the autoscaler maintain 5 live replicas (8/1.6 = 5).\n\n## Autoscaling responsiveness\n\nAssuming that `window` and `upscale_stabilization_period` are set to their default values (1 minute), it could take up to 2 minutes of increased traffic before an extra replica is requested. As soon as the additional replica is requested, the replica request will be visible in the output of `cortex get`, but the replica won't yet be running. If an extra instance is required to schedule the newly requested replica, it could take a few minutes for AWS to provision the instance (depending on the instance type), plus a few minutes for the newly provisioned instance to download your api image and for the api to initialize.\n\nKeep these delays in mind when considering overprovisioning (see above) and when determining appropriate values for `window` and `upscale_stabilization_period`. If you want the autoscaler to react as quickly as possible, set `upscale_stabilization_period` and `window` to their minimum values (0s and 10s respectively).\n"
  },
  {
    "path": "docs/workloads/realtime/configuration.md",
    "content": "# Configuration\n\n```yaml\n- name: <string>  # name of the API (required)\n  kind: RealtimeAPI  # must be \"RealtimeAPI\" for realtime APIs (required)\n  pod:  # pod configuration (required)\n    port: <int>  # port to which requests will be sent (default: 8080; exported as $CORTEX_PORT)\n    max_concurrency: <int>  # maximum number of requests that will be concurrently sent into the container (default: 1)\n    max_queue_length: <int>  # maximum number of requests per replica which will be queued (beyond max_concurrency) before requests are rejected with error code 503 (default: 100)\n    containers:  # configurations for the containers to run (at least one constainer must be provided)\n      - name: <string>  # name of the container (required)\n        image: <string>  # docker image to use for the container (required)\n        command: <list[string]>  # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's ENTRYPOINT)\n        args: <list[string]>  # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: the docker image's CMD)\n        env: <map[string:string]>  # dictionary of environment variables to set in the container (optional)\n        compute:  # compute resource requests (default: see below)\n          cpu: <string|int|float>  # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the \"m\" suffix (default: 200m)\n          gpu: <int>  # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0)\n          inf: <int>  # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0)\n          mem: <string>  # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null)\n          shm: <string>  # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null)\n        readiness_probe:  # periodic probe of container readiness; traffic will not be sent into the pod unless all containers' readiness probes are succeeding (optional)\n          http_get:  # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          tcp_socket:  # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n          exec:  # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified)\n            command: <list[string]>  # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)\n          initial_delay_seconds: <int>  # number of seconds after the container has started before the probe is initiated (default: 0)\n          timeout_seconds: <int>  # number of seconds until the probe times out (default: 1)\n          period_seconds: <int>  # how often (in seconds) to perform the probe (default: 10)\n          success_threshold: <int>  # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)\n          failure_threshold: <int>  # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)\n        liveness_probe:  # periodic probe of container liveness; container will be restarted if the probe fails (optional)\n          http_get:  # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          tcp_socket:  # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n          exec:  # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified)\n            command: <list[string]>  # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)\n          initial_delay_seconds: <int>  # number of seconds after the container has started before the probe is initiated (default: 0)\n          timeout_seconds: <int>  # number of seconds until the probe times out (default: 1)\n          period_seconds: <int>  # how often (in seconds) to perform the probe (default: 10)\n          success_threshold: <int>  # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)\n          failure_threshold: <int>  # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)\n        pre_stop:  # a pre-stop lifecycle hook for the container; will be executed before container termination (optional)\n          http_get:  # specifies an http endpoint to send a request to (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          exec:  # specifies a command to run (only one of http_get, tcp_socket, and exec may be specified)\n            command: <list[string]>  # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)\n  autoscaling:  # autoscaling configuration (default: see below)\n    min_replicas: <int>  # minimum number of replicas (default: 1)\n    max_replicas: <int>  # maximum number of replicas (default: 100)\n    init_replicas: <int>  # initial number of replicas (default: <min_replicas>)\n    target_in_flight: <float>  # desired number of in-flight requests per replica (including requests actively being processed as well as queued), which the autoscaler tries to maintain (default: <max_concurrency>)\n    window: <duration>  # duration over which to average the API's in-flight requests per replica (default: 60s)\n    downscale_stabilization_period: <duration>  # the API will not scale below the highest recommendation made during this period (default: 5m)\n    upscale_stabilization_period: <duration>  # the API will not scale above the lowest recommendation made during this period (default: 1m)\n    max_downscale_factor: <float>  # maximum factor by which to scale down the API on a single scaling event (default: 0.75)\n    max_upscale_factor: <float>  # maximum factor by which to scale up the API on a single scaling event (default: 1.5)\n    downscale_tolerance: <float>  # any recommendation falling within this factor below the current number of replicas will not trigger a scale down event (default: 0.05)\n    upscale_tolerance: <float>  # any recommendation falling within this factor above the current number of replicas will not trigger a scale-up event (default: 0.05)\n  node_groups: <list[string]>  # a list of node groups on which this API can run (default: all node groups are eligible)\n  update_strategy:  # deployment strategy to use when replacing existing replicas with new ones (default: see below)\n    max_surge: <string|int>  # maximum number of replicas that can be scheduled above the desired number of replicas during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%) (set to 0 to disable rolling updates)\n    max_unavailable: <string|int>  # maximum number of replicas that can be unavailable during an update; can be an absolute number, e.g. 5, or a percentage of desired replicas, e.g. 10% (default: 25%)\n  networking:  # networking configuration (default: see below)\n    endpoint: <string>  # endpoint for the API (default: <api_name>)\n```\n"
  },
  {
    "path": "docs/workloads/realtime/containers.md",
    "content": "# Containers\n\n## Handling requests\n\nIn order to handle requests to your Realtime API, one of your containers must run a web server which is listening for HTTP requests on the port which is configured in the `pod.port` field of your [API configuration](configuration.md) (default: 8080).\n\nSubpaths are supported; for example, if your API is named `hello-world`, a request to `<load_balancer_url>/hello-world` will be routed to the root (`/`) of your web server, and a request to `<load_balancer_url>/hello-world/subpatch` will be routed to `/subpath` on your web server.\n\n## Readiness checks\n\nIt is often important to implement a readiness check for your API. By default, as soon as your web server has bound to the port, it will start receiving traffic. In some cases, the web server may start listening on the port before its workers are ready to handle traffic (e.g. `tiangolo/uvicorn-gunicorn-fastapi` behaves this way). Readiness checks ensure that traffic is not sent into your web server before it's ready to handle them.\n\nThere are three types of readiness checks which are supported: `http_get`, `tcp_socket`, and `exec` (see [API configuration](configuration.md) for usage instructions). A simple and often effective approach is to add a route to your web server (e.g. `/healthz`) which responds with status code 200, and configure your readiness probe accordingly:\n\n```yaml\nreadiness_probe:\n  http_get:\n    port: 8080\n    path: /healthz\n```\n\n## Multiple containers\n\nYour API pod can contain multiple containers, only one of which can be listening for requests on the target port (it can be any of the containers).\n\nThe `/mnt` directory is mounted to each container's file system, and is shared across all containers.\n\n## Resource requests\n\nEach container in the pod requests its own amount of CPU, memory, GPU, and Inferentia resources. In addition, Cortex's proxy sidecar container (which is automatically added to the pod) requests 100m CPU and 100Mi memory.\n\n## Observability\n\nSee docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md).\n\n## Using the Cortex CLI or client\n\nIt is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`).\n\nNote: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable).\n\n## Chaining APIs\n\nIt is possible to make requests to Realtime APIs from any Cortex API within a Cortex cluster. Requests can be made to `http://ingressgateway-apis.istio-system.svc.cluster.local/<api_name>`, where `<api_name>` is the name of the Realtime API you are making a request to.\n\nFor example, if there is a Realtime API named `hello-world` running in the cluster, you can make a request to it from a different API in Python by using:\n\n```python\nimport requests\n\nresponse = requests.post(\n    \"http://ingressgateway-apis.istio-system.svc.cluster.local/hello-world\",\n    json={\"text\": \"hello world\"},\n)\n```\n\nNote that if the API making the request is a Realtime API or Async API, its autoscaling configuration (i.e. `target_in_flight`) should be modified with the understanding that requests will be considered \"in-flight\" in the first API as the request is being fulfilled by the second API.\n\nTo make requests from your Realtime API to a Batch, Async, or Task API running within the cluster, see the \"Chaining APIs\" docs associated with the target workload type.\n"
  },
  {
    "path": "docs/workloads/realtime/example.md",
    "content": "# RealtimeAPI\n\n### Define an API\n\n```python\n# main.py\n\nfrom fastapi import FastAPI\nfrom pydantic import BaseModel\n\napp = FastAPI()\n\nclass Data(BaseModel):\n    msg: str\n\n@app.post(\"/\")\ndef handle_post(data: Data):\n    return data\n```\n\n### Create a `Dockerfile`\n\n```Dockerfile\nFROM python:3.8-slim\n\nRUN pip install --no-cache-dir fastapi uvicorn\n\nCOPY main.py /\n\nCMD uvicorn --host 0.0.0.0 --port 8080 main:app\n```\n\n### Build an image\n\n```bash\ndocker build . -t hello-world\n```\n\n### Run a container locally\n\n```bash\ndocker run -p 8080:8080 hello-world\n```\n\n### Make a request\n\n```bash\ncurl -X POST -H \"Content-Type: application/json\" -d '{\"msg\": \"hello world\"}' localhost:8080\n```\n\n### Login to ECR\n\n```bash\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com\n```\n\n### Create a repository\n\n```bash\naws ecr create-repository --repository-name hello-world\n```\n\n### Tag the image\n\n```bash\ndocker tag hello-world <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Push the image\n\n```bash\ndocker push <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Configure a Cortex deployment\n\n```yaml\n# cortex.yaml\n\n- name: hello-world\n  kind: RealtimeAPI\n  pod:\n    containers:\n    - name: api\n      image: <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Create a Cortex deployment\n\n```bash\ncortex deploy\n```\n\n### Wait for the API to be ready\n\n```bash\ncortex get --watch\n```\n\n### Get the API endpoint\n\n```bash\ncortex get hello-world\n```\n\n### Make a request\n\n```bash\ncurl -X POST -H \"Content-Type: application/json\" -d '{\"msg\": \"hello world\"}' http://***.amazonaws.com/hello-world\n```\n"
  },
  {
    "path": "docs/workloads/realtime/metrics.md",
    "content": "# Metrics\n\nThe `cortex get` and `cortex get API_NAME` commands display the request time (averaged over the past 2 weeks) and\nresponse code counts (summed over the past 2 weeks) for your APIs:\n\n```bash\ncortex get\n\nenv      api                         status   up-to-date   requested   last update   avg request   2XX\ncortex   iris-classifier             live     1            1           17m           24ms          1223\ncortex   text-generator              live     1            1           8m            180ms         433\ncortex   image-classifier-resnet50   live     2            2           1h            32ms          1121126\n```\n\nThe `cortex get API_NAME` command also provides a link to a Grafana dashboard:\n\n![dashboard](https://user-images.githubusercontent.com/7456627/107253455-9c6b7b80-6a36-11eb-8600-f36a7bab6d3b.png)\n\n---\n\n## Metrics in the dashboard\n\n| Panel             | Description                                                                        | Note                                                                                               |\n|-------------------|------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------|\n| Request Rate      | Request rate, computed over every minute, of an API                                |                                                                                                    |\n| In Flight Request | Active in-flight requests for an API.                                              | In-flight requests are recorded every 10 seconds, which will correspond to the minimum resolution. |\n| Active Replicas   | Active replicas for an API                                                         |                                                                                                    |\n| 2XX Responses     | Request rate, computed over a minute, for responses with status code 2XX of an API |                                                                                                    |\n| 4XX Responses     | Request rate, computed over a minute, for responses with status code 4XX of an API |                                                                                                    |\n| 5XX Responses     | Request rate, computed over a minute, for responses with status code 5XX of an API |                                                                                                    |\n| p99 Latency       | 99th percentile latency, computed over a minute, for an API                        | Value might not be accurate because the histogram buckets are not dynamically set.                 |\n| p90 Latency       | 90th percentile latency, computed over a minute, for an API                        | Value might not be accurate because the histogram buckets are not dynamically set.                 |\n| p50 Latency       | 50th percentile latency, computed over a minute, for an API                        | Value might not be accurate because the histogram buckets are not dynamically set.                 |\n| Average Latency   | Average latency, computed over a minute, for an API                                |                                                                                                    |\n"
  },
  {
    "path": "docs/workloads/realtime/realtime.md",
    "content": "# Realtime\n\nRealtime APIs respond to requests synchronously and autoscale based on in-flight request volumes.\n\nRealtime APIs are a good fit for users who want to run stateless containers as a scalable microservice (for example, deploying machine learning models as APIs).\n\n**Key features**\n\n* respond to requests synchronously\n* autoscale based on request volume\n* avoid cold starts\n* scale to zero\n* perform rolling updates\n* automatically recover from failures and spot instance termination\n* perform A/B tests and canary deployments\n\n## How it works\n\nWhen you deploy a Realtime API, Cortex initializes a pool of worker pods and attaches a proxy sidecar to each of the pods.\n\nThe proxy is responsible for receiving incoming requests, queueing them (if necessary), and forwarding them to your pod when it is ready. Autoscaling is based on aggregate in-flight request volume, which is published by the proxy sidecars.\n\n![](https://user-images.githubusercontent.com/808475/146854245-ed0fc153-d083-47d8-a7e2-ac5beb114ee6.png)\n"
  },
  {
    "path": "docs/workloads/realtime/statuses.md",
    "content": "# Replica states\n\nThe replica states of an API can be inspected by running `cortex describe <api-name>`. Here are the possible states for each replica in an API:\n\n| State | Meaning |\n|:---|:---|\n| Ready | Replica is running and it has passed the readiness checks |\n| ReadyOutOfDate | Replica is running and it has passed the readiness checks (for an out-of-date replica) |\n| NotReady | Replica is running but it's not passing the readiness checks; make sure the server is listening on the designed port of the API |\n| Pending | Replica is in a pending state (waiting to get scheduled onto a node) |\n| Creating | Replica is in the process of having its containers created |\n| ErrImagePull | Replica was not created because one of the specified Docker images was inaccessible at runtime; check that your API's docker images exist and are accessible via your cluster's AWS credentials |\n| Failed | Replica couldn't start due to an error; run `cortex logs <name>` to view the logs |\n| Killed | Replica has had one of its containers killed |\n| KilledOOM | Replica was terminated due to excessive memory usage; try allocating more memory to the API and re-deploy |\n| Stalled | Replica has been in a pending state for more than 15 minutes; see [troubleshooting](../realtime/troubleshooting.md) |\n| Terminating | Replica is currently in the process of being terminated |\n| Unknown | Replica is in an unknown state |\n"
  },
  {
    "path": "docs/workloads/realtime/traffic-splitter.md",
    "content": "# Traffic Splitter\n\nTraffic Splitters can be used to expose multiple RealtimeAPIs as a single endpoint for A/B tests, multi-armed bandits, or canary deployments.\n\n## Configuration\n\n```yaml\n- name: <string>  # name of the traffic splitter (required)\n  kind: TrafficSplitter  # must be \"TrafficSplitter\" for traffic splitters (required)\n  networking:  # networking configuration (default: see below)\n    endpoint: <string>  # the endpoint for the traffic splitter (default: <name>)\n  apis:  # list of Realtime APIs to target (required)\n    - name: <string>  # name of a Realtime API that is already running or is included in the same configuration file (required)\n      weight: <int>   # percentage of traffic to route to the Realtime API (all non-shadow weights must sum to 100) (required)\n      shadow: <bool>  # duplicate incoming traffic and send fire-and-forget to this api (only one shadow per traffic splitter) (default: false)\n```\n\n## Example\n\nThis example showcases Cortex's Python client, but these steps can also be performed by using the Cortex CLI.\n\n### Deploy a traffic splitter\n\n```python\ntraffic_splitter_spec = {\n    \"name\": \"sentiment-analyzer\",\n    \"kind\": \"TrafficSplitter\",\n    \"apis\": [\n        {\"name\": \"sentiment-analyzer-a\", \"weight\": 50},\n        {\"name\": \"sentiment-analyzer-b\", \"weight\": 50},\n    ],\n}\n\ncx.deploy(traffic_splitter_spec)\n```\n\n### Update the weights\n\n```python\nnew_traffic_splitter_spec = {\n    \"name\": \"sentiment-analyzer\",\n    \"kind\": \"TrafficSplitter\",\n    \"apis\": [\n        {\"name\": \"sentiment-analyzer-a\", \"weight\": 1},\n        {\"name\": \"sentiment-analyzer-b\", \"weight\": 99},\n    ],\n}\n\ncx.deploy(new_traffic_splitter_spec)\n```\n\n### Update the target APIs\n\n```python\nnew_traffic_splitter_spec = {\n    \"name\": \"sentiment-analyzer\",\n    \"kind\": \"TrafficSplitter\",\n    \"apis\": [\n        {\"name\": \"sentiment-analyzer-b\", \"weight\": 50},\n        {\"name\": \"sentiment-analyzer-c\", \"weight\": 50},\n    ],\n}\n\ncx.deploy(new_traffic_splitter_spec)\n```\n"
  },
  {
    "path": "docs/workloads/realtime/troubleshooting.md",
    "content": "# Troubleshooting\n\n## 503 error responses from API requests\n\nWhen making requests to your API, it's possible to get a `no healthy upstream` error message (with HTTP status code `503`). This means that there are currently no live replicas running for your API. This could happen for a few reasons:\n\n1. It's possible that your API is simply not ready yet. You can check the number of ready replicas on your API with `cortex get API_NAME`, and inspect the logs in CloudWatch with the help of `cortex logs API_NAME`.\n1. Your API may have errored during initialization or while responding to a previous request. `cortex describe API_NAME` will show the number of replicas that have failed to start on your API, and you can view the logs for all replicas by visiting the CloudWatch Insights URL from `cortex logs API_NAME`.\n\nIf you are using API Gateway in front of your API endpoints, it is also possible to receive a `{\"message\":\"Service Unavailable\"}` error message (with HTTP status code `503`) after 29 seconds if your request exceeds API Gateway's 29 second timeout. If this is the case, you can either modify your code to take less time, run on faster hardware (e.g. GPUs), or don't use API Gateway (there is no timeout when using the API's endpoint directly).\n\n## API is stuck updating\n\nIf your API has pods stuck in the \"pending\" or \"stalled\" states (which is displayed when running `cortex describe API_NAME`), there are a few possible causes. Here are some things to check:\n\n### Inspect API logs in CloudWatch\n\nUse `cortex logs API_NAME` for a URL to view logs for your API in CloudWatch. In addition to output from your containers, you will find logs from other parts of the Cortex infrastructure that may help your troubleshooting.\n\n### Check `max_instances` for your cluster\n\nWhen you created your Cortex cluster, you configured `max_instances` for each node group that you specified (via the cluster configuration file, e.g. `cluster.yaml`). If your cluster already has `min_instances` running instances for a given node group, additional instances cannot be created and APIs may not be able to deploy, scale, or update.\n\nYou can check the current value of `max_instances` for the selected node group by running `cortex cluster info --config cluster.yaml` (or `cortex cluster info --name <CLUSTER-NAME> --region <CLUSTER-REGION>` if you have the name and region of the cluster).\n\nOnce you have the name and region of the cluster, you can update the `max_instances` field by following the [instructions](../../clusters/management/update.md) to update an existing cluster.\n\n## Check your AWS auto scaling group activity history\n\nIn most cases when AWS is unable to provision additional instances, the reason will be logged in the auto scaling group's activity history.\n\nHere is how you can check these logs:\n\n1. Log in to the AWS console and go to the EC2 service page\n2. Click \"Auto Scaling Groups\" on the bottom of the side panel on the left\n3. Select one of the \"worker\" autoscaling groups for your cluster (there may be two)\n4. Click the \"Activity\" tab at the bottom half of the screen (it may also be called \"Activity History\" depending on which AWS console UI you're using)\n5. Scroll down (if necessary) and inspect the activity history, looking for any errors and their causes\n6. Repeat steps 3-5 for the other worker autoscaling group (if applicable)\n\nHere is how it looks on the new console UI:\n\n![new ui](https://user-images.githubusercontent.com/808475/78153371-852d2c00-742a-11ea-9bde-dbad5c603f8f.png)\n\nOn the old UI:\n\n![old ui](https://user-images.githubusercontent.com/808475/78153350-7e9eb480-742a-11ea-9221-1f6559db45fd.png)\n\nThe most common reason AWS is unable to provision instances is that you have reached your instance limit. There is an instance limit associated with your AWS account for each instance family in each region, for on-demand and for spot instances. You can check your current limit and request an increase [here](https://console.aws.amazon.com/servicequotas/home?#!/services/ec2/quotas) (set the region in the upper right corner to your desired region, type \"on-demand\" or \"spot\" in the search bar, and click on the quota that matches your instance type). Note that the quota values indicate the number of vCPUs available, not the number of instances; different instances have a different numbers of vCPUs, which can be seen [here](https://aws.amazon.com/ec2/instance-types).\n\nIf you're using spot instances for your node group, it is also possible that AWS has run out of spot instances for your requested instance type and region. To address this, you can try adding additional alternative instance types in `instance_distribution` or changing the cluster's region to one that has a higher availability.\n\n### Disabling rolling updates\n\nBy default, cortex performs rolling updates on all APIs. This is to ensure that traffic can continue to be served during updates, and that there is no downtime if there's an error in the new version. However, this can lead to APIs getting stuck in the \"updating\" state when the cluster is unable to increase its instance count (e.g. for one of the reasons above).\n\nHere is an example: You set `max_instances` to 1, or your AWS account limits you to a single `g4dn.xlarge` instance (i.e. your G instance vCPU limit is 4). You have an API running which requested 1 GPU. When you update your API via `cortex deploy`, Cortex attempts to deploy the updated version, and will only take down the old version once the new one is running. In this case, since there is no GPU available on the single running instance (it's taken by the old version of your API), the new version of your API requests a new instance to run on. Normally this will be ok (it might just take a few minutes since a new instance has to spin up): the new instance will become live, the new API replica will run on it, once it starts up successfully the old replica will be terminated, and eventually the old instance will spin down. In this case, however, the new version gets stuck because the second instance cannot be created, and the first instance cannot be freed up until the new version is running.\n\nIf you're running in a development environment, this rolling update behavior can be undesirable.\n\nYou can disable rolling updates for your API in your API configuration: set `max_surge` to 0 in the `update_strategy` section, E.g.:\n\n```yaml\n- name: hello-world\n  kind: RealtimeAPI\n  # ...\n  update_strategy:\n    max_surge: 0\n```\n"
  },
  {
    "path": "docs/workloads/task/configuration.md",
    "content": "# Configuration\n\n```yaml\n- name: <string>  # name of the API (required)\n  kind: TaskAPI  # must be \"TaskAPI\" for task APIs (required)\n  pod:  # pod configuration (required)\n    containers:  # configurations for the containers to run (at least one constainer must be provided)\n      - name: <string>  # name of the container (required)\n        image: <string>  # docker image to use for the container (required)\n        command: <list[string]>  # entrypoint (not executed within a shell); env vars can be used with e.g. $(CORTEX_PORT) (required)\n        args: <list[string]>  # arguments to the entrypoint; env vars can be used with e.g. $(CORTEX_PORT) (default: no args)\n        env: <map[string:string]>  # dictionary of environment variables to set in the container (optional)\n        compute:  # compute resource requests (default: see below)\n          cpu: <string|int|float>  # CPU request for the container; one unit of CPU corresponds to one virtual CPU; fractional requests are allowed, and can be specified as a floating point number or via the \"m\" suffix (default: 200m)\n          gpu: <int>  # GPU request for the container; one unit of GPU corresponds to one virtual GPU (default: 0)\n          inf: <int>  # Inferentia request for the container; one unit of inf corresponds to one virtual Inferentia chip (default: 0)\n          mem: <string>  # memory request for the container; one unit of memory is one byte and can be expressed as an integer or by using one of these suffixes: K, M, G, T (or their power-of two counterparts: Ki, Mi, Gi, Ti) (default: Null)\n          shm: <string>  # size of shared memory (/dev/shm) for sharing data between multiple processes, e.g. 64Mi or 1Gi (default: Null)\n        liveness_probe:  # periodic probe of container liveness; container will be restarted if the probe fails (optional)\n          http_get:  # specifies an http endpoint which must respond with status code 200 (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n            path: <string>  # the path to access on the HTTP server (default: /)\n          tcp_socket:  # specifies a port which must be ready to receive traffic (only one of http_get, tcp_socket, and exec may be specified)\n            port: <int|string>  # the port to access on the container (required)\n          exec:  # specifies a command to run which must exit with code 0 (only one of http_get, tcp_socket, and exec may be specified)\n            command: <list[string]>  # the command to execute inside the container, which is exec'd (not run inside a shell); the working directory is root ('/') in the container's filesystem (required)\n          initial_delay_seconds: <int>  # number of seconds after the container has started before the probe is initiated (default: 0)\n          timeout_seconds: <int>  # number of seconds until the probe times out (default: 1)\n          period_seconds: <int>  # how often (in seconds) to perform the probe (default: 10)\n          success_threshold: <int>  # minimum consecutive successes for the probe to be considered successful after having failed (default: 1)\n          failure_threshold: <int>  # minimum consecutive failures for the probe to be considered failed after having succeeded (default: 3)\n  node_groups: <list[string]>  # a list of node groups on which this API can run (default: all node groups are eligible)\n  networking:  # networking configuration (default: see below)\n    endpoint: <string>  # endpoint for the API (default: <api_name>)\n```\n"
  },
  {
    "path": "docs/workloads/task/containers.md",
    "content": "# Containers\n\n## Job specification\n\nIf you need access to any parameters in the job submission (e.g. `config`), the entire job specification is available at `/cortex/spec/job.json` in your API containers' filesystems.\n\n## Multiple containers\n\nYour Task's pod can contain multiple containers. The `/mnt` directory is mounted to each container's filesystem, and is shared across all containers.\n\n## Observability\n\nSee docs for [logging](../../clusters/observability/logging.md), [metrics](../../clusters/observability/metrics.md), and [alerting](../../clusters/observability/metrics.md).\n\n## Using the Cortex CLI or client\n\nIt is possible to use the Cortex CLI or client to interact with your cluster's APIs from within your API containers. All containers will have a CLI configuration file present at `/cortex/client/cli.yaml`, which is configured to connect to the cluster. In addition, the `CORTEX_CLI_CONFIG_DIR` environment variable is set to `/cortex/client` by default. Therefore, no additional configuration is required to use the CLI or Python client (which can be instantiated via `cortex.client()`).\n\nNote: your Cortex CLI or client must match the version of your cluster (available in the `CORTEX_VERSION` environment variable).\n\n## Chaining APIs\n\nIt is possible to submit Task jobs from any Cortex API within a Cortex cluster. Jobs can be submitted to `http://ingressgateway-operator.istio-system.svc.cluster.local/tasks/<api_name>`, where `<api_name>` is the name of the Task API you are making a request to.\n\nFor example, if there is a Task API named `hello-world` running in the cluster, you can make a request to it from a different API in Python by using:\n\n```python\nimport requests\n\nresponse = requests.post(\n    \"http://ingressgateway-operator.istio-system.svc.cluster.local/tasks/hello-world\",\n    json={\"config\": {\"my_key\": \"my_value\"}},\n)\n```\n\nTo make requests from your Task API to a Realtime, Batch, or Async API running within the cluster, see the \"Chaining APIs\" docs associated with the target workload type.\n"
  },
  {
    "path": "docs/workloads/task/example.md",
    "content": "# TaskAPI\n\n### Define an API\n\n```python\n# main.py\n\nprint(\"hello world\")\n```\n\n### Create a `Dockerfile`\n\n```Dockerfile\nFROM python:3.8-slim\n\nCOPY main.py /\n\nCMD exec python main.py\n```\n\n### Build an image\n\n```bash\ndocker build . -t hello-world\n```\n\n### Run a container locally\n\n```bash\ndocker run -it --rm hello-world\n```\n\n### Login to ECR\n\n```bash\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com\n```\n\n### Create a repository\n\n```bash\naws ecr create-repository --repository-name hello-world\n```\n\n### Tag the image\n\n```bash\ndocker tag hello-world <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Push the image\n\n```bash\ndocker push <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n```\n\n### Configure a Cortex deployment\n\n```yaml\n# cortex.yaml\n\n- name: hello-world\n  kind: TaskAPI\n  pod:\n    containers:\n    - name: api\n      image: <AWS_ACCOUNT_ID>.dkr.ecr.us-east-1.amazonaws.com/hello-world\n      command: [\"python\", \"main.py\"]\n```\n\n### Create a Cortex deployment\n\n```bash\ncortex deploy\n```\n\n### Get the API endpoint\n\n```bash\ncortex get hello-world\n```\n\n### Make a request\n\n```bash\ncurl -X POST -H \"Content-Type: application/json\" -d '{}' http://***.amazonaws.com/hello-world\n```\n\n### View the logs\n\n```bash\ncortex logs hello-world <JOB_ID>\n```\n"
  },
  {
    "path": "docs/workloads/task/jobs.md",
    "content": "# TaskAPI jobs\n\n## Get the Task API's endpoint\n\n```bash\ncortex get <task_api_name>\n```\n\n## Submit a Job\n\n```yaml\nPOST <task_api_endpoint>:\n{\n    \"timeout\": <int>,   # duration in seconds since the submission of a job before it is terminated (optional)\n    \"config\": {         # arbitrary input for this specific job (optional)\n        \"string\": <any>\n    }\n}\n\nRESPONSE:\n{\n    \"job_id\": <string>,\n    \"api_name\": <string>,\n    \"kind\": \"TaskAPI\",\n    \"workers\": 1,\n    \"config\": {<string>: <any>},\n    \"api_id\": <string>,\n    \"timeout\": <int>,\n    \"created_time\": <string>\n}\n```\n\nThe entire job specification is written to `/cortex/spec/job.json` in the API containers.\n\n## Get a job's status\n\n```bash\ncortex get <task_api_name> <job_id>\n```\n\nOr make a GET request to `<task_api_endpoint>?jobID=<jobID>`:\n\n```yaml\nGET <task_api_endpoint>?jobID=<jobID>:\n\nRESPONSE:\n{\n    \"job_status\": {\n        \"job_id\": <string>,\n        \"api_name\": <string>,\n        \"kind\": \"TaskAPI\",\n        \"workers\": 1,\n        \"config\": {<string>: <any>},\n        \"api_id\": <string>,\n        \"status\": <string>,\n        \"created_time\": <string>\n        \"start_time\": <string>\n        \"end_time\": <string> (optional)\n    },\n    \"endpoint\": <string>\n    \"api_spec\": {\n        ...\n    }\n}\n```\n\n## Stop a job\n\n```bash\ncortex delete <task_api_name> <job_id>\n```\n\nOr make a DELETE request to `<task_api_endpoint>?jobID=<jobID>`:\n\n```yaml\nDELETE <task_api_endpoint>?jobID=<jobID>:\n\nRESPONSE:\n{\"message\":\"stopped job <job_id>\"}\n```\n"
  },
  {
    "path": "docs/workloads/task/statuses.md",
    "content": "# Job statuses\n\n| Status                   | Meaning |\n| :--- | :--- |\n| running                  | Task is running |\n| succeeded                | Task has finished without errors |\n| worker error             | The task has experienced an irrecoverable error, causing the job to fail; check job logs for more details |\n| out of memory            | The task has ran out of memory, causing the job to fail; check job logs for more details |\n| timed out                | Job was terminated after the specified timeout has elapsed |\n| stopped                  | Job was stopped by the user or the Task API was deleted |\n"
  },
  {
    "path": "docs/workloads/task/task.md",
    "content": "# Task\n\nTask APIs provide a lambda-style execution of containers. They are useful for running your containers on demand.\n\nTask APIs are a good fit when you need to trigger container execution via an HTTP request. They can be used to run tasks (e.g. training models), and can be configured as task runners for orchestrators (such as airflow).\n\n**Key Features**\n\n* run containers on-demand\n* scale to 0 (when there are no tasks)\n* automatically recover from failures and spot instance termination\n\n## How it works\n\nWhen you deploy a Task API, an endpoint is created to receive task submissions. Upon submitting a Task, Cortex will respond with a Task ID and will asynchronously trigger the execution of a Task.\n\nCortex will initialize a worker pod based on your API specification. After the worker pod runs to completion, the Task is marked as completed and the pod is terminated.\n\nYou can make GET requests to the Task API endpoint to retrieve the status of the Task.\n\n![](https://user-images.githubusercontent.com/808475/146854267-3785e8ee-4233-4473-a0db-37a5c5438fb4.png)\n"
  },
  {
    "path": "get-cli.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -e\n\nCORTEX_VERSION_BRANCH_STABLE=master\nCORTEX_INSTALL_PATH=\"${CORTEX_INSTALL_PATH:-/usr/local/bin/cortex}\"\n\n# replace ~ with the home directory path\nCORTEX_INSTALL_PATH=\"${CORTEX_INSTALL_PATH/#\\~/$HOME}\"\n\ncase \"$OSTYPE\" in\n  darwin*)  parsed_os=\"darwin\" ;;\n  linux*)   parsed_os=\"linux\" ;;\n  *)        echo -e \"\\nerror: only mac and linux are supported\"; exit 1 ;;\nesac\n\nfunction main() {\n  echo -e \"\\ndownloading cli (${CORTEX_INSTALL_PATH}) ...\\n\"\n\n  cortex_sh_tmp_dir=\"$HOME/.cortex-sh-tmp\"\n  rm -rf $cortex_sh_tmp_dir && mkdir -p $cortex_sh_tmp_dir\n\n  if command -v curl >/dev/null; then\n    curl -s -w \"\" -o $cortex_sh_tmp_dir/cortex https://s3-us-west-2.amazonaws.com/get-cortex/$CORTEX_VERSION_BRANCH_STABLE/cli/$parsed_os/cortex\n  elif command -v wget >/dev/null; then\n    wget -q -O $cortex_sh_tmp_dir/cortex https://s3-us-west-2.amazonaws.com/get-cortex/$CORTEX_VERSION_BRANCH_STABLE/cli/$parsed_os/cortex\n  else\n    echo \"error: please install \\`curl\\` or \\`wget\\`\"\n    exit 1\n  fi\n\n  chmod +x $cortex_sh_tmp_dir/cortex\n\n  if [ $(id -u) = 0 ]; then\n    mv -f $cortex_sh_tmp_dir/cortex $CORTEX_INSTALL_PATH\n  else\n    ask_sudo\n    sudo mv -f $cortex_sh_tmp_dir/cortex $CORTEX_INSTALL_PATH\n  fi\n\n  rm -rf $cortex_sh_tmp_dir\n  echo \"✓ installed cli\"\n\n  # prompt to update bash profile if running interactively\n  if [ -t 1 ]; then\n    update_bash_profile\n  fi\n}\n\nfunction ask_sudo() {\n  if ! sudo -n true 2>/dev/null; then\n    echo -e \"please enter your sudo password\\n\"\n  fi\n}\n\nfunction get_bash_profile_path() {\n  if [ \"$parsed_os\" = \"darwin\" ]; then\n    if [ -f $HOME/.bash_profile ]; then\n      echo $HOME/.bash_profile\n      return\n    elif [ -f $HOME/.bashrc ]; then\n      echo $HOME/.bashrc\n      return\n    fi\n  else\n    if [ -f $HOME/.bashrc ]; then\n      echo $HOME/.bashrc\n      return\n    elif [ -f $HOME/.bash_profile ]; then\n      echo $HOME/.bash_profile\n      return\n    fi\n  fi\n\n  echo \"\"\n}\n\nfunction get_zsh_profile_path() {\n  if [ -f $HOME/.zshrc ]; then\n    echo $HOME/.zshrc\n    return\n  fi\n\n  echo \"\"\n}\n\nfunction guess_if_bash_completion_installed() {\n  if [ -f $HOME/.bash_profile ]; then\n    if grep -q -e \"^[^#]*bash-completion\" -e \"^[^#]*bash_completion\" \"$HOME/.bash_profile\"; then\n      echo \"true\"\n      return\n    fi\n  fi\n\n  if [ -f $HOME/.bashrc ]; then\n    if grep -q -e \"^[^#]*bash-completion\" -e \"^[^#]*bash_completion\" \"$HOME/.bashrc\"; then\n      echo \"true\"\n      return\n    fi\n  fi\n\n  echo \"false\"\n}\n\nfunction update_bash_profile() {\n  bash_profile_path=$(get_bash_profile_path)\n  zsh_profile_path=$(get_zsh_profile_path)\n  maybe_bash_completion_installed=$(guess_if_bash_completion_installed)\n  did_locate_shell_profile=\"false\"\n\n  if [ \"$bash_profile_path\" != \"\" ]; then\n    did_locate_shell_profile=\"true\"\n    if ! grep -Fxq \"source <(cortex completion bash)\" \"$bash_profile_path\"; then\n      echo\n      read -p \"Would you like to modify your bash profile ($bash_profile_path) to enable cortex command completion and the cx alias in bash? [y/n] \" -r\n      echo\n      if [ \"$REPLY\" = \"y\" ] || [ \"$REPLY\" = \"Y\" ] || [ \"$REPLY\" = \"yes\" ] || [ \"$REPLY\" = \"Yes\" ] || [ \"$REPLY\" = \"YES\" ]; then\n        echo -e \"\\nsource <(cortex completion bash)\" >> $bash_profile_path\n        echo -e \"✓ Your bash profile has been updated\"\n        echo -e \"\\nCommand to update your current terminal session:\"\n        echo \"  source $bash_profile_path\"\n        if [ ! \"$maybe_bash_completion_installed\" = \"true\" ]; then\n          echo -e \"Note: \\`bash-completion\\` must be installed on your system for cortex command completion to function properly\"\n        fi\n      else\n        echo -e \"Your bash profile has not been modified (run \\`cortex completion --help\\` to show how to enable bash completion manually)\"\n      fi\n    fi\n  fi\n\n  if [ \"$zsh_profile_path\" != \"\" ]; then\n    did_locate_shell_profile=\"true\"\n    if ! grep -Fxq \"source <(cortex completion zsh)\" \"$zsh_profile_path\"; then\n      echo\n      read -p \"Would you like to modify your zsh profile ($zsh_profile_path) to enable cortex command completion and the cx alias in zsh? [y/n] \" -r\n      echo\n      if [ \"$REPLY\" = \"y\" ] || [ \"$REPLY\" = \"Y\" ] || [ \"$REPLY\" = \"yes\" ] || [ \"$REPLY\" = \"Yes\" ] || [ \"$REPLY\" = \"YES\" ]; then\n        echo -e \"\\nsource <(cortex completion zsh)\" >> $zsh_profile_path\n        echo -e \"✓ Your zsh profile has been updated\"\n        echo -e \"\\nStart a new zsh shell for completion to take effect. If completion still doesn't work, you can try adding this line to the top of $zsh_profile_path: autoload -Uz compinit && compinit\"\n      else\n        echo -e \"Your zsh profile has not been modified (run \\`cortex completion --help\\` to show how to enable zsh completion manually)\"\n      fi\n    fi\n  fi\n\n  if [ \"$did_locate_shell_profile\" = \"false\" ]; then\n    echo -e \"\\nIf you would like to enable cortex bash completion and the cx alias, run \\`cortex completion --help\\` for instructions\"\n  fi\n}\n\nmain\n"
  },
  {
    "path": "go.mod",
    "content": "module github.com/cortexlabs/cortex\n\ngo 1.17\n\nrequire (\n\tgithub.com/DataDog/datadog-go v4.8.0+incompatible\n\tgithub.com/aws/amazon-vpc-cni-k8s v1.11.3\n\tgithub.com/aws/aws-sdk-go v1.43.29\n\tgithub.com/cortexlabs/go-input v0.0.0-20200503032952-8b67a7a7b28d\n\tgithub.com/cortexlabs/yaml v0.0.0-20210628201654-31e52ba8433b\n\tgithub.com/davecgh/go-spew v1.1.1\n\tgithub.com/denormal/go-gitignore v0.0.0-20180930084346-ae8ad1d07817\n\tgithub.com/docker/docker v20.10.7+incompatible\n\tgithub.com/fatih/color v1.12.0\n\tgithub.com/getsentry/sentry-go v0.11.0\n\tgithub.com/go-logr/logr v0.4.0\n\tgithub.com/gobwas/glob v0.2.3\n\tgithub.com/google/uuid v1.2.0\n\tgithub.com/gorilla/handlers v1.5.1\n\tgithub.com/gorilla/mux v1.8.0\n\tgithub.com/gorilla/websocket v1.4.2\n\tgithub.com/mitchellh/go-homedir v1.1.0\n\tgithub.com/onsi/ginkgo v1.16.4\n\tgithub.com/onsi/gomega v1.15.0\n\tgithub.com/ory/dockertest/v3 v3.7.0\n\tgithub.com/patrickmn/go-cache v2.1.0+incompatible\n\tgithub.com/pkg/errors v0.9.1\n\tgithub.com/prometheus/client_golang v1.11.0\n\tgithub.com/prometheus/common v0.29.0\n\tgithub.com/shirou/gopsutil v3.21.6+incompatible\n\tgithub.com/spf13/cobra v1.5.0\n\tgithub.com/spf13/pflag v1.0.5\n\tgithub.com/stretchr/testify v1.8.0\n\tgithub.com/ugorji/go/codec v1.2.6\n\tgithub.com/xlab/treeprint v1.0.0\n\tgithub.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c\n\tgo.uber.org/atomic v1.8.0\n\tgo.uber.org/zap v1.19.0\n\tgopkg.in/segmentio/analytics-go.v3 v3.1.0\n\tistio.io/api v0.0.0-20220304045306-249321c725a9\n\tistio.io/client-go v1.10.6\n\tk8s.io/api v0.22.11\n\tk8s.io/apimachinery v0.22.11\n\tk8s.io/client-go v0.22.11\n\tk8s.io/metrics v0.22.11\n\tsigs.k8s.io/aws-iam-authenticator v0.5.3\n\tsigs.k8s.io/controller-runtime v0.10.3\n)\n\nrequire (\n\tcloud.google.com/go v0.81.0 // indirect\n\tgithub.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect\n\tgithub.com/Azure/go-autorest v14.2.0+incompatible // indirect\n\tgithub.com/Azure/go-autorest/autorest v0.11.18 // indirect\n\tgithub.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect\n\tgithub.com/Azure/go-autorest/autorest/date v0.3.0 // indirect\n\tgithub.com/Azure/go-autorest/logger v0.2.1 // indirect\n\tgithub.com/Azure/go-autorest/tracing v0.6.0 // indirect\n\tgithub.com/Microsoft/go-winio v0.5.0 // indirect\n\tgithub.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect\n\tgithub.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/cenkalti/backoff/v4 v4.1.1 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.1.1 // indirect\n\tgithub.com/containerd/containerd v1.5.4 // indirect\n\tgithub.com/containerd/continuity v0.1.0 // indirect\n\tgithub.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect\n\tgithub.com/docker/cli v20.10.7+incompatible // indirect\n\tgithub.com/docker/distribution v2.7.1+incompatible // indirect\n\tgithub.com/docker/go-connections v0.4.0 // indirect\n\tgithub.com/docker/go-units v0.4.0 // indirect\n\tgithub.com/evanphx/json-patch v4.11.0+incompatible // indirect\n\tgithub.com/felixge/httpsnoop v1.0.2 // indirect\n\tgithub.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect\n\tgithub.com/fsnotify/fsnotify v1.5.4 // indirect\n\tgithub.com/go-logr/zapr v0.4.0 // indirect\n\tgithub.com/go-ole/go-ole v1.2.5 // indirect\n\tgithub.com/gofrs/flock v0.7.0 // indirect\n\tgithub.com/gogo/protobuf v1.3.2 // indirect\n\tgithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect\n\tgithub.com/golang/protobuf v1.5.2 // indirect\n\tgithub.com/google/go-cmp v0.5.6 // indirect\n\tgithub.com/google/gofuzz v1.2.0 // indirect\n\tgithub.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect\n\tgithub.com/googleapis/gnostic v0.5.5 // indirect\n\tgithub.com/imdario/mergo v0.3.12 // indirect\n\tgithub.com/inconshreveable/mousetrap v1.0.0 // indirect\n\tgithub.com/jmespath/go-jmespath v0.4.0 // indirect\n\tgithub.com/json-iterator/go v1.1.11 // indirect\n\tgithub.com/kr/pretty v0.3.0 // indirect\n\tgithub.com/mattn/go-colorable v0.1.12 // indirect\n\tgithub.com/mattn/go-isatty v0.0.14 // indirect\n\tgithub.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect\n\tgithub.com/mitchellh/mapstructure v1.4.1 // indirect\n\tgithub.com/moby/spdystream v0.2.0 // indirect\n\tgithub.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect\n\tgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect\n\tgithub.com/modern-go/reflect2 v1.0.1 // indirect\n\tgithub.com/morikuni/aec v1.0.0 // indirect\n\tgithub.com/nxadm/tail v1.4.8 // indirect\n\tgithub.com/opencontainers/go-digest v1.0.0 // indirect\n\tgithub.com/opencontainers/image-spec v1.0.1 // indirect\n\tgithub.com/opencontainers/runc v1.0.0 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.0 // indirect\n\tgithub.com/prometheus/client_model v0.2.0 // indirect\n\tgithub.com/prometheus/procfs v0.6.0 // indirect\n\tgithub.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect\n\tgithub.com/sirupsen/logrus v1.8.1 // indirect\n\tgithub.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect\n\tgithub.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect\n\tgithub.com/xeipuuv/gojsonschema v1.2.0 // indirect\n\tgo.uber.org/multierr v1.7.0 // indirect\n\tgolang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect\n\tgolang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect\n\tgolang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect\n\tgolang.org/x/sys v0.0.0-20220704084225-05e143d24a9e // indirect\n\tgolang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect\n\tgolang.org/x/text v0.3.7 // indirect\n\tgolang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect\n\tgolang.org/x/tools v0.1.11 // indirect\n\tgomodules.xyz/jsonpatch/v2 v2.2.0 // indirect\n\tgoogle.golang.org/appengine v1.6.7 // indirect\n\tgoogle.golang.org/genproto v0.0.0-20210701133433-6b8dcf568a95 // indirect\n\tgoogle.golang.org/grpc v1.39.0 // indirect\n\tgoogle.golang.org/protobuf v1.27.1 // indirect\n\tgopkg.in/inf.v0 v0.9.1 // indirect\n\tgopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect\n\tgopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect\n\tgopkg.in/yaml.v2 v2.4.0 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n\tistio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f // indirect\n\tk8s.io/apiextensions-apiserver v0.22.2 // indirect\n\tk8s.io/component-base v0.22.2 // indirect\n\tk8s.io/klog/v2 v2.9.0 // indirect\n\tk8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513 // indirect\n\tk8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect\n\tsigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect\n\tsigs.k8s.io/yaml v1.2.0 // indirect\n)\n\nreplace github.com/docker/docker => github.com/docker/engine v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible\n"
  },
  {
    "path": "go.sum",
    "content": "bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=\ncloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=\ncloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=\ncloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=\ncloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=\ncloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=\ncloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=\ncloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=\ncloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=\ncloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=\ncloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=\ncloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=\ncloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=\ncloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=\ncloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=\ncloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=\ncloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=\ncloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=\ncloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=\ncloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=\ncloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=\ncloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=\ncloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=\ncloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=\ncloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=\ncloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=\ncloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=\ncloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=\ncloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=\ncloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=\ncloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=\ncloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=\ncloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=\ncloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=\ncloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=\ncloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=\ncloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=\ncloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=\ndmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=\ngithub.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=\ngithub.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=\ngithub.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=\ngithub.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=\ngithub.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=\ngithub.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=\ngithub.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=\ngithub.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=\ngithub.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=\ngithub.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=\ngithub.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=\ngithub.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM=\ngithub.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=\ngithub.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=\ngithub.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=\ngithub.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=\ngithub.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q=\ngithub.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=\ngithub.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=\ngithub.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=\ngithub.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=\ngithub.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=\ngithub.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=\ngithub.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=\ngithub.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=\ngithub.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=\ngithub.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=\ngithub.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=\ngithub.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=\ngithub.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=\ngithub.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=\ngithub.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=\ngithub.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=\ngithub.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=\ngithub.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=\ngithub.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=\ngithub.com/DataDog/datadog-go v4.8.0+incompatible h1:vjzonG+3XzZgYrumNmdrA4QpXju/ZXrwb0mRjpYYbuo=\ngithub.com/DataDog/datadog-go v4.8.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=\ngithub.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=\ngithub.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=\ngithub.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=\ngithub.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=\ngithub.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=\ngithub.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=\ngithub.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=\ngithub.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=\ngithub.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=\ngithub.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU=\ngithub.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=\ngithub.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=\ngithub.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=\ngithub.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ=\ngithub.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=\ngithub.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=\ngithub.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=\ngithub.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=\ngithub.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=\ngithub.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=\ngithub.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=\ngithub.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=\ngithub.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=\ngithub.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=\ngithub.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=\ngithub.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=\ngithub.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=\ngithub.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=\ngithub.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=\ngithub.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=\ngithub.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=\ngithub.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=\ngithub.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY=\ngithub.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=\ngithub.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=\ngithub.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=\ngithub.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=\ngithub.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=\ngithub.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=\ngithub.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=\ngithub.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=\ngithub.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=\ngithub.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=\ngithub.com/aws/amazon-vpc-cni-k8s v1.11.3 h1:VlxS3LrwMbs6/wjZiOphj+9cjYaBN4SgQxmhwwMormY=\ngithub.com/aws/amazon-vpc-cni-k8s v1.11.3/go.mod h1:w8f1LLPue3xzRw7F9NPNWZt1B/yj7YNpypIuFMBmnyU=\ngithub.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=\ngithub.com/aws/aws-sdk-go v1.37.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=\ngithub.com/aws/aws-sdk-go v1.43.29 h1:P6tBpMLwVLS/QwPkaBxfDIF3SmPouoacIk+/7NKnDxY=\ngithub.com/aws/aws-sdk-go v1.43.29/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=\ngithub.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=\ngithub.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=\ngithub.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=\ngithub.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=\ngithub.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=\ngithub.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=\ngithub.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=\ngithub.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=\ngithub.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=\ngithub.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=\ngithub.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=\ngithub.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=\ngithub.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=\ngithub.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=\ngithub.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=\ngithub.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=\ngithub.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=\ngithub.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=\ngithub.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=\ngithub.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=\ngithub.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=\ngithub.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=\ngithub.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=\ngithub.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=\ngithub.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=\ngithub.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=\ngithub.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=\ngithub.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=\ngithub.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=\ngithub.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=\ngithub.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=\ngithub.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg=\ngithub.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=\ngithub.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=\ngithub.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=\ngithub.com/cilium/ebpf v0.6.1/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=\ngithub.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=\ngithub.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=\ngithub.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=\ngithub.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=\ngithub.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=\ngithub.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=\ngithub.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=\ngithub.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=\ngithub.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=\ngithub.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=\ngithub.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=\ngithub.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=\ngithub.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E=\ngithub.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=\ngithub.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=\ngithub.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=\ngithub.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=\ngithub.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=\ngithub.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=\ngithub.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=\ngithub.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=\ngithub.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=\ngithub.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=\ngithub.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=\ngithub.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=\ngithub.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=\ngithub.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=\ngithub.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=\ngithub.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=\ngithub.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=\ngithub.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=\ngithub.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=\ngithub.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=\ngithub.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=\ngithub.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=\ngithub.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ=\ngithub.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=\ngithub.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=\ngithub.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=\ngithub.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=\ngithub.com/containerd/containerd v1.5.4 h1:uPF0og3ByFzDnaStfiQj3fVGTEtaSNyU+bW7GR/nqGA=\ngithub.com/containerd/containerd v1.5.4/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw=\ngithub.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=\ngithub.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=\ngithub.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=\ngithub.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=\ngithub.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=\ngithub.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=\ngithub.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=\ngithub.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=\ngithub.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=\ngithub.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=\ngithub.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=\ngithub.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=\ngithub.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=\ngithub.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=\ngithub.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=\ngithub.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU=\ngithub.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=\ngithub.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=\ngithub.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=\ngithub.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=\ngithub.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=\ngithub.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=\ngithub.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0=\ngithub.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA=\ngithub.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=\ngithub.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=\ngithub.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=\ngithub.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=\ngithub.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=\ngithub.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=\ngithub.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=\ngithub.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=\ngithub.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=\ngithub.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=\ngithub.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=\ngithub.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=\ngithub.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=\ngithub.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=\ngithub.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw=\ngithub.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y=\ngithub.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=\ngithub.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=\ngithub.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=\ngithub.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=\ngithub.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=\ngithub.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=\ngithub.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=\ngithub.com/containernetworking/plugins v0.9.0/go.mod h1:dbWv4dI0QrBGuVgj+TuVQ6wJRZVOhrCQj91YyC92sxg=\ngithub.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=\ngithub.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=\ngithub.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4=\ngithub.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=\ngithub.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=\ngithub.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=\ngithub.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=\ngithub.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=\ngithub.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=\ngithub.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=\ngithub.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=\ngithub.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=\ngithub.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=\ngithub.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=\ngithub.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=\ngithub.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=\ngithub.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=\ngithub.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=\ngithub.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=\ngithub.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=\ngithub.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=\ngithub.com/cortexlabs/go-input v0.0.0-20200503032952-8b67a7a7b28d h1:2SmODObcy2ekPA0eLFlR/+Vu5Yo2hoVbNMJ+vWzinpo=\ngithub.com/cortexlabs/go-input v0.0.0-20200503032952-8b67a7a7b28d/go.mod h1:rxijm42+fHjyPbFGqUTzUUhioEVg5LUi+w53dlY0WUM=\ngithub.com/cortexlabs/yaml v0.0.0-20210628201654-31e52ba8433b h1:jS+jWCjvtnZJ9f9GGR9eA6EgPBhKZ3ILUvTmMglYzyw=\ngithub.com/cortexlabs/yaml v0.0.0-20210628201654-31e52ba8433b/go.mod h1:qYZ6ij3BG8ss8YpnSPcpdl9BCWIWyStg8OcyVOFlJKI=\ngithub.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=\ngithub.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=\ngithub.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=\ngithub.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=\ngithub.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=\ngithub.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=\ngithub.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=\ngithub.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=\ngithub.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=\ngithub.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=\ngithub.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=\ngithub.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=\ngithub.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/denormal/go-gitignore v0.0.0-20180930084346-ae8ad1d07817 h1:0nsrg//Dc7xC74H/TZ5sYR8uk4UQRNjsw8zejqH5a4Q=\ngithub.com/denormal/go-gitignore v0.0.0-20180930084346-ae8ad1d07817/go.mod h1:C/+sI4IFnEpCn6VQ3GIPEp+FrQnQw+YQP3+n+GdGq7o=\ngithub.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=\ngithub.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=\ngithub.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=\ngithub.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=\ngithub.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=\ngithub.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=\ngithub.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=\ngithub.com/docker/cli v20.10.7+incompatible h1:pv/3NqibQKphWZiAskMzdz8w0PRbtTaEB+f6NwdU7Is=\ngithub.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=\ngithub.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=\ngithub.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=\ngithub.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=\ngithub.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=\ngithub.com/docker/engine v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible h1:nnCzIfwUkdP7f5ZYf8el5qKoWSwuQxEeTcDwWHLKsKg=\ngithub.com/docker/engine v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=\ngithub.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=\ngithub.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=\ngithub.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=\ngithub.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=\ngithub.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=\ngithub.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=\ngithub.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=\ngithub.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=\ngithub.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=\ngithub.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=\ngithub.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=\ngithub.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=\ngithub.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=\ngithub.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=\ngithub.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=\ngithub.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=\ngithub.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=\ngithub.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=\ngithub.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=\ngithub.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=\ngithub.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=\ngithub.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=\ngithub.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=\ngithub.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=\ngithub.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=\ngithub.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=\ngithub.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=\ngithub.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=\ngithub.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs=\ngithub.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=\ngithub.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=\ngithub.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=\ngithub.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=\ngithub.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc=\ngithub.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=\ngithub.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=\ngithub.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=\ngithub.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=\ngithub.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=\ngithub.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=\ngithub.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c=\ngithub.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=\ngithub.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=\ngithub.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=\ngithub.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=\ngithub.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=\ngithub.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=\ngithub.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=\ngithub.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=\ngithub.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=\ngithub.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=\ngithub.com/getsentry/sentry-go v0.11.0 h1:qro8uttJGvNAMr5CLcFI9CHR0aDzXl0Vs3Pmw/oTPg8=\ngithub.com/getsentry/sentry-go v0.11.0/go.mod h1:KBQIxiZAetw62Cj8Ri964vAEWVdgfaUCn30Q3bCvANo=\ngithub.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=\ngithub.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=\ngithub.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=\ngithub.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=\ngithub.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=\ngithub.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=\ngithub.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=\ngithub.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=\ngithub.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=\ngithub.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=\ngithub.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=\ngithub.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=\ngithub.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=\ngithub.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=\ngithub.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=\ngithub.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=\ngithub.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=\ngithub.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=\ngithub.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=\ngithub.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=\ngithub.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU=\ngithub.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM=\ngithub.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=\ngithub.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=\ngithub.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=\ngithub.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=\ngithub.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=\ngithub.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=\ngithub.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=\ngithub.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=\ngithub.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=\ngithub.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=\ngithub.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=\ngithub.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=\ngithub.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=\ngithub.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=\ngithub.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=\ngithub.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=\ngithub.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=\ngithub.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=\ngithub.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=\ngithub.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=\ngithub.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=\ngithub.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=\ngithub.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=\ngithub.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=\ngithub.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=\ngithub.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=\ngithub.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=\ngithub.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=\ngithub.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=\ngithub.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g=\ngithub.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=\ngithub.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=\ngithub.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=\ngithub.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=\ngithub.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=\ngithub.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=\ngithub.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=\ngithub.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=\ngithub.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=\ngithub.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=\ngithub.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=\ngithub.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=\ngithub.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=\ngithub.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=\ngithub.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=\ngithub.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=\ngithub.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=\ngithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=\ngithub.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw=\ngithub.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=\ngithub.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=\ngithub.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=\ngithub.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=\ngithub.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=\ngithub.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=\ngithub.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=\ngithub.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=\ngithub.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=\ngithub.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=\ngithub.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=\ngithub.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=\ngithub.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=\ngithub.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=\ngithub.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=\ngithub.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=\ngithub.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=\ngithub.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=\ngithub.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=\ngithub.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=\ngithub.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=\ngithub.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=\ngithub.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=\ngithub.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=\ngithub.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=\ngithub.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=\ngithub.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=\ngithub.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=\ngithub.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=\ngithub.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=\ngithub.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=\ngithub.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=\ngithub.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=\ngithub.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=\ngithub.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=\ngithub.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=\ngithub.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=\ngithub.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=\ngithub.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=\ngithub.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=\ngithub.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=\ngithub.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=\ngithub.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=\ngithub.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=\ngithub.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=\ngithub.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=\ngithub.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=\ngithub.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=\ngithub.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=\ngithub.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=\ngithub.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=\ngithub.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=\ngithub.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=\ngithub.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=\ngithub.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=\ngithub.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=\ngithub.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=\ngithub.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=\ngithub.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=\ngithub.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=\ngithub.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=\ngithub.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=\ngithub.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=\ngithub.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=\ngithub.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=\ngithub.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=\ngithub.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=\ngithub.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=\ngithub.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=\ngithub.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=\ngithub.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=\ngithub.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=\ngithub.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=\ngithub.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=\ngithub.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=\ngithub.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=\ngithub.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=\ngithub.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=\ngithub.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=\ngithub.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=\ngithub.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=\ngithub.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=\ngithub.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=\ngithub.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=\ngithub.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=\ngithub.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=\ngithub.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=\ngithub.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=\ngithub.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=\ngithub.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=\ngithub.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=\ngithub.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=\ngithub.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8=\ngithub.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE=\ngithub.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=\ngithub.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=\ngithub.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=\ngithub.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=\ngithub.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=\ngithub.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=\ngithub.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=\ngithub.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=\ngithub.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=\ngithub.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=\ngithub.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=\ngithub.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=\ngithub.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2 h1:hRGSmZu7j271trc9sneMrpOW7GN5ngLm8YUZIPzf394=\ngithub.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=\ngithub.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=\ngithub.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=\ngithub.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=\ngithub.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=\ngithub.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=\ngithub.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=\ngithub.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=\ngithub.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=\ngithub.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=\ngithub.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=\ngithub.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=\ngithub.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=\ngithub.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=\ngithub.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=\ngithub.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=\ngithub.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=\ngithub.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=\ngithub.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=\ngithub.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=\ngithub.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=\ngithub.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=\ngithub.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=\ngithub.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=\ngithub.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=\ngithub.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=\ngithub.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=\ngithub.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=\ngithub.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=\ngithub.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=\ngithub.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=\ngithub.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=\ngithub.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=\ngithub.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=\ngithub.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=\ngithub.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=\ngithub.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=\ngithub.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=\ngithub.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=\ngithub.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=\ngithub.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=\ngithub.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=\ngithub.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=\ngithub.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=\ngithub.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=\ngithub.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=\ngithub.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=\ngithub.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=\ngithub.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=\ngithub.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=\ngithub.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=\ngithub.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=\ngithub.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=\ngithub.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc=\ngithub.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=\ngithub.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=\ngithub.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=\ngithub.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=\ngithub.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=\ngithub.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=\ngithub.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=\ngithub.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=\ngithub.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=\ngithub.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=\ngithub.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=\ngithub.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=\ngithub.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=\ngithub.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=\ngithub.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=\ngithub.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=\ngithub.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=\ngithub.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=\ngithub.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=\ngithub.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=\ngithub.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=\ngithub.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=\ngithub.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=\ngithub.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=\ngithub.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=\ngithub.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=\ngithub.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=\ngithub.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=\ngithub.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=\ngithub.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=\ngithub.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=\ngithub.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=\ngithub.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=\ngithub.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=\ngithub.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=\ngithub.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=\ngithub.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=\ngithub.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=\ngithub.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=\ngithub.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=\ngithub.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=\ngithub.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=\ngithub.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=\ngithub.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=\ngithub.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=\ngithub.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=\ngithub.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=\ngithub.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=\ngithub.com/opencontainers/runc v1.0.0 h1:QOhAQAYUlKeofuyeKdR6ITvOnXLPbEAjPMjz9wCUXcU=\ngithub.com/opencontainers/runc v1.0.0/go.mod h1:MU2S3KEB2ZExnhnAQYbwjdYV6HwKtDlNbA2Z2OeNDeA=\ngithub.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=\ngithub.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=\ngithub.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=\ngithub.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=\ngithub.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=\ngithub.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=\ngithub.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=\ngithub.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=\ngithub.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=\ngithub.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=\ngithub.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=\ngithub.com/ory/dockertest/v3 v3.7.0 h1:Bijzonc69Ont3OU0a3TWKJ1Rzlh3TsDXP1JrTAkSmsM=\ngithub.com/ory/dockertest/v3 v3.7.0/go.mod h1:PvCCgnP7AfBZeVrzwiUTjZx/IUXlGLC1zQlUQrLIlUE=\ngithub.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=\ngithub.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=\ngithub.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=\ngithub.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=\ngithub.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=\ngithub.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=\ngithub.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=\ngithub.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=\ngithub.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=\ngithub.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=\ngithub.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=\ngithub.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=\ngithub.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=\ngithub.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=\ngithub.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=\ngithub.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=\ngithub.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=\ngithub.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=\ngithub.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=\ngithub.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=\ngithub.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=\ngithub.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=\ngithub.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=\ngithub.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=\ngithub.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=\ngithub.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=\ngithub.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE=\ngithub.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=\ngithub.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=\ngithub.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=\ngithub.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=\ngithub.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=\ngithub.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=\ngithub.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=\ngithub.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=\ngithub.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=\ngithub.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=\ngithub.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=\ngithub.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=\ngithub.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=\ngithub.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=\ngithub.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=\ngithub.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=\ngithub.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=\ngithub.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=\ngithub.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=\ngithub.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=\ngithub.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=\ngithub.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=\ngithub.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=\ngithub.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=\ngithub.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=\ngithub.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 h1:ZuhckGJ10ulaKkdvJtiAqsLTiPrLaXSdnVgXJKJkTxE=\ngithub.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3/go.mod h1:9/Rh6yILuLysoQnZ2oNooD2g7aBnvM7r/fNVxRNWfBc=\ngithub.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=\ngithub.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=\ngithub.com/shirou/gopsutil v3.21.6+incompatible h1:mmZtAlWSd8U2HeRTjswbnDLPxqsEoK01NK+GZ1P+nEM=\ngithub.com/shirou/gopsutil v3.21.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=\ngithub.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=\ngithub.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=\ngithub.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=\ngithub.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=\ngithub.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=\ngithub.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=\ngithub.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=\ngithub.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=\ngithub.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=\ngithub.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=\ngithub.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=\ngithub.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=\ngithub.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=\ngithub.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=\ngithub.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=\ngithub.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=\ngithub.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=\ngithub.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=\ngithub.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=\ngithub.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=\ngithub.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=\ngithub.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=\ngithub.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=\ngithub.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=\ngithub.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=\ngithub.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=\ngithub.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=\ngithub.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=\ngithub.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=\ngithub.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=\ngithub.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=\ngithub.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=\ngithub.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=\ngithub.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=\ngithub.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=\ngithub.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=\ngithub.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=\ngithub.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=\ngithub.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=\ngithub.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=\ngithub.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=\ngithub.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=\ngithub.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=\ngithub.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=\ngithub.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=\ngithub.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=\ngithub.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E=\ngithub.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0=\ngithub.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=\ngithub.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=\ngithub.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=\ngithub.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=\ngithub.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=\ngithub.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=\ngithub.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=\ngithub.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=\ngithub.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=\ngithub.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=\ngithub.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=\ngithub.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=\ngithub.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=\ngithub.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=\ngithub.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=\ngithub.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=\ngithub.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=\ngithub.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=\ngithub.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=\ngithub.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=\ngithub.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=\ngithub.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=\ngithub.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=\ngithub.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=\ngithub.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=\ngithub.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=\ngithub.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=\ngithub.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=\ngithub.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=\ngithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=\ngithub.com/xlab/treeprint v1.0.0 h1:J0TkWtiuYgtdlrkkrDLISYBQ92M+X5m4LrIIMKrbDTs=\ngithub.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg=\ngithub.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=\ngithub.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g=\ngithub.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM=\ngithub.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=\ngithub.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=\ngithub.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=\ngithub.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=\ngithub.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=\ngithub.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=\ngithub.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=\ngithub.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=\ngithub.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=\ngo.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=\ngo.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=\ngo.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=\ngo.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=\ngo.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=\ngo.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=\ngo.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=\ngo.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=\ngo.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=\ngo.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=\ngo.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=\ngo.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=\ngo.hein.dev/go-version v0.1.0/go.mod h1:WOEm7DWMroRe5GdUgHMvx+Pji5WWIpMuXmK/3foylXs=\ngo.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=\ngo.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=\ngo.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=\ngo.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=\ngo.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=\ngo.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=\ngo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=\ngo.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=\ngo.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=\ngo.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=\ngo.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=\ngo.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=\ngo.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=\ngo.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=\ngo.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=\ngo.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=\ngo.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=\ngo.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=\ngo.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=\ngo.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=\ngo.uber.org/atomic v1.8.0 h1:CUhrE4N1rqSE6FM9ecihEjRkLQu8cDfgDyoOs83mEY4=\ngo.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=\ngo.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=\ngo.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=\ngo.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=\ngo.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=\ngo.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=\ngo.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=\ngo.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=\ngo.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=\ngo.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=\ngo.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=\ngo.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=\ngo.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=\ngo.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=\ngo.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=\ngolang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=\ngolang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=\ngolang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=\ngolang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=\ngolang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=\ngolang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=\ngolang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=\ngolang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=\ngolang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=\ngolang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=\ngolang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=\ngolang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=\ngolang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=\ngolang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=\ngolang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=\ngolang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=\ngolang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=\ngolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=\ngolang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=\ngolang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=\ngolang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=\ngolang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=\ngolang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 h1:3B43BWw0xEBsLZ/NO1VALz6fppU3481pik+2Ksv45z8=\ngolang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190812172437-4e8604ab3aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220704084225-05e143d24a9e h1:CsOuNlbOuf0mzxJIefr6Q4uAUetRUwZE4qt7VfzP+xo=\ngolang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=\ngolang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190812233024-afc3694995b6/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=\ngolang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=\ngolang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=\ngolang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=\ngolang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=\ngolang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=\ngolang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=\ngolang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=\ngolang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY=\ngolang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618=\ngolang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=\ngomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=\ngomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=\ngomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=\ngonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=\ngonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=\ngonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=\ngoogle.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=\ngoogle.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=\ngoogle.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=\ngoogle.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=\ngoogle.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=\ngoogle.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=\ngoogle.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=\ngoogle.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=\ngoogle.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=\ngoogle.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=\ngoogle.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=\ngoogle.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=\ngoogle.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=\ngoogle.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=\ngoogle.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=\ngoogle.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=\ngoogle.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=\ngoogle.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=\ngoogle.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=\ngoogle.golang.org/genproto v0.0.0-20210701133433-6b8dcf568a95 h1:xyRjacsGcaSoZ2fTcaLCSzh2JEceLLOT4X8k32Q0xAQ=\ngoogle.golang.org/genproto v0.0.0-20210701133433-6b8dcf568a95/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U=\ngoogle.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=\ngoogle.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=\ngoogle.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=\ngoogle.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=\ngoogle.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=\ngoogle.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=\ngoogle.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=\ngoogle.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=\ngoogle.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=\ngoogle.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=\ngoogle.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=\ngoogle.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=\ngoogle.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=\ngoogle.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=\ngoogle.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=\ngoogle.golang.org/grpc v1.39.0 h1:Klz8I9kdtkIN6EpHHUOMLCYhTn/2WAe5a0s1hcBkdTI=\ngoogle.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=\ngoogle.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=\ngoogle.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=\ngoogle.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=\ngoogle.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=\ngopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=\ngopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=\ngopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=\ngopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=\ngopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=\ngopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=\ngopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=\ngopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=\ngopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=\ngopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=\ngopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=\ngopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=\ngopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=\ngopkg.in/segmentio/analytics-go.v3 v3.1.0 h1:UzxH1uaGZRpMKDhJyBz0pexz6yUoBU3x8bJsRk/HV6U=\ngopkg.in/segmentio/analytics-go.v3 v3.1.0/go.mod h1:4QqqlTlSSpVlWA9/9nDcPw+FkM2yv1NQoYjUbL9/JAw=\ngopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=\ngopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=\ngopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=\ngopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=\ngopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\ngopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=\ngotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=\ngotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=\ngotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=\ngotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=\nhonnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=\nhonnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=\nistio.io/api v0.0.0-20211015181651-ddbde26ea264/go.mod h1:nsSFw1LIMmGL7r/+6fJI6FxeG/UGlLxRK8bkojIvBVs=\nistio.io/api v0.0.0-20220304045306-249321c725a9 h1:CYmLAMT4hKCQ4m3LgwlbP/VEHAZK2lfUCEYN1oe2K0E=\nistio.io/api v0.0.0-20220304045306-249321c725a9/go.mod h1:nsSFw1LIMmGL7r/+6fJI6FxeG/UGlLxRK8bkojIvBVs=\nistio.io/client-go v1.10.6 h1:DMcNQOnsx/D5P24U/8IcCNIfyaJgqmOwVW8mnbf2j8Q=\nistio.io/client-go v1.10.6/go.mod h1:VXhAHzCaFpMKDL3Kul+hCz+dX3FGjM3GuTXGoi7tTv4=\nistio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f h1:9710FpGLvIJ1GGEbpuTh1smVBv+r8cJfR3G82ouSxIQ=\nistio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f/go.mod h1:6BwTZRNbWS570wHX/uR1Wqk5e0157TofTAUMzT7N4+s=\nk8s.io/api v0.16.8/go.mod h1:a8EOdYHO8en+YHhPBLiW5q+3RfHTr7wxTqqp7emJ7PM=\nk8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=\nk8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8=\nk8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=\nk8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=\nk8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8=\nk8s.io/api v0.22.11 h1:1CGTTsEy2VCZK9uDXyEauDN7VRiMCyVOFeI0Bj3JLts=\nk8s.io/api v0.22.11/go.mod h1:l2IQ0P1TtTvSeWnjqTrFQ7sX5LR1H9a9Knb0N2ILJKE=\nk8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk=\nk8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4=\nk8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA=\nk8s.io/apimachinery v0.16.8/go.mod h1:Xk2vD2TRRpuWYLQNM6lT9R7DSFZUYG03SarNkbGrnKE=\nk8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=\nk8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=\nk8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=\nk8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=\nk8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=\nk8s.io/apimachinery v0.22.11 h1:wOAkOC3Vzzf2s2Bqmmk7fmh5jIl2xH8kxom6qAjQqO4=\nk8s.io/apimachinery v0.22.11/go.mod h1:ZvVLP5iLhwVFg2Yx9Gh5W0um0DUauExbRhe+2Z8I1EU=\nk8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=\nk8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=\nk8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=\nk8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI=\nk8s.io/client-go v0.16.8/go.mod h1:WmPuN0yJTKHXoklExKxzo3jSXmr3EnN+65uaTb5VuNs=\nk8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=\nk8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE=\nk8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=\nk8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=\nk8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U=\nk8s.io/client-go v0.22.11 h1:2lQhSQqqnNRVOHzK0i9rG4p+jYKjct7t8lXl++GhjrY=\nk8s.io/client-go v0.22.11/go.mod h1:zNje4oH4oU3DchJmlmU71VeOgPxiJ+r/Rh8VRq7Jq6E=\nk8s.io/code-generator v0.16.8/go.mod h1:wFdrXdVi/UC+xIfLi+4l9elsTT/uEF61IfcN2wOLULQ=\nk8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg=\nk8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o=\nk8s.io/code-generator v0.22.11/go.mod h1:iOZwYADSgFPNGWfqHFfg1V0TNJnl1t0WyZluQp4baqU=\nk8s.io/component-base v0.16.8/go.mod h1:Q8UWOWShpP3MZZny4n/15gOncfaaVtc9SbCdkM5MhUE=\nk8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=\nk8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0=\nk8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=\nk8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=\nk8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M=\nk8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug=\nk8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28/go.mod h1:9a7E6pmKLfuq8ZL31k2PDpgvSdyZfUOH9czlEmpblFk=\nk8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=\nk8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=\nk8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=\nk8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=\nk8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=\nk8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=\nk8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=\nk8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=\nk8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=\nk8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=\nk8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=\nk8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=\nk8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=\nk8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=\nk8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=\nk8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=\nk8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=\nk8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=\nk8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=\nk8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=\nk8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=\nk8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=\nk8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513 h1:pbudjNtv90nOgR0/DUhPwKHnQ55Khz8+sNhJBIK7A5M=\nk8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=\nk8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=\nk8s.io/metrics v0.22.11 h1:XjuWs2rh0/WrT0Zi+ZS9k3utawKCHRtMFK8PjLKixSA=\nk8s.io/metrics v0.22.11/go.mod h1:zTDGPesMSDXU5HznhD3r/jrZn2L/RGrc95vw3vUtB6w=\nk8s.io/sample-controller v0.16.8/go.mod h1:aXlORS1ekU77qhGybB5t3JORDurzDpWgvMYxmCsiuos=\nk8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=\nk8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=\nk8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=\nk8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=\nk8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=\nk8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=\nmodernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=\nmodernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=\nmodernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=\nmodernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=\nmodernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=\nrsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=\nrsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=\nrsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=\nsigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=\nsigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=\nsigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=\nsigs.k8s.io/aws-iam-authenticator v0.5.3 h1:EyqQ/uxzbe2mDETZZmuMnv0xHITnyLhZfPlGb6Mma20=\nsigs.k8s.io/aws-iam-authenticator v0.5.3/go.mod h1:DIq7gy0lvnyaG88AgFyJzUVeix+ia5msHEp4RL0102I=\nsigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=\nsigs.k8s.io/controller-runtime v0.10.3 h1:s5Ttmw/B4AuIbwrXD3sfBkXwnPMMWrqpVj4WRt1dano=\nsigs.k8s.io/controller-runtime v0.10.3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=\nsigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ=\nsigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=\nsigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=\nsigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=\nsigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=\nsigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=\nsigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=\nsigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=\nsigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=\nsigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=\n"
  },
  {
    "path": "images/activator/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM golang:1.17.3 as builder\n\nWORKDIR /workspace\nCOPY go.mod go.mod\nCOPY go.sum go.sum\nRUN go mod download\n\nCOPY pkg pkg\nCOPY cmd/activator cmd/activator\nWORKDIR /workspace/cmd/activator\n\nRUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /workspace/bin/activator main.go\n\nFROM gcr.io/distroless/static:nonroot\nWORKDIR /\nCOPY --from=builder /workspace/bin/activator .\nUSER 65532:65532\n\nENTRYPOINT [\"/activator\"]\n"
  },
  {
    "path": "images/async-gateway/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nARG TARGETARCH, TARGETOS\n\nFROM golang:1.17.3 as builder\n\nCOPY go.mod go.sum /workspace/\nWORKDIR /workspace\nRUN go mod download\n\nCOPY pkg/consts pkg/consts\nCOPY pkg/lib pkg/lib\nCOPY pkg/async-gateway pkg/async-gateway\nCOPY pkg/types pkg/types\nCOPY cmd/async-gateway cmd/async-gateway\n\nRUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -installsuffix cgo -o async-gateway ./cmd/async-gateway\n\nFROM alpine:3.15\n\nRUN apk update && apk add ca-certificates\n\nCOPY --from=builder /workspace/async-gateway /root/\nRUN chmod +x /root/async-gateway\n\nENTRYPOINT [\"/root/async-gateway\"]\n"
  },
  {
    "path": "images/autoscaler/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM golang:1.17.3 as builder\n\nWORKDIR /workspace\nCOPY go.mod go.mod\nCOPY go.sum go.sum\nRUN go mod download\n\nCOPY pkg pkg\nCOPY cmd/autoscaler cmd/autoscaler\nWORKDIR /workspace/cmd/autoscaler\n\nRUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /workspace/bin/autoscaler main.go\n\nFROM gcr.io/distroless/static:nonroot\nWORKDIR /\nCOPY --from=builder /workspace/bin/autoscaler .\nUSER 65532:65532\n\nENTRYPOINT [\"/autoscaler\"]\n"
  },
  {
    "path": "images/cluster-autoscaler/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nARG TARGETARCH, TARGETOS\n\nFROM golang:1.17.5 AS builder\nRUN git clone -b cluster-autoscaler-1.22.2-cortex --depth 1 https://github.com/cortexlabs/autoscaler /k8s.io/autoscaler\nWORKDIR /k8s.io/autoscaler/cluster-autoscaler\nRUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build --installsuffix cgo -o cluster-autoscaler k8s.io/autoscaler/cluster-autoscaler \\\n    && cp cluster-autoscaler /usr/local/bin\n\nFROM alpine:3.8\nRUN apk add -U --no-cache ca-certificates && rm -rf /var/cache/apk/*\nCOPY --from=builder /usr/local/bin/cluster-autoscaler .\n"
  },
  {
    "path": "images/controller-manager/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM golang:1.17.3 as builder\n\nWORKDIR /workspace\nCOPY go.mod go.mod\nCOPY go.sum go.sum\nRUN go mod download\n\nCOPY pkg/config pkg/config\nCOPY pkg/consts pkg/consts\nCOPY pkg/crds pkg/crds\nCOPY pkg/lib pkg/lib\nCOPY pkg/types pkg/types\nCOPY pkg/workloads pkg/workloads\n\nWORKDIR /workspace/pkg/crds\n\nRUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o /workspace/bin/manager main.go\n\nFROM gcr.io/distroless/static:nonroot\nWORKDIR /\nCOPY --from=builder /workspace/bin/manager .\nUSER 65532:65532\n\nENTRYPOINT [\"/manager\"]\n"
  },
  {
    "path": "images/dequeuer/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nARG TARGETARCH, TARGETOS\n\nFROM golang:1.17.3 as builder\n\nCOPY go.mod go.sum /workspace/\nWORKDIR /workspace\nRUN go mod download\n\nCOPY pkg/config pkg/config\nCOPY pkg/consts pkg/consts\nCOPY pkg/lib pkg/lib\nCOPY pkg/dequeuer pkg/dequeuer\nCOPY pkg/probe pkg/probe\nCOPY pkg/types pkg/types\nCOPY pkg/crds pkg/crds\nCOPY pkg/workloads pkg/workloads\nCOPY cmd/dequeuer cmd/dequeuer\n\nRUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -o dequeuer ./cmd/dequeuer\n\nFROM gcr.io/distroless/static:nonroot\nWORKDIR /\nCOPY --from=builder /workspace/dequeuer .\nUSER nonroot:nonroot\n\nENTRYPOINT [\"/dequeuer\"]\n"
  },
  {
    "path": "images/enqueuer/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nARG TARGETARCH, TARGETOS\n\nFROM golang:1.17.3 as builder\n\nCOPY go.mod go.sum /workspace/\nWORKDIR /workspace\nRUN go mod download\n\nCOPY pkg/config pkg/config\nCOPY pkg/consts pkg/consts\nCOPY pkg/lib pkg/lib\nCOPY pkg/enqueuer pkg/enqueuer\nCOPY pkg/types pkg/types\nCOPY pkg/crds pkg/crds\nCOPY pkg/workloads pkg/workloads\nCOPY cmd/enqueuer cmd/enqueuer\n\nRUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build -o enqueuer ./cmd/enqueuer\n\nFROM gcr.io/distroless/static:nonroot\nWORKDIR /\nCOPY --from=builder /workspace/enqueuer .\nUSER nonroot:nonroot\n\nENTRYPOINT [\"/enqueuer\"]\n"
  },
  {
    "path": "images/event-exporter/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM ghcr.io/opsgenie/kubernetes-event-exporter:v0.11\n"
  },
  {
    "path": "images/fluent-bit/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM amazon/aws-for-fluent-bit:2.23.3\n"
  },
  {
    "path": "images/grafana/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM grafana/grafana:8.0.4\n"
  },
  {
    "path": "images/istio-pilot/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM docker.io/istio/pilot:1.11.8\n"
  },
  {
    "path": "images/istio-proxy/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM docker.io/istio/proxyv2:1.11.8\n"
  },
  {
    "path": "images/kube-rbac-proxy/Dockerfile",
    "content": "# Copyright 2021 Kube RBAC Proxy Authors rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\nARG TARGETARCH, TARGETOS\n\nFROM golang:1.18 AS builder\nRUN git clone -b v0.13.0 --depth 1 https://github.com/brancz/kube-rbac-proxy /go/src/github.com/brancz/kube-rbac-proxy\nWORKDIR /go/src/github.com/brancz/kube-rbac-proxy\nRUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build --installsuffix cgo -o kube-rbac-proxy github.com/brancz/kube-rbac-proxy \\\n    && cp kube-rbac-proxy /usr/local/bin\n\nFROM alpine:3.8\nRUN apk add -U --no-cache ca-certificates && rm -rf /var/cache/apk/*\nCOPY --from=builder /usr/local/bin/kube-rbac-proxy .\nENTRYPOINT [\"./kube-rbac-proxy\"]\nEXPOSE 8080\n"
  },
  {
    "path": "images/kubexit/Dockerfile",
    "content": "# Copyright 2020 Karl Isenberg (https://github.com/karlkfi/kubexit)\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\nARG TARGETARCH, TARGETOS\n\nFROM golang:1.14 AS builder\n\nRUN mkdir /tmp/kubexit\nRUN git clone -b v0.1.0-cortex --depth 1  https://github.com/cortexlabs/kubexit.git /tmp/kubexit\n\nWORKDIR /tmp/kubexit\nRUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o kubexit ./cmd/kubexit\n\nFROM alpine:3.11\nRUN apk --no-cache add ca-certificates tzdata\nCOPY --from=builder /tmp/kubexit/kubexit /bin/\nENTRYPOINT [\"kubexit\"]\n"
  },
  {
    "path": "images/manager/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM python:3.7-alpine3.16\n\nWORKDIR /root\n\nENV PATH /root/.local/bin:$PATH\nENV AWS_RETRY_MODE standard\nENV AWS_MAX_ATTEMPTS 10\n\nCOPY manager/requirements.txt /root/requirements.txt\n\nRUN pip install --upgrade pip && \\\n    pip install awscli --upgrade --user && \\\n    pip install -r /root/requirements.txt && \\\n    rm -rf /root/.cache/pip*\n\nRUN apk add --no-cache bash curl gettext jq openssl\n\nRUN curl --location \"https://github.com/weaveworks/eksctl/releases/download/v0.107.0/eksctl_$(uname -s)_amd64.tar.gz\" | tar xz -C /tmp && \\\n    mv /tmp/eksctl /usr/local/bin\n\nRUN curl -o aws-iam-authenticator https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.2/2021-07-05/bin/linux/amd64/aws-iam-authenticator && \\\n    chmod +x ./aws-iam-authenticator && \\\n    mv ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator\n\nRUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl && \\\n    chmod +x ./kubectl && \\\n    mv ./kubectl /usr/local/bin/kubectl\n\nRUN curl -L \"https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv4.1.2/kustomize_v4.1.2_linux_amd64.tar.gz\" | tar xz -C /tmp && \\\n    mv /tmp/kustomize /usr/local/bin\n\nENV ISTIO_VERSION 1.11.8\nRUN curl -L https://istio.io/downloadIstio | sh -\n\nCOPY manager /root\nCOPY pkg/crds/config /root/config\n\nENTRYPOINT [\"/bin/bash\"]\n"
  },
  {
    "path": "images/metrics-server/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM k8s.gcr.io/metrics-server/metrics-server:v0.6.1\n"
  },
  {
    "path": "images/neuron-device-plugin/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM public.ecr.aws/neuron/neuron-device-plugin:1.8.2.0\n"
  },
  {
    "path": "images/neuron-scheduler/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM public.ecr.aws/neuron/neuron-scheduler:1.8.2.0\n"
  },
  {
    "path": "images/nvidia-device-plugin/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM nvidia/k8s-device-plugin:v0.11.0\n"
  },
  {
    "path": "images/operator/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM golang:1.17.3 as builder\n\nRUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl && \\\n    mv ./kubectl /tmp/kubectl\n\nCOPY go.mod go.sum /workspace/\nWORKDIR /workspace\nRUN go mod download\n\nCOPY pkg/config pkg/config\nCOPY pkg/consts pkg/consts\nCOPY pkg/lib pkg/lib\nCOPY pkg/operator pkg/operator\nCOPY pkg/types pkg/types\nCOPY pkg/crds pkg/crds\nCOPY pkg/workloads pkg/workloads\nCOPY cmd/operator cmd/operator\n\nRUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -installsuffix cgo -o operator ./cmd/operator\n\n\nFROM alpine:3.15\n\nCOPY --from=builder /tmp/kubectl /usr/local/bin/kubectl\nRUN chmod +x /usr/local/bin/kubectl\n\nRUN apk --no-cache add ca-certificates bash\n\nCOPY --from=builder /workspace/operator /root/\nRUN chmod +x /root/operator\n\nEXPOSE 8888\nENTRYPOINT [\"/root/operator\"]\n"
  },
  {
    "path": "images/prometheus/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM quay.io/prometheus/prometheus:v2.28.0\n"
  },
  {
    "path": "images/prometheus-config-reloader/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM quay.io/prometheus-operator/prometheus-config-reloader:v0.48.1\n"
  },
  {
    "path": "images/prometheus-dcgm-exporter/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM nvcr.io/nvidia/k8s/dcgm-exporter:2.1.4-2.3.1-ubuntu18.04\n\n# uncomment DCGM_FI_DEV_GPU_UTIL metric\nRUN sed -i '/DCGM_FI_DEV_GPU_UTIL/s/^# //g' /etc/dcgm-exporter/default-counters.csv\n"
  },
  {
    "path": "images/prometheus-kube-state-metrics/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.1.0\n"
  },
  {
    "path": "images/prometheus-node-exporter/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM quay.io/prometheus/node-exporter:v1.3.1\n"
  },
  {
    "path": "images/prometheus-operator/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM quay.io/prometheus-operator/prometheus-operator:v0.48.1\n"
  },
  {
    "path": "images/prometheus-statsd-exporter/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nFROM prom/statsd-exporter:v0.22.4\n"
  },
  {
    "path": "images/proxy/Dockerfile",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nARG TARGETARCH, TARGETOS\n\nFROM golang:1.17.3 as builder\n\nWORKDIR /workspace\nCOPY go.mod go.mod\nCOPY go.sum go.sum\nRUN go mod download\n\nCOPY pkg pkg\nCOPY cmd/proxy cmd/proxy\nWORKDIR /workspace/cmd/proxy\n\nRUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH}  GO111MODULE=on go build -a -o /workspace/bin/proxy main.go\n\nFROM gcr.io/distroless/static:nonroot\nWORKDIR /\nCOPY --from=builder /workspace/bin/proxy .\nUSER 65532:65532\n\nENTRYPOINT [\"/proxy\"]\n"
  },
  {
    "path": "manager/check_cortex_version.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -e\n\nCORTEX_VERSION=master\n\nif [ \"$CORTEX_VERSION\" != \"$CORTEX_CLI_VERSION\" ]; then\n  echo \"error: your CLI version ($CORTEX_CLI_VERSION) doesn't match your Cortex manager image version ($CORTEX_VERSION); please update your CLI (pip install cortex==$CORTEX_VERSION) to match the version of your Cortex manager image\"\n  exit 1\nfi\n"
  },
  {
    "path": "manager/cluster_config_env.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport yaml\nimport json\nfrom copy import deepcopy\n\n\ndef export(base_key, value):\n    if base_key.lower() == \"cortex_tags\":\n        exportTags(value, \"CORTEX_TAGS\")\n        exportTags(\n            value, \"CORTEX_OPERATOR_LOAD_BALANCER_TAGS\", {\"cortex.dev/load-balancer\": \"operator\"}\n        )\n        exportTags(value, \"CORTEX_API_LOAD_BALANCER_TAGS\", {\"cortex.dev/load-balancer\": \"api\"})\n        return\n\n    if value is None:\n        return\n    elif type(value) is list:\n        print(f'export {base_key.upper()}=\"{yaml.dump(value, default_flow_style=True).strip()}\"')\n    elif type(value) is dict:\n        for key, child in value.items():\n            export(base_key + \"_\" + key, child)\n    else:\n        print(f'export {base_key.upper()}=\"{value}\"')\n        if base_key.lower().startswith(\"cortex_image_\"):\n            hub = value.rsplit(\"/\", 1)[0]\n            suffix = value.rsplit(\"/\", 1)[1]\n            if \":\" in suffix:\n                image = suffix.split(\":\")[0]\n                tag = suffix.split(\":\")[1]\n            else:\n                image = suffix\n                tag = \"latest\"\n\n            print(f'export {base_key.upper()}_HUB=\"{hub}\"')\n            print(f'export {base_key.upper()}_IMAGE=\"{image}\"')\n            print(f'export {base_key.upper()}_TAG=\"{tag}\"')\n\n\ndef exportTags(tags, env_var_name, tag_overrides={}):\n    tags = deepcopy(tags)\n    for k, v in tag_overrides.items():\n        tags[k] = v\n    inlined_tags = \",\".join([f\"{k}={v}\" for k, v in tags.items()])\n    print(f\"export {env_var_name}='{inlined_tags}'\")\n    print(f\"export {env_var_name}_JSON='{json.dumps(tags)}'\")\n\n\nfor config_path in sys.argv[1:]:\n    with open(config_path, \"r\") as f:\n        config = yaml.safe_load(f)\n\n    export(\"CORTEX\", config)\n"
  },
  {
    "path": "manager/debug.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset +e\n\nCORTEX_VERSION_MINOR=master\n\ndebug_out_path=\"$1\"\nmkdir -p \"$(dirname \"$debug_out_path\")\"\n\nif ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION >/dev/null 2>&1; then\n  echo \"error: there is no cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION; please update your configuration to point to an existing cortex cluster or create a cortex cluster with \\`cortex cluster up\\`\"\n  exit 1\nfi\n\neksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION  --verbose=0 | (grep -v \"saved kubeconfig as\" || true)\nout=$(kubectl get pods 2>&1 || true); if [[ \"$out\" == *\"must be logged in to the server\"* ]]; then echo \"error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortexlabs.com/v/${CORTEX_VERSION_MINOR}/\"; exit 1; fi\n\necho -n \"gathering cluster data\"\n\nmkdir -p /cortex-debug/k8s\nfor resource in pods pods.metrics nodes nodes.metrics daemonsets deployments hpa services virtualservices gateways ingresses configmaps jobs replicasets events; do\n  kubectl describe $resource --all-namespaces > \"/cortex-debug/k8s/${resource}\" 2>&1\n  kubectl get $resource --all-namespaces > \"/cortex-debug/k8s/${resource}-list\" 2>&1\n  echo -n \".\"\ndone\n\nmkdir -p /cortex-debug/logs\nkubectl get pods --all-namespaces -o json | jq '.items[] | . as $parent | $parent.spec.containers[]? | \"kubectl logs -n \\($parent.metadata.namespace) \\($parent.metadata.name) \\(.name) --timestamps --tail=10000 > /cortex-debug/logs/\\($parent.metadata.namespace).\\($parent.metadata.name).\\(.name) 2>&1; echo -n .\"' | xargs -n 1 bash -c\nkubectl get pods --all-namespaces -o json | jq '.items[] | . as $parent | $parent.spec.containers[]? | \"kubectl logs -n \\($parent.metadata.namespace) \\($parent.metadata.name) \\(.name) --previous --timestamps --tail=10000 > /cortex-debug/logs/\\($parent.metadata.namespace).\\($parent.metadata.name).\\(.name).previous 2>&1; if [ $? -ne 0 ]; then rm /cortex-debug/logs/\\($parent.metadata.namespace).\\($parent.metadata.name).\\(.name).previous; fi; echo -n .\"' | xargs -n 1 bash -c\necho -n \".\"\nkubectl get pods --all-namespaces -o json | jq '.items[] | . as $parent | $parent.spec.initContainers[]? | \"kubectl logs -n \\($parent.metadata.namespace) \\($parent.metadata.name) \\(.name) --timestamps --tail=10000 > /cortex-debug/logs/\\($parent.metadata.namespace).\\($parent.metadata.name).init.\\(.name) 2>&1; echo -n .\"' | xargs -n 1 bash -c\nkubectl get pods --all-namespaces -o json | jq '.items[] | . as $parent | $parent.spec.initContainers[]? | \"kubectl logs -n \\($parent.metadata.namespace) \\($parent.metadata.name) \\(.name) --previous --timestamps --tail=10000 > /cortex-debug/logs/\\($parent.metadata.namespace).\\($parent.metadata.name).init.\\(.name).previous 2>&1; if [ $? -ne 0 ]; then rm /cortex-debug/logs/\\($parent.metadata.namespace).\\($parent.metadata.name).init.\\(.name).previous; fi; echo -n .\"' | xargs -n 1 bash -c\necho -n \".\"\n\nkubectl top pods --all-namespaces --containers=true > \"/cortex-debug/k8s/top_pods\" 2>&1\necho -n \".\"\nkubectl top nodes > \"/cortex-debug/k8s/top_nodes\" 2>&1\necho -n \".\"\n\nmkdir -p /cortex-debug/aws/amis\n\naws autoscaling describe-auto-scaling-groups --region=$CORTEX_REGION --output json > \"/cortex-debug/aws/asgs\" 2>&1\necho -n \".\"\naws autoscaling describe-scaling-activities --max-items 1000 --region=$CORTEX_REGION --output json > \"/cortex-debug/aws/asg-activities\" 2>&1\necho -n \".\"\n\naws ec2 describe-instances --filters Name=tag:cortex.dev/cluster-name,Values=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --output json > \"/cortex-debug/aws/instances\" 2>&1\necho -n \".\"\naws ec2 describe-instance-status --include-all-instances --region=$CORTEX_REGION --output json > \"/cortex-debug/aws/instance-statuses\" 2>&1\necho -n \".\"\naws ec2 describe-instances --filters Name=tag:cortex.dev/cluster-name,Values=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --output json | jq \"[.Reservations[].Instances[].ImageId] | unique | .[] | \\\"aws ec2 describe-images --image-ids \\(.) --region=$CORTEX_REGION --output json > /cortex-debug/aws/amis/\\(.) 2>&1\\\"\" | xargs -n 1 bash -c\necho -n \".\"\npython get_operator_load_balancer_state.py > \"/cortex-debug/aws/operator_load_balancer_state\" 2>&1\necho -n \".\"\npython get_api_load_balancer_state.py > \"/cortex-debug/aws/api_load_balancer_state\" 2>&1\necho -n \".\"\npython get_operator_target_group_status.py > \"/cortex-debug/aws/operator_load_balancer_target_group_status\" 2>&1\necho -n \".\"\n\nmkdir -p /cortex-debug/misc\noperator_endpoint=$(kubectl -n=istio-system get service ingressgateway-operator -o json 2>/dev/null | tr -d '[:space:]' | sed 's/.*{\\\"hostname\\\":\\\"\\(.*\\)\\\".*/\\1/')\necho \"$operator_endpoint\" > /cortex-debug/misc/operator_endpoint\nif [ \"$operator_endpoint\" == \"\" ]; then\n  echo \"unable to get operator endpoint\" > /cortex-debug/misc/operator_curl\nelse\n  curl -sv --max-time 5 \"${operator_endpoint}/verifycortex\" > /cortex-debug/misc/operator_curl 2>&1\nfi\necho -n \".\"\n\n(cd / && tar -czf cortex-debug.tgz cortex-debug)\nmv /cortex-debug.tgz $debug_out_path\n\necho \" ✓\"\n"
  },
  {
    "path": "manager/generate_eks.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport json\nimport click\n\nfrom collections import namedtuple\nimport re\nimport yaml\n\nK8S_VERSION = \"1.22\"\n\nParsedInstanceType = namedtuple(\n    \"ParsedInstanceType\", [\"family\", \"generation\", \"capabilities\", \"size\"]\n)\n\n\ndef parse_instance_type(instance_type: str) -> ParsedInstanceType:\n    parts = instance_type.split(\".\")\n    if len(parts) != 2:\n        raise ValueError(f\"unexpected invalid instance type: {instance_type}\")\n\n    prefix = parts[0]\n    size = parts[1]\n\n    family = re.search(\"[a-z]*\", prefix.lower()).group()\n    generation = re.sub(\"\\D\", \"\", prefix.lower())\n    capabilities = prefix[len(family) + len(generation) :]\n\n    return ParsedInstanceType(family, generation, capabilities, size)\n\n\n# kubelet config schema:\n# https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go\ndef default_nodegroup(cluster_config):\n    partition = \"aws\"\n    if \"us-gov\" in cluster_config[\"region\"]:\n        partition = \"aws-us-gov\"\n    return {\n        \"iam\": {\n            \"withAddonPolicies\": {\"autoScaler\": True},\n            \"attachPolicyARNs\": [\n                f\"arn:{partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy\",\n                f\"arn:{partition}:iam::aws:policy/AmazonEKS_CNI_Policy\",\n                f\"arn:{partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\",\n                f\"arn:{partition}:iam::aws:policy/ElasticLoadBalancingFullAccess\",\n                cluster_config[\"cortex_policy_arn\"],\n            ]\n            + cluster_config.get(\"iam_policy_arns\", []),\n        },\n        \"privateNetworking\": cluster_config.get(\"subnet_visibility\", \"public\") != \"public\",\n        \"kubeletExtraConfig\": {\n            \"kubeReserved\": {\"cpu\": \"150m\", \"memory\": \"300Mi\", \"ephemeral-storage\": \"1Gi\"},\n            \"kubeReservedCgroup\": \"/kube-reserved\",\n            \"systemReserved\": {\"cpu\": \"150m\", \"memory\": \"300Mi\", \"ephemeral-storage\": \"1Gi\"},\n            \"evictionHard\": {\"memory.available\": \"200Mi\", \"nodefs.available\": \"5%\"},\n            \"registryPullQPS\": 10,\n        },\n        \"preBootstrapCommands\": [\n            \"sudo yum install -y ipvsadm\",\n            \"sudo modprobe ip_vs\",  # IP virtual server\n            \"sudo modprobe ip_vs_rr\",  # round robing load balancer\n            \"sudo modprobe ip_vs_lc\",  # least connected load balancer\n            \"sudo modprobe ip_vs_wrr\",  # weighted round robin load balancer\n            \"sudo modprobe ip_vs_sh\",  # source-hashing load balancer\n            \"sudo modprobe nf_conntrack_ipv4\",\n        ],\n        \"overrideBootstrapCommand\": \"\\n\".join(\n            [\n                \"#!/bin/bash\",\n                \"source /var/lib/cloud/scripts/eksctl/bootstrap.helper.sh\",\n                f\"/etc/eks/bootstrap.sh {cluster_config['cluster_name']} --container-runtime dockerd --kubelet-extra-args \\\"--node-labels=${{NODE_LABELS}} --register-with-taints=${{NODE_TAINTS}}\\\"\",\n            ]\n        ),\n    }\n\n\ndef merge_override(a, b):\n    \"merges b into a\"\n    for key in b:\n        if key in a:\n            if isinstance(a[key], dict) and isinstance(b[key], dict):\n                merge_override(a[key], b[key])\n            elif isinstance(a[key], list) and isinstance(b[key], list):\n                a[key] += b[key]\n            else:\n                a[key] = b[key]\n        else:\n            a[key] = b[key]\n    return a\n\n\ndef apply_worker_settings(nodegroup, config):\n    worker_settings = {\n        \"name\": \"cx-wd-\" + config[\"name\"],\n        \"asgSuspendProcesses\": [\"AZRebalance\"],\n        \"labels\": {\"workload\": \"true\"},\n        \"taints\": [\n            {\n                \"key\": \"workload\",\n                \"value\": \"true\",\n                \"effect\": \"NoSchedule\",\n            },\n        ],\n        \"tags\": {\n            \"k8s.io/cluster-autoscaler/enabled\": \"true\",\n            \"k8s.io/cluster-autoscaler/node-template/label/workload\": \"true\",\n        },\n    }\n\n    return merge_override(nodegroup, worker_settings)\n\n\ndef apply_clusterconfig(nodegroup, config):\n    clusterconfig_settings = {\n        \"instanceType\": config[\"instance_type\"],\n        \"volumeSize\": config[\"instance_volume_size\"],\n        \"minSize\": config[\"min_instances\"],\n        \"maxSize\": config[\"max_instances\"],\n        \"volumeType\": config[\"instance_volume_type\"],\n        \"desiredCapacity\": 1 if config[\"min_instances\"] == 0 else config[\"min_instances\"],\n    }\n    # add iops to settings if volume_type is io1/gp3\n    if config[\"instance_volume_type\"] in [\"io1\", \"gp3\"]:\n        clusterconfig_settings[\"volumeIOPS\"] = config[\"instance_volume_iops\"]\n    if config[\"instance_volume_type\"] == \"gp3\":\n        clusterconfig_settings[\"volumeThroughput\"] = config[\"instance_volume_throughput\"]\n\n    return merge_override(nodegroup, clusterconfig_settings)\n\n\ndef apply_spot_settings(nodegroup, config):\n    spot_settings = {\n        \"name\": \"cx-ws-\" + config[\"name\"],\n        \"instanceType\": \"mixed\",\n        \"instancesDistribution\": {\n            \"instanceTypes\": config[\"spot_config\"][\"instance_distribution\"],\n            \"onDemandBaseCapacity\": config[\"spot_config\"][\"on_demand_base_capacity\"],\n            \"onDemandPercentageAboveBaseCapacity\": config[\"spot_config\"][\n                \"on_demand_percentage_above_base_capacity\"\n            ],\n            \"maxPrice\": config[\"spot_config\"][\"max_price\"],\n            \"spotInstancePools\": config[\"spot_config\"][\"instance_pools\"],\n        },\n        \"labels\": {\"lifecycle\": \"Ec2Spot\"},\n    }\n\n    return merge_override(nodegroup, spot_settings)\n\n\ndef apply_gpu_settings(nodegroup):\n    gpu_settings = {\n        \"tags\": {\n            \"k8s.io/cluster-autoscaler/node-template/label/nvidia.com/gpu\": \"true\",\n            \"k8s.io/cluster-autoscaler/node-template/taint/dedicated\": \"nvidia.com/gpu=true\",\n            \"k8s.io/cluster-autoscaler/node-template/label/k8s.amazonaws.com/accelerator\": \"true\",  # accepted values are GPU type such as nvidia-tesla-k80 but using \"true\" as a placeholder for now because the value doesn't matter for AWS cluster autoscaler\n        },\n        \"labels\": {\n            \"nvidia.com/gpu\": \"true\",\n            \"k8s.amazonaws.com/accelerator\": \"true\",  # accepted values are GPU type such as nvidia-tesla-k80 but using \"true\" as a placeholder for now because the value doesn't matter for AWS cluster autoscaler\n        },\n        \"taints\": [\n            {\n                \"key\": \"nvidia.com/gpu\",\n                \"value\": \"true\",\n                \"effect\": \"NoSchedule\",\n            },\n        ],\n    }\n\n    return merge_override(nodegroup, gpu_settings)\n\n\ndef is_gpu(instance_type):\n    parsed_instance_type = parse_instance_type(instance_type)\n    return parsed_instance_type.family in [\"g\", \"p\"]\n\n\ndef apply_inf_settings(nodegroup, config):\n    instance_type = config[\"instance_type\"]\n\n    num_chips, num_hugepages = get_inf_resources(instance_type)\n    inf_settings = {\n        \"tags\": {\n            \"k8s.io/cluster-autoscaler/node-template/label/aws.amazon.com/neuron\": \"true\",\n            \"k8s.io/cluster-autoscaler/node-template/taint/dedicated\": \"aws.amazon.com/neuron=true\",\n            \"k8s.io/cluster-autoscaler/node-template/resources/aws.amazon.com/neuron\": str(\n                num_chips\n            ),\n            \"k8s.io/cluster-autoscaler/node-template/resources/hugepages-2Mi\": num_hugepages,\n        },\n        \"labels\": {\"aws.amazon.com/neuron\": \"true\"},\n        \"taints\": [\n            {\n                \"key\": \"aws.amazon.com/neuron\",\n                \"value\": \"true\",\n                \"effect\": \"NoSchedule\",\n            },\n        ],\n    }\n    return merge_override(nodegroup, inf_settings)\n\n\ndef is_inf(instance_type):\n    parsed_instance_type = parse_instance_type(instance_type)\n    return parsed_instance_type.family == \"inf\"\n\n\ndef get_inf_resources(instance_type):\n    num_chips = 0\n    if instance_type in [\"inf1.xlarge\", \"inf1.2xlarge\"]:\n        num_chips = 1\n    elif instance_type == \"inf1.6xlarge\":\n        num_chips = 4\n    elif instance_type == \"inf1.24xlarge\":\n        num_chips = 16\n\n    return num_chips, f\"{128 * num_chips}Mi\"\n\n\ndef is_arm64(instance_type: str):\n    parsed_instance_type = parse_instance_type(instance_type)\n    return parsed_instance_type.family == \"a\" or \"g\" in parsed_instance_type.capabilities\n\n\ndef get_all_worker_nodegroups(ami_map: dict, cluster_config: dict) -> list:\n    \"\"\"\n    Gets all node groups in EKS-dict format.\n    \"\"\"\n    worker_nodegroups = []\n    for ng in cluster_config[\"node_groups\"]:\n        worker_nodegroups.append(get_worker_nodegroup(ami_map, ng, cluster_config))\n    return worker_nodegroups\n\n\ndef get_worker_nodegroup(ami_map: dict, nodegroup_config: dict, cluster_config: dict) -> dict:\n    \"\"\"\n    Converts Cortex-dict nodegroup config to EKS-dict format.\n    \"\"\"\n    worker_nodegroup = default_nodegroup(cluster_config)\n    worker_nodegroup[\"ami\"] = get_ami(ami_map, nodegroup_config[\"instance_type\"])\n\n    apply_worker_settings(worker_nodegroup, nodegroup_config)\n    apply_clusterconfig(worker_nodegroup, nodegroup_config)\n\n    if nodegroup_config[\"spot\"]:\n        apply_spot_settings(worker_nodegroup, nodegroup_config)\n\n    if is_gpu(nodegroup_config[\"instance_type\"]):\n        apply_gpu_settings(worker_nodegroup)\n\n    if is_inf(nodegroup_config[\"instance_type\"]):\n        apply_inf_settings(worker_nodegroup, nodegroup_config)\n\n    return worker_nodegroup\n\n\ndef get_nodegroup_config_by_name(cluster_config: dict, ng_name: str) -> dict:\n    \"\"\"\n    Gets a nodegroup in Cortex-dict format from Cortex cluster config.\n    \"\"\"\n    for ng in cluster_config[\"node_groups\"]:\n        if ng[\"name\"] == ng_name:\n            return ng\n\n\ndef get_empty_eks_nodegroup(name: str) -> dict:\n    \"\"\"\n    Gets an empty nodegroup in EKS-dict format that only has the Cortex nodegroup name filled out.\n    \"\"\"\n    return {\"name\": name}\n\n\ndef get_ami(ami_map: dict, instance_type: str) -> str:\n    if is_gpu(instance_type) or is_inf(instance_type):\n        return ami_map[\"accelerated_amd64\"]\n    if is_arm64(instance_type):\n        return ami_map[\"cpu_arm64\"]\n    return ami_map[\"cpu_amd64\"]\n\n\n@click.command()\n@click.argument(\"cluster-config_file\", type=click.File(\"r\"))\n@click.argument(\"ami-json-file\", type=click.File(\"r\"))\n@click.option(\n    \"--add-cortex-node-groups\",\n    type=str,\n    help=\"specific cortex nodegroups to add to the generated eks file; use this for existing clusters\",\n)\n@click.option(\n    \"--remove-eks-node-groups\",\n    type=str,\n    help=\"specific eks nodegroup stacks to add to the generated eks file; use this for existing clusters\",\n)\ndef generate_eks(\n    cluster_config_file, ami_json_file, add_cortex_node_groups: str, remove_eks_node_groups: str\n):\n    cluster_config = yaml.safe_load(cluster_config_file)\n    region = cluster_config[\"region\"]\n    name = cluster_config[\"cluster_name\"]\n    prometheus_instance_type = cluster_config[\"prometheus_instance_type\"]\n    ami_map = json.load(ami_json_file)[K8S_VERSION][region]\n\n    eks = {\n        \"apiVersion\": \"eksctl.io/v1alpha5\",\n        \"kind\": \"ClusterConfig\",\n        \"metadata\": {\n            \"name\": name,\n            \"region\": region,\n            \"version\": K8S_VERSION,\n        },\n    }\n\n    if add_cortex_node_groups:\n        node_group_names = add_cortex_node_groups.split(\",\")\n        eks[\"nodeGroups\"] = []\n        for node_group_name in node_group_names:\n            nodegroup_config = get_nodegroup_config_by_name(cluster_config, node_group_name)\n            eks[\"nodeGroups\"].append(\n                get_worker_nodegroup(ami_map, nodegroup_config, cluster_config)\n            )\n        click.echo(yaml.dump(eks, Dumper=IgnoreAliases, default_flow_style=False, default_style=\"\"))\n        return\n\n    if remove_eks_node_groups:\n        stacks_names = remove_eks_node_groups.split(\",\")\n        eks[\"nodeGroups\"] = []\n        for stack_name in stacks_names:\n            eks[\"nodeGroups\"].append(get_empty_eks_nodegroup(stack_name))\n        click.echo(yaml.dump(eks, Dumper=IgnoreAliases, default_flow_style=False, default_style=\"\"))\n        return\n\n    operator_nodegroup = default_nodegroup(cluster_config)\n    operator_settings = {\n        \"ami\": get_ami(ami_map, \"t3.medium\"),\n        \"name\": \"cx-operator\",\n        \"instanceType\": \"t3.medium\",\n        \"minSize\": 2,\n        \"maxSize\": 25,\n        \"desiredCapacity\": 2,\n        \"volumeType\": \"gp3\",\n        \"volumeSize\": 20,\n        \"volumeIOPS\": 3000,\n        \"volumeThroughput\": 125,\n        \"labels\": {\"operator\": \"true\"},\n    }\n    operator_nodegroup = merge_override(operator_nodegroup, operator_settings)\n\n    prometheus_nodegroup = default_nodegroup(cluster_config)\n    prometheus_settings = {\n        \"ami\": get_ami(ami_map, prometheus_instance_type),\n        \"name\": \"cx-prometheus\",\n        \"instanceType\": prometheus_instance_type,\n        \"minSize\": 1,\n        \"maxSize\": 1,\n        \"desiredCapacity\": 1,\n        \"volumeType\": \"gp3\",\n        \"volumeSize\": 20,\n        \"volumeIOPS\": 3000,\n        \"volumeThroughput\": 125,\n        \"labels\": {\"prometheus\": \"true\"},\n        \"taints\": [\n            {\n                \"key\": \"prometheus\",\n                \"value\": \"true\",\n                \"effect\": \"NoSchedule\",\n            },\n        ],\n    }\n    prometheus_nodegroup = merge_override(prometheus_nodegroup, prometheus_settings)\n\n    worker_nodegroups = get_all_worker_nodegroups(ami_map, cluster_config)\n\n    nat_gateway = \"Disable\"\n    if cluster_config[\"nat_gateway\"] == \"single\":\n        nat_gateway = \"Single\"\n    elif cluster_config[\"nat_gateway\"] == \"highly_available\":\n        nat_gateway = \"HighlyAvailable\"\n\n    eks = {\n        \"apiVersion\": \"eksctl.io/v1alpha5\",\n        \"kind\": \"ClusterConfig\",\n        \"metadata\": {\n            \"name\": name,\n            \"region\": region,\n            \"version\": K8S_VERSION,\n            \"tags\": cluster_config[\"tags\"],\n        },\n        \"vpc\": {\"nat\": {\"gateway\": nat_gateway}},\n        \"nodeGroups\": [operator_nodegroup, prometheus_nodegroup] + worker_nodegroups,\n        \"addons\": [\n            {\n                \"name\": \"vpc-cni\",\n                \"version\": \"1.11.3\",\n            },\n        ],\n    }\n\n    if (\n        len(cluster_config.get(\"availability_zones\", [])) > 0\n        and len(cluster_config.get(\"subnets\", [])) == 0\n    ):\n        eks[\"availabilityZones\"] = cluster_config[\"availability_zones\"]\n\n    if len(cluster_config.get(\"subnets\", [])) > 0:\n        eks_subnet_configs = {}\n        for subnet_config in cluster_config[\"subnets\"]:\n            eks_subnet_configs[subnet_config[\"availability_zone\"]] = {\n                \"id\": subnet_config[\"subnet_id\"]\n            }\n\n        if cluster_config.get(\"subnet_visibility\", \"public\") == \"private\":\n            eks[\"vpc\"][\"subnets\"] = {\"private\": eks_subnet_configs}\n        else:\n            eks[\"vpc\"][\"subnets\"] = {\"public\": eks_subnet_configs}\n\n    if cluster_config.get(\"vpc_cidr\", \"\") != \"\":\n        eks[\"vpc\"][\"cidr\"] = cluster_config[\"vpc_cidr\"]\n\n    click.echo(yaml.dump(eks, Dumper=IgnoreAliases, default_flow_style=False, default_style=\"\"))\n\n\nclass IgnoreAliases(yaml.Dumper):\n    \"\"\"By default, yaml dumper tries to compress yaml by annotating collections (lists and maps)\n    and replacing subsequent identical collections with aliases. This class overrides the default\n    behaviour to preserve the duplication of arrays.\n    \"\"\"\n\n    def ignore_aliases(self, data):\n        return True\n\n\nif __name__ == \"__main__\":\n    generate_eks()\n"
  },
  {
    "path": "manager/get_api_load_balancer_state.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport boto3\nimport os\n\nfrom helpers import get_api_load_balancer_v2, get_api_load_balancer, get_api_load_balancer_health\n\n\ndef get_api_load_balancer_state():\n    cluster_name = os.environ[\"CORTEX_CLUSTER_NAME\"]\n    region = os.environ[\"CORTEX_REGION\"]\n    load_balancer_type = os.environ[\"CORTEX_API_LOAD_BALANCER_TYPE\"]\n\n    if load_balancer_type == \"nlb\":\n        client_elbv2 = boto3.client(\"elbv2\", region_name=region)\n        load_balancer = get_api_load_balancer_v2(cluster_name, client_elbv2)\n        return load_balancer[\"State\"][\"Code\"]\n    else:\n        client_elb = boto3.client(\"elb\", region_name=region)\n        load_balancer = get_api_load_balancer(cluster_name, client_elb)\n        return get_api_load_balancer_health(load_balancer[\"LoadBalancerName\"], client_elb)\n\n\nif __name__ == \"__main__\":\n    print(get_api_load_balancer_state(), end=\"\")\n"
  },
  {
    "path": "manager/get_operator_load_balancer_state.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport boto3\nimport os\n\nfrom helpers import get_operator_load_balancer_v2\n\n\ndef get_operator_load_balancer_state():\n    cluster_name = os.environ[\"CORTEX_CLUSTER_NAME\"]\n    region = os.environ[\"CORTEX_REGION\"]\n\n    client_elbv2 = boto3.client(\"elbv2\", region_name=region)\n\n    load_balancer = get_operator_load_balancer_v2(cluster_name, client_elbv2)\n    return load_balancer[\"State\"][\"Code\"]\n\n\nif __name__ == \"__main__\":\n    print(get_operator_load_balancer_state(), end=\"\")\n"
  },
  {
    "path": "manager/get_operator_target_group_status.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport boto3\nimport os\nimport json\n\nfrom helpers import get_operator_load_balancer_v2\n\n\ndef get_operator_target_group_status():\n    cluster_name = os.environ[\"CORTEX_CLUSTER_NAME\"]\n    region = os.environ[\"CORTEX_REGION\"]\n\n    client_elbv2 = boto3.client(\"elbv2\", region_name=region)\n\n    load_balancer_arn = get_operator_load_balancer_v2(cluster_name, client_elbv2)[\"LoadBalancerArn\"]\n    target_group_arn = get_load_balancer_https_target_group_arn(load_balancer_arn, client_elbv2)\n    return get_target_health(target_group_arn, client_elbv2)\n\n\ndef get_load_balancer_https_target_group_arn(load_balancer_arn, client_elbv2):\n    paginator = client_elbv2.get_paginator(\"describe_listeners\")\n    for listener_page in paginator.paginate(LoadBalancerArn=load_balancer_arn):\n        for listener in listener_page[\"Listeners\"]:\n            if listener[\"Port\"] == 443:\n                return listener[\"DefaultActions\"][0][\"TargetGroupArn\"]\n\n    raise Exception(\n        f\"unable to find https target group for operator load balancer ({load_balancer_arn})\"\n    )\n\n\ndef get_target_health(target_group_arn, client_elbv2):\n    response = client_elbv2.describe_target_health(TargetGroupArn=target_group_arn)\n    for health_description in response[\"TargetHealthDescriptions\"]:\n        if health_description[\"TargetHealth\"][\"State\"] == \"healthy\":\n            return \"healthy\"\n\n    return json.dumps(response[\"TargetHealthDescriptions\"])\n\n\nif __name__ == \"__main__\":\n    print(get_operator_target_group_status(), end=\"\")\n"
  },
  {
    "path": "manager/helpers.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\ndef get_operator_load_balancer_v2(cluster_name, client_elbv2):\n    return _get_load_balancer_v2(\"operator\", cluster_name, client_elbv2)\n\n\ndef get_api_load_balancer_v2(cluster_name, client_elbv2):\n    return _get_load_balancer_v2(\"api\", cluster_name, client_elbv2)\n\n\ndef get_api_load_balancer(cluster_name, client_elb):\n    return _get_load_balancer(\"api\", cluster_name, client_elb)\n\n\ndef get_api_load_balancer_health(load_balancer_name, client_elb):\n    instance_health = client_elb.describe_instance_health(\n        LoadBalancerName=load_balancer_name,\n    )\n    for instance_state in instance_health[\"InstanceStates\"]:\n        if instance_state[\"State\"] != \"InService\":\n            return \"inactive\"\n    return \"active\"\n\n\ndef _get_load_balancer_v2(load_balancer_tag, cluster_name, client_elbv2):\n    paginator = client_elbv2.get_paginator(\"describe_load_balancers\")\n    for load_balancer_page in paginator.paginate(PaginationConfig={\"PageSize\": 20}):\n        load_balancers = {\n            load_balancer[\"LoadBalancerArn\"]: load_balancer\n            for load_balancer in load_balancer_page[\"LoadBalancers\"]\n        }\n        tag_descriptions = client_elbv2.describe_tags(ResourceArns=list(load_balancers.keys()))[\n            \"TagDescriptions\"\n        ]\n        for tag_description in tag_descriptions:\n            foundClusterNameTag = False\n            foundLoadBalancerTag = False\n            for tags in tag_description[\"Tags\"]:\n                if tags[\"Key\"] == \"cortex.dev/cluster-name\" and tags[\"Value\"] == cluster_name:\n                    foundClusterNameTag = True\n                if tags[\"Key\"] == \"cortex.dev/load-balancer\" and tags[\"Value\"] == load_balancer_tag:\n                    foundLoadBalancerTag = True\n            if foundClusterNameTag and foundLoadBalancerTag:\n                return load_balancers[tag_description[\"ResourceArn\"]]\n\n    raise Exception(f\"unable to find {load_balancer_tag} load balancer\")\n\n\ndef _get_load_balancer(load_balancer_tag, cluster_name, client_elb):\n    paginator = client_elb.get_paginator(\"describe_load_balancers\")\n    for load_balancer_page in paginator.paginate(PaginationConfig={\"PageSize\": 20}):\n        load_balancers = {\n            load_balancer[\"LoadBalancerName\"]: load_balancer\n            for load_balancer in load_balancer_page[\"LoadBalancerDescriptions\"]\n        }\n        tag_descriptions = client_elb.describe_tags(LoadBalancerNames=list(load_balancers.keys()))[\n            \"TagDescriptions\"\n        ]\n        for tag_description in tag_descriptions:\n            foundClusterNameTag = False\n            foundLoadBalancerTag = False\n            for tags in tag_description[\"Tags\"]:\n                if tags[\"Key\"] == \"cortex.dev/cluster-name\" and tags[\"Value\"] == cluster_name:\n                    foundClusterNameTag = True\n                if tags[\"Key\"] == \"cortex.dev/load-balancer\" and tags[\"Value\"] == load_balancer_tag:\n                    foundLoadBalancerTag = True\n            if foundClusterNameTag and foundLoadBalancerTag:\n                return load_balancers[tag_description[\"LoadBalancerName\"]]\n\n    raise Exception(f\"unable to find {load_balancer_tag} load balancer\")\n"
  },
  {
    "path": "manager/install.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -eo pipefail\n\nexport CORTEX_VERSION=master\nexport CORTEX_VERSION_MINOR=master\nEKSCTL_CLUSTER_TIMEOUT=45m\nEKSCTL_NODEGROUP_TIMEOUT=30m\nmkdir /workspace\n\narg1=\"$1\"\n\nfunction main() {\n  if [ \"$arg1\" = \"--configure\" ]; then\n    cluster_configure\n  else\n    cluster_up\n  fi\n}\n\nfunction cluster_up() {\n  create_eks\n\n  echo -n \"￮ updating cluster configuration \"\n  setup_namespaces\n  setup_configmap\n  echo \"✓\"\n\n  echo -n \"￮ configuring networking (this will take a few minutes) \"\n  setup_ipvs\n  setup_istio\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/apis.yaml.j2 | kubectl apply -f - >/dev/null\n  echo \"✓\"\n\n  echo -n \"￮ configuring autoscaling \"\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/autoscaler.yaml.j2 | kubectl apply -f - >/dev/null\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/activator.yaml.j2 | kubectl apply -f - >/dev/null\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/cluster-autoscaler.yaml.j2 | kubectl apply -f - >/dev/null\n  echo \"✓\"\n\n  echo -n \"￮ configuring async gateway \"\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/async-gateway.yaml.j2 | kubectl apply -f - >/dev/null\n  echo \"✓\"\n\n  echo -n \"￮ configuring logging \"\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/fluent-bit.yaml.j2 | kubectl apply -f - >/dev/null\n  envsubst < manifests/event-exporter.yaml | kubectl apply -f - >/dev/null\n  echo \"✓\"\n\n  echo -n \"￮ configuring metrics \"\n  envsubst < manifests/metrics-server.yaml | kubectl apply -f - >/dev/null\n  setup_prometheus\n  setup_grafana\n  echo \"✓\"\n\n  echo -n \"￮ configuring gpu support (for nodegroups that may require it) \"\n  envsubst < manifests/nvidia.yaml | kubectl apply -f - >/dev/null\n  NVIDIA_COM_GPU_VALUE=true envsubst < manifests/prometheus-dcgm-exporter.yaml | kubectl apply -f - >/dev/null\n  echo \"✓\"\n\n  echo -n \"￮ configuring inf support (for nodegroups that may require it) \"\n  envsubst < manifests/inferentia.yaml | kubectl apply -f - >/dev/null\n  echo \"✓\"\n\n  restart_operator\n  start_controller_manager\n\n  validate_cortex\n\n  echo -e \"\\ncortex is ready!\"\n  if [ \"$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME\" == \"internal\" ]; then\n    echo -e \"\\nnote: you will need to configure VPC Peering to connect to your cluster: https://docs.cortexlabs.com/v/${CORTEX_VERSION_MINOR}/\"\n  fi\n\n  print_endpoints\n}\n\nfunction cluster_configure() {\n  check_eks\n\n  resize_nodegroups\n  add_nodegroups\n  remove_nodegroups\n\n  update_networking\n\n  echo -n \"￮ updating cluster configuration \"\n  setup_configmap\n  echo \"✓\"\n\n  # this is necessary since max_instances may have been updated\n  echo -n \"￮ configuring autoscaling \"\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/cluster-autoscaler.yaml.j2 | kubectl apply -f - >/dev/null\n  echo \"✓\"\n\n  restart_controller_manager\n\n  restart_operator\n\n  validate_cortex\n\n  echo -e \"\\ncortex is ready!\"\n\n  print_endpoints\n}\n\n# creates the eks cluster and configures kubectl\nfunction create_eks() {\n  set +e\n  cluster_info=$(eksctl get cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION -o json 2> /dev/null)\n  cluster_info_exit_code=$?\n  set -e\n\n  # cluster already exists\n  if [ $cluster_info_exit_code -eq 0 ]; then\n    set +e\n    # cluster statuses: https://github.com/aws/aws-sdk-go/blob/master/service/eks/api.go#L6883\n    cluster_status=$(echo \"$cluster_info\" | jq -r 'first | .Status')\n    set -e\n\n    if [ \"$cluster_status\" == \"ACTIVE\" ]; then\n      echo \"error: there is already a cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION\"\n      exit 1\n    elif [ \"$cluster_status\" == \"DELETING\" ]; then\n      echo \"error: your cortex cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION is currently spinning down; please try again once it is completely deleted (may take a few minutes)\"\n      exit 1\n    elif [ \"$cluster_status\" == \"CREATING\" ]; then\n      echo \"error: your cortex cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION is currently spinning up; please try again once it is ready\"\n      exit 1\n    elif [ \"$cluster_status\" == \"UPDATING\" ]; then\n      echo \"error: your cortex cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION is currently updating; please try again once it is ready\"\n      exit 1\n    elif [ \"$cluster_status\" == \"FAILED\" ]; then\n      echo \"error: your cortex cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION is failed; delete it with \\`eksctl delete cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --disable-nodegroup-eviction\\` and try again\"\n      exit 1\n    else  # cluster exists, but is has an unknown status (unexpected)\n      echo \"error: there is already a cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION (status: ${cluster_status})\"\n      exit 1\n    fi\n  fi\n\n  echo -e \"￮ spinning up the cluster (this will take about 30 minutes) ...\\n\"\n  python generate_eks.py $CORTEX_CLUSTER_CONFIG_FILE manifests/ami.json > /workspace/eks.yaml\n  eksctl create cluster --timeout=$EKSCTL_CLUSTER_TIMEOUT --install-neuron-plugin=false --install-nvidia-plugin=false -f /workspace/eks.yaml\n  echo\n\n  write_kubeconfig\n}\n\n# checks that the eks cluster is active and configures kubectl\nfunction check_eks() {\n  set +e\n  cluster_info=$(eksctl get cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION -o json 2> /dev/null)\n  cluster_info_exit_code=$?\n  set -e\n\n  # no cluster\n  if [ $cluster_info_exit_code -ne 0 ]; then\n    echo \"error: there is no cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION; please update your configuration to point to an existing cortex cluster or create a cortex cluster with \\`cortex cluster up\\`\"\n    exit 1\n  fi\n\n  set +e\n  # cluster statuses: https://github.com/aws/aws-sdk-go/blob/master/service/eks/api.go#L6883\n  cluster_status=$(echo \"$cluster_info\" | jq -r 'first | .Status')\n  set -e\n\n  if [ \"$cluster_status\" == \"DELETING\" ]; then\n    echo \"error: your cortex cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION is currently spinning down; please try again once it is completely deleted (may take a few minutes)\"\n    exit 1\n  elif [ \"$cluster_status\" == \"CREATING\" ]; then\n    echo \"error: your cortex cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION is currently spinning up; please try again once it is ready\"\n    exit 1\n  elif [ \"$cluster_status\" == \"UPDATING\" ]; then\n    echo \"error: your cortex cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION is currently updating; please try again once it is ready\"\n    exit 1\n  elif [ \"$cluster_status\" == \"FAILED\" ]; then\n    echo \"error: your cortex cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION is failed; delete it with \\`eksctl delete cluster --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --disable-nodegroup-eviction\\` and try again\"\n    exit 1\n  fi\n\n  # cluster status is ACTIVE or unknown (in which case we'll assume things are ok instead of erroring)\n\n  write_kubeconfig\n}\n\nfunction write_kubeconfig() {\n  eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --verbose=0 | (grep -v \"saved kubeconfig as\" || true)\n  out=$(kubectl get pods 2>&1 || true); if [[ \"$out\" == *\"must be logged in to the server\"* ]]; then echo \"error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortexlabs.com/v/${CORTEX_VERSION_MINOR}/\"; exit 1; fi\n}\n\nfunction setup_namespaces() {\n  # doing a patch to prevent getting the kubectl.kubernetes.io/last-applied-configuration annotation warning\n  kubectl patch namespace default -p '{\"metadata\": {\"labels\": {\"istio-discovery\": \"enabled\"}}}' >/dev/null\n  kubectl apply -f manifests/namespaces.yaml >/dev/null\n}\n\nfunction setup_configmap() {\n  envsubst < manifests/default_cortex_cli_config.yaml > tmp_cli_config.yaml\n  kubectl -n=default create configmap 'client-config' \\\n    --from-file='cli.yaml'=tmp_cli_config.yaml \\\n    -o yaml --dry-run=client | kubectl apply -f - >/dev/null\n  rm tmp_cli_config.yaml\n\n  kubectl -n=default create configmap 'cluster-config' \\\n    --from-file='cluster.yaml'=$CORTEX_CLUSTER_CONFIG_FILE \\\n    -o yaml --dry-run=client | kubectl apply -f - >/dev/null\n\n  kubectl -n=default create configmap 'env-vars' \\\n    --from-literal='CORTEX_VERSION'=$CORTEX_VERSION \\\n    --from-literal='CORTEX_REGION'=$CORTEX_REGION \\\n    --from-literal='AWS_DEFAULT_REGION'=$CORTEX_REGION \\\n    --from-literal='AWS_RETRY_MODE'=\"standard\" \\\n    --from-literal='AWS_MAX_ATTEMPTS'=\"5\" \\\n    --from-literal='CORTEX_TELEMETRY_DISABLE'=$CORTEX_TELEMETRY_DISABLE \\\n    --from-literal='CORTEX_TELEMETRY_SENTRY_DSN'=$CORTEX_TELEMETRY_SENTRY_DSN \\\n    --from-literal='CORTEX_TELEMETRY_SEGMENT_WRITE_KEY'=$CORTEX_TELEMETRY_SEGMENT_WRITE_KEY \\\n    --from-literal='CORTEX_DEV_DEFAULT_IMAGE_REGISTRY'=$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY \\\n    -o yaml --dry-run=client | kubectl apply -f - >/dev/null\n}\n\nfunction setup_prometheus() {\n  envsubst < manifests/prometheus-operator.yaml | kubectl apply -f - >/dev/null\n  envsubst < manifests/prometheus-statsd-exporter.yaml | kubectl apply -f - >/dev/null\n  envsubst < manifests/prometheus-kubelet-exporter.yaml | kubectl apply -f - >/dev/null\n  envsubst < manifests/prometheus-kube-state-metrics.yaml | kubectl apply -f - >/dev/null\n  envsubst < manifests/prometheus-node-exporter.yaml | kubectl apply -f - >/dev/null\n  envsubst < manifests/prometheus-monitoring.yaml | kubectl apply -f - >/dev/null\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/prometheus-additional-scrape-configs.yaml.j2 > prometheus-additional-scrape-configs.yaml\n  if ! kubectl get secret -n prometheus additional-scrape-configs >/dev/null 2>&1; then\n    kubectl create secret generic -n prometheus additional-scrape-configs --from-file=prometheus-additional-scrape-configs.yaml > /dev/null\n  fi\n}\n\nfunction setup_grafana() {\n  kubectl apply -f manifests/grafana/grafana-dashboard-realtime.yaml >/dev/null\n  kubectl apply -f manifests/grafana/grafana-dashboard-async.yaml >/dev/null\n  kubectl apply -f manifests/grafana/grafana-dashboard-batch.yaml >/dev/null\n  kubectl apply -f manifests/grafana/grafana-dashboard-task.yaml >/dev/null\n  kubectl apply -f manifests/grafana/grafana-dashboard-cluster.yaml >/dev/null\n  kubectl apply -f manifests/grafana/grafana-dashboard-nodes.yaml >/dev/null\n  if [ \"$CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD\" = \"true\" ]; then\n    kubectl apply -f manifests/grafana/grafana-dashboard-control-plane.yaml >/dev/null\n  fi\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/grafana/grafana.yaml.j2 | kubectl apply -f - >/dev/null\n}\n\nfunction restart_operator() {\n  echo -n \"￮ starting operator \"\n  kubectl -n=default delete --ignore-not-found=true --grace-period=10 deployment operator >/dev/null 2>&1\n  printed_dot=\"false\"\n  until [ \"$(kubectl -n=default get pods -l workloadID=operator -o json | jq -j '.items | length')\" -eq \"0\" ]; do echo -n \".\"; printed_dot=\"true\"; sleep 2; done\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/operator.yaml.j2 > /workspace/operator.yaml\n  kubectl apply -f /workspace/operator.yaml >/dev/null\n  if [ \"$printed_dot\" == \"true\" ]; then echo \" ✓\"; else echo \"✓\"; fi\n}\n\nfunction start_controller_manager() {\n  echo -n \"￮ starting controller manager \"\n\n  kustomize build config/default | kubectl delete --ignore-not-found=true -f - >/dev/null\n\n  cd config/manager \\\n    && kustomize edit set image controller=${CORTEX_IMAGE_CONTROLLER_MANAGER} \\\n    && cd ../.. > /dev/null\n\n  kustomize build config/default | kubectl apply -f - >/dev/null\n  echo \"✓\"\n}\n\nfunction restart_controller_manager() {\n  echo -n \"￮ restarting controller manager \"\n\n  kubectl rollout restart deployments/operator-controller-manager >/dev/null\n\n  echo \"✓\"\n}\n\nfunction resize_nodegroups() {\n  if [ -z \"$CORTEX_NODEGROUP_NAMES_TO_UPDATE\" ]; then\n    return\n  fi\n\n  eksctl get nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --verbose=0 -o json > nodegroups.json\n  eks_ng_len=$(cat nodegroups.json | jq -r length)\n  cfg_ng_len=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups | yq -r length)\n\n  for cfg_ng_name in $CORTEX_NODEGROUP_NAMES_TO_UPDATE; do\n    has_ng=\"false\"\n    for eks_idx in $(seq 0 $(($eks_ng_len-1))); do\n      stack_ng=$(cat nodegroups.json | jq -r .[$eks_idx].Name)\n      if [ \"$stack_ng\" = \"cx-operator\" ]; then\n        continue\n      fi\n      if [[ \"$stack_ng\" == *\"$cfg_ng_name\" ]]; then\n        has_ng=\"true\"\n        break\n      fi\n    done\n\n    if [ \"$has_ng\" == \"false\" ]; then\n      echo -e \"error: \\\"$cfg_ng_name\\\" nodegroup (\\\"cx-*-$cfg_ng_name\\\" on aws) couldn't be scaled because stack couldn't be found\\n\"\n      exit 1\n    fi\n\n    for cfg_idx in $(seq 0 $(($cfg_ng_len-1))); do\n      cfg_ng=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups[$cfg_idx].name)\n      if [ \"$cfg_ng\" = \"$cfg_ng_name\" ]; then\n        break\n      fi\n    done\n\n    desired=$(cat nodegroups.json | jq -r .[$eks_idx].DesiredCapacity)\n    existing_min=$(cat nodegroups.json | jq -r .[$eks_idx].MinSize)\n    existing_max=$(cat nodegroups.json | jq -r .[$eks_idx].MaxSize)\n    updating_min=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups[$cfg_idx].min_instances)\n    updating_max=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .node_groups[$cfg_idx].max_instances)\n\n    if [ \"$desired\" -lt $updating_min ]; then\n      desired=$updating_min\n    fi\n    if [ \"$desired\" -gt $updating_max ]; then\n      desired=$updating_max\n    fi\n\n    if [ \"$existing_min\" != \"$updating_min\" ] && [ \"$existing_max\" != \"$updating_max\" ]; then\n      echo \"￮ nodegroup $cfg_ng_name: updating min instances to $updating_min and max instances to $updating_max\"\n      eksctl scale nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION $stack_ng --nodes $desired --nodes-min $updating_min --nodes-max $updating_max --timeout \"60m\"\n      echo\n    elif [ \"$existing_min\" != \"$updating_min\" ]; then\n      echo \"￮ nodegroup $cfg_ng_name: updating min instances to $updating_min\"\n      eksctl scale nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION $stack_ng --nodes $desired --nodes-min $updating_min --timeout \"60m\"\n      echo\n    elif [ \"$existing_max\" != \"$updating_max\" ]; then\n      echo \"￮ nodegroup $cfg_ng_name: updating max instances to $updating_max\"\n      eksctl scale nodegroup --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION $stack_ng --nodes $desired --nodes-max $updating_max --timeout \"60m\"\n      echo\n    fi\n  done\n\n  rm nodegroups.json\n}\n\nfunction add_nodegroups() {\n  if [ -z \"$CORTEX_NODEGROUP_NAMES_TO_ADD\" ]; then\n    return\n  fi\n\n  nodegroup_names=\"$(join_by , $CORTEX_NODEGROUP_NAMES_TO_ADD)\"\n\n  echo \"￮ adding new nodegroup(s) to the cluster ...\"\n  python generate_eks.py $CORTEX_CLUSTER_CONFIG_FILE manifests/ami.json --add-cortex-node-groups=\"$nodegroup_names\" > /workspace/nodegroups.yaml\n  eksctl create nodegroup --timeout=$EKSCTL_NODEGROUP_TIMEOUT --install-neuron-plugin=false --install-nvidia-plugin=false --skip-outdated-addons-check -f /workspace/nodegroups.yaml\n  rm /workspace/nodegroups.yaml\n  echo\n}\n\nfunction remove_nodegroups() {\n  if [ -z \"$CORTEX_EKS_NODEGROUP_NAMES_TO_REMOVE\" ]; then\n    return\n  fi\n\n  eks_nodegroup_names=\"$(join_by , $CORTEX_EKS_NODEGROUP_NAMES_TO_REMOVE)\"\n\n  echo \"￮ removing nodegroup(s) from the cluster ...\"\n  python generate_eks.py $CORTEX_CLUSTER_CONFIG_FILE manifests/ami.json --remove-eks-node-groups=\"$eks_nodegroup_names\" > /workspace/nodegroups.yaml\n  eksctl delete nodegroup --timeout=$EKSCTL_NODEGROUP_TIMEOUT --approve -f /workspace/nodegroups.yaml\n  rm /workspace/nodegroups.yaml\n  echo\n}\n\nfunction setup_ipvs() {\n  # get a random kube-proxy pod\n  kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null\n  kube_proxy_pod=$(kubectl get pod -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadata.name}' | cut -d \" \" -f1)\n\n  # export kube-proxy's current config\n  kubectl exec -it -n kube-system ${kube_proxy_pod} -- cat /var/lib/kube-proxy-config/config > proxy_config.yaml\n\n  # upgrade proxy mode from the exported kube-proxy config\n  python upgrade_kube_proxy_mode.py proxy_config.yaml > upgraded_proxy_config.yaml\n\n  # update kube-proxy's configmap to include the updated configuration\n  kubectl get configmap -n kube-system kube-proxy -o yaml | yq --arg replace \"`cat upgraded_proxy_config.yaml`\" '.data.config=$replace' | kubectl apply -f - >/dev/null\n\n  # patch the kube-proxy daemonset\n  kubectl patch ds -n kube-system kube-proxy --patch \"$(cat manifests/kube-proxy.patch.yaml)\" >/dev/null\n  kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null\n}\n\nfunction setup_istio() {\n  if ! grep -q \"istio-customgateway-certs\" <<< $(kubectl get secret -n istio-system); then\n    WEBSITE=localhost\n    openssl req -subj \"/C=US/CN=$WEBSITE\" -newkey rsa:2048 -nodes -keyout $WEBSITE.key -x509 -days 3650 -out $WEBSITE.crt >/dev/null 2>&1\n    kubectl create -n istio-system secret tls istio-customgateway-certs --key $WEBSITE.key --cert $WEBSITE.crt >/dev/null\n  fi\n\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/istio.yaml.j2 > /workspace/istio.yaml\n  output_if_error istio-${ISTIO_VERSION}/bin/istioctl install --skip-confirmation --filename /workspace/istio.yaml\n}\n\nfunction update_networking() {\n  prev_ssl_certificate_arn=$(kubectl get svc ingressgateway-apis -n=istio-system -o json | jq -r '.metadata.annotations.\"service.beta.kubernetes.io/aws-load-balancer-ssl-cert\"')\n\n  if [ \"$prev_ssl_certificate_arn\" = \"null\" ]; then\n      prev_ssl_certificate_arn=\"\"\n  fi\n\n  new_ssl_certificate_arn=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .ssl_certificate_arn)\n\n  if [ \"$new_ssl_certificate_arn\" = \"null\" ]; then\n      new_ssl_certificate_arn=\"\"\n  fi\n\n  prev_api_whitelist_ip_address=$(kubectl get svc ingressgateway-apis  -n=istio-system -o yaml | yq -r -c \".spec.loadBalancerSourceRanges\")\n  prev_operator_whitelist_ip_address=$(kubectl get svc ingressgateway-operator  -n=istio-system -o yaml | yq -r -c \".spec.loadBalancerSourceRanges\")\n\n  new_api_whitelist_ip_address=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r -c \".api_load_balancer_cidr_white_list\")\n  new_operator_whitelist_ip_address=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r -c \".operator_load_balancer_cidr_white_list\")\n\n  if [ \"$prev_ssl_certificate_arn\" = \"$new_ssl_certificate_arn\" ] && [ \"$prev_api_whitelist_ip_address\" = \"$new_api_whitelist_ip_address\" ] && [ \"$prev_operator_whitelist_ip_address\" = \"$new_operator_whitelist_ip_address\" ] ; then\n      return\n  fi\n\n  echo -n \"￮ updating networking configuration \"\n\n  if [ \"$new_ssl_certificate_arn\" != \"$prev_ssl_certificate_arn\" ] ; then\n      # there is a bug where changing the certificate annotation will not cause the HTTPS listener in the NLB to update\n      # the current workaround is to delete the HTTPS listener and have it recreated with istioctl\n      if [ \"$prev_ssl_certificate_arn\" != \"\" ] ; then\n        kubectl patch svc ingressgateway-apis -n=istio-system --type=json -p=\"[{'op': 'remove', 'path': '/metadata/annotations/service.beta.kubernetes.io~1aws-load-balancer-ssl-cert'}]\" >/dev/null\n      fi\n      https_index=$(kubectl get svc ingressgateway-apis -n=istio-system -o json  | jq '.spec.ports | map(.name == \"https\") | index(true)')\n      if [ \"$https_index\" != \"null\" ] ; then\n        kubectl patch svc ingressgateway-apis -n=istio-system --type=json -p=\"[{'op': 'remove', 'path': '/spec/ports/$https_index'}]\" >/dev/null\n      fi\n  fi\n\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/istio.yaml.j2 > /workspace/istio.yaml\n  output_if_error istio-${ISTIO_VERSION}/bin/istioctl install --skip-confirmation --filename /workspace/istio.yaml\n  python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/apis.yaml.j2 > /workspace/apis.yaml\n  kubectl apply -f /workspace/apis.yaml >/dev/null\n\n  echo \"✓\"\n}\n\nfunction validate_cortex() {\n  set +e\n\n  validation_start_time=\"$(date +%s)\"\n\n  echo -n \"￮ waiting for load balancers \"\n\n  operator_pod_name=\"\"\n  operator_pod_is_ready=\"\"\n  operator_pod_status=\"\"\n  operator_endpoint=\"\"\n  api_load_balancer_endpoint=\"\"\n  operator_load_balancer_state=\"\"\n  api_load_balancer_state=\"\"\n  operator_target_group_status=\"\"\n  operator_endpoint_reachable=\"\"\n  prometheus_ready=\"\"\n  success_cycles=0\n\n  while true; do\n    # 30 minute timeout\n    now=\"$(date +%s)\"\n    if [ \"$now\" -ge \"$(($validation_start_time+1800))\" ]; then\n      echo -e \"\\n\\ntimeout has occurred when validating your cortex cluster\"\n      echo -e \"\\ndebugging info:\"\n      if [ \"$operator_pod_name\" != \"\" ]; then\n        echo \"operator pod name: $operator_pod_name\"\n      fi\n      if [ \"$operator_pod_is_ready\" != \"\" ]; then\n        echo \"operator pod is ready: $operator_pod_is_ready\"\n      fi\n      if [ \"$operator_pod_status\" != \"\" ]; then\n        echo \"operator pod status: $operator_pod_status\"\n      fi\n      if [ \"$operator_endpoint\" != \"\" ]; then\n        echo \"operator endpoint: $operator_endpoint\"\n      fi\n      if [ \"${prometheus_ready}\" != \"\" ]; then\n        echo \"prometheus is ready: $prometheus_ready\"\n      fi\n      if [ \"$api_load_balancer_endpoint\" != \"\" ]; then\n        echo \"api load balancer endpoint: $api_load_balancer_endpoint\"\n      fi\n      if [ \"$operator_load_balancer_state\" != \"\" ]; then\n        echo \"operator load balancer state: $operator_load_balancer_state\"\n      fi\n      if [ \"$api_load_balancer_state\" != \"\" ]; then\n        echo \"api load balancer state: $api_load_balancer_state\"\n      fi\n      if [ \"$operator_target_group_status\" != \"\" ]; then\n        echo \"operator target group status: $operator_target_group_status\"\n      fi\n      if [ \"$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME\" == \"internet-facing\" ] && [ \"$operator_endpoint_reachable\" != \"\" ]; then\n        echo \"operator endpoint reachable: $operator_endpoint_reachable\"\n      fi\n      if [ \"$operator_endpoint\" != \"\" ]; then\n        echo \"operator curl response:\"\n        curl --max-time 3 \"${operator_endpoint}/verifycortex\"\n      fi\n\n      echo \"additional networking events:\"\n      kubectl get events -n=istio-system --field-selector involvedObject.kind=Service --sort-by=\".metadata.managedFields[0].time\" | tail -10\n      kubectl get events -n=istio-system --field-selector involvedObject.kind=Pod --sort-by=\".metadata.managedFields[0].time\" | tail -10\n      echo\n\n      exit 1\n    fi\n\n    echo -n \".\"\n    sleep 5\n\n    operator_pod_name=$(kubectl -n=default get pods -o=name --sort-by=.metadata.creationTimestamp | (grep \"^pod/operator-\" || true) | tail -1)\n    if [ \"$operator_pod_name\" == \"\" ]; then\n      success_cycles=0\n      continue\n    fi\n\n    operator_pod_is_ready=$(kubectl -n=default get \"$operator_pod_name\" -o jsonpath='{.status.containerStatuses[0].ready}')\n    if [ \"$operator_pod_is_ready\" != \"true\" ]; then\n      operator_pod_status=$(kubectl -n=default get \"$operator_pod_name\" -o jsonpath='{.status.containerStatuses[0]}')\n      if [[ \"$operator_pod_status\" == *\"ImagePullBackOff\"* ]]; then\n        echo -e \"\\nerror: the operator image you specified could not be pulled:\"\n        echo $operator_pod_status\n        echo\n        exit 1\n      fi\n\n      num_restarts=$(kubectl -n=default get \"$operator_pod_name\" -o jsonpath='{.status.containerStatuses[0].restartCount}')\n      if [[ $num_restarts -ge 2 ]]; then\n        echo -e \"\\n\\nan error occurred when starting the cortex operator\"\n        echo -e \"\\noperator logs (currently running container):\\n\"\n        kubectl -n=default logs \"$operator_pod_name\"\n        echo -e \"\\noperator logs (previous container):\\n\"\n        kubectl -n=default logs \"$operator_pod_name\" --previous\n        echo\n        exit 1\n      fi\n\n      success_cycles=0\n      continue\n    fi\n    operator_pod_status=\"\"  # reset operator_pod_status since now the operator is active\n\n    if [ \"$operator_endpoint\" == \"\" ]; then\n      out=$(kubectl -n=istio-system get service ingressgateway-operator -o json | tr -d '[:space:]')\n      if [[ $out != *'\"loadBalancer\":{\"ingress\":[{\"'* ]]; then\n        success_cycles=0\n        continue\n      fi\n      operator_endpoint=$(kubectl -n=istio-system get service ingressgateway-operator -o json | tr -d '[:space:]' | sed 's/.*{\\\"hostname\\\":\\\"\\(.*\\)\\\".*/\\1/')\n    fi\n\n    if [ \"$prometheus_ready\" == \"\" ]; then\n      readyReplicas=$(kubectl get statefulset -n prometheus prometheus-prometheus -o jsonpath='{.status.readyReplicas}' 2> /dev/null)\n      desiredReplicas=$(kubectl get statefulset -n prometheus prometheus-prometheus -o jsonpath='{.status.replicas}' 2> /dev/null)\n\n      if [ \"$readyReplicas\" != \"\" ] && [ \"$desiredReplicas\" != \"\" ]; then\n        if [ \"$readyReplicas\" == \"$desiredReplicas\" ]; then\n          prometheus_ready=\"true\"\n        else\n          prometheus_ready=\"false\"\n        fi\n      fi\n\n      if [ \"$prometheus_ready\" != \"true\" ]; then\n        success_cycles=0\n        continue\n      fi\n    fi\n\n    if [ \"$api_load_balancer_endpoint\" == \"\" ]; then\n      out=$(kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]')\n      if [[ $out != *'\"loadBalancer\":{\"ingress\":[{\"'* ]]; then\n        success_cycles=0\n        continue\n      fi\n      api_load_balancer_endpoint=$(kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]' | sed 's/.*{\\\"hostname\\\":\\\"\\(.*\\)\\\".*/\\1/')\n    fi\n\n    operator_load_balancer_state=\"$(python get_operator_load_balancer_state.py)\"  # don't cache this result\n    if [ \"$operator_load_balancer_state\" != \"active\" ]; then\n      success_cycles=0\n      continue\n    fi\n\n    api_load_balancer_state=\"$(python get_api_load_balancer_state.py)\"  # don't cache this result\n    if [ \"$api_load_balancer_state\" != \"active\" ]; then\n      success_cycles=0\n      continue\n    fi\n\n    operator_target_group_status=\"$(python get_operator_target_group_status.py)\"  # don't cache this result\n    if [ \"$operator_target_group_status\" != \"healthy\" ]; then\n      success_cycles=0\n      continue\n    fi\n\n    if [ \"$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME\" == \"internet-facing\" ]; then\n      operator_endpoint_reachable=\"false\"  # don't cache this result\n      if ! curl --max-time 3 \"${operator_endpoint}/verifycortex\" >/dev/null 2>&1; then\n        success_cycles=0\n        continue\n      fi\n      operator_endpoint_reachable=\"true\"\n    fi\n\n    if [[ $success_cycles -lt 5 ]]; then\n      ((success_cycles++))\n      continue\n    fi\n\n    break\n  done\n\n  echo \" ✓\"\n}\n\nfunction print_endpoints() {\n  echo \"\"\n\n  operator_endpoint=$(get_operator_endpoint)\n  api_load_balancer_endpoint=$(get_api_load_balancer_endpoint)\n\n  echo \"operator:          $operator_endpoint\"  # before modifying this, search for this prefix\n  echo \"api load balancer: $api_load_balancer_endpoint\"\n}\n\nfunction get_operator_endpoint() {\n  kubectl -n=istio-system get service ingressgateway-operator -o json | tr -d '[:space:]' | sed 's/.*{\\\"hostname\\\":\\\"\\(.*\\)\\\".*/\\1/'\n}\n\nfunction get_api_load_balancer_endpoint() {\n  kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]' | sed 's/.*{\\\"hostname\\\":\\\"\\(.*\\)\\\".*/\\1/'\n}\n\nfunction output_if_error() {\n  set +e\n  rm --force /tmp/suppress.out 2> /dev/null\n  ${1+\"$@\"} > /tmp/suppress.out 2>&1\n  if [ \"$?\" != \"0\" ]; then\n    echo\n    cat /tmp/suppress.out\n    exit 1\n  fi\n  rm --force /tmp/suppress.out 2> /dev/null\n  set -e\n}\n\nfunction join_by {\n  local IFS=\"$1\"\n  shift\n  echo \"$*\"\n}\n\nmain\n"
  },
  {
    "path": "manager/manifests/activator.yaml.j2",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: activator\n  namespace: default\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: activator-role\nrules:\n- apiGroups:\n  - \"networking.istio.io\"\n  resources:\n  - virtualservices\n  verbs:\n  - list\n  - watch\n- apiGroups:\n  - \"apps\"\n  resources:\n  - deployments\n  verbs:\n  - list\n  - watch\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: activator-rolebinding\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: activator-role\nsubjects:\n- kind: ServiceAccount\n  name: activator\n  namespace: default\n\n---\n\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: activator\nspec:\n  selector:\n    matchLabels:\n      app: activator\n  template:\n    metadata:\n      name: activator\n      labels:\n        app: activator\n    spec:\n      serviceAccountName: activator\n      containers:\n        - name: activator\n          imagePullPolicy: Always\n          image: {{ config['image_activator'] }}\n          args:\n            - \"--in-cluster\"\n            - \"--port=8000\"\n            - \"--autoscaler-url=http://autoscaler.default:8000\"\n            - \"--namespace=default\"\n          ports:\n            - name: http\n              containerPort: 8000\n            - name: admin\n              containerPort: 15000\n          livenessProbe:\n            httpGet:\n              port: 8000\n              path: /\n              httpHeaders:\n                - name: X-Cortex-Probe\n                  value: \"true\"\n          readinessProbe:\n            httpGet:\n              port: 8000\n              path: /\n              httpHeaders:\n                - name: X-Cortex-Probe\n                  value: \"true\"\n          resources:\n            requests:\n              cpu: 100m\n              memory: 100Mi\n            limits:\n              cpu: 250m\n              memory: 300Mi\n          envFrom:\n          - configMapRef:\n              name: env-vars\n---\n\napiVersion: v1\nkind: Service\nmetadata:\n  name: activator\nspec:\n  type: ClusterIP\n  selector:\n    app: activator\n  ports:\n    - port: 8000\n\n---\n\napiVersion: autoscaling/v2beta2\nkind: HorizontalPodAutoscaler\nmetadata:\n  name: activator-hpa\nspec:\n  scaleTargetRef:\n    apiVersion: apps/v1\n    kind: Deployment\n    name: activator\n  minReplicas: 1\n  maxReplicas: 10\n  metrics:\n    - type: Resource\n      resource:\n        name: cpu\n        target:\n          type: Utilization\n          averageUtilization: 70\n    - type: Resource\n      resource:\n        name: memory\n        target:\n          type: Utilization\n          averageUtilization: 70\n"
  },
  {
    "path": "manager/manifests/ami.json",
    "content": "{\n\t\"1.22\": {\n\t\t\"af-south-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-060db0c99048ca916\",\n\t\t\t\"cpu_amd64\": \"ami-05b1996f962e1a840\",\n\t\t\t\"cpu_arm64\": \"ami-03156c2c7ecbe55ca\"\n\t\t},\n\t\t\"ap-east-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0ac2ce8b81291a5f1\",\n\t\t\t\"cpu_amd64\": \"ami-0bcbff72f7d2bb056\",\n\t\t\t\"cpu_arm64\": \"ami-07e2412bfc3c401b5\"\n\t\t},\n\t\t\"ap-northeast-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0daf96135f91047cf\",\n\t\t\t\"cpu_amd64\": \"ami-05955794c71f6cde9\",\n\t\t\t\"cpu_arm64\": \"ami-035ae428df1978df6\"\n\t\t},\n\t\t\"ap-northeast-2\": {\n\t\t\t\"accelerated_amd64\": \"ami-040df236ed16db21b\",\n\t\t\t\"cpu_amd64\": \"ami-0f2bcb5a524a100a4\",\n\t\t\t\"cpu_arm64\": \"ami-0c4d775e231b413f5\"\n\t\t},\n\t\t\"ap-northeast-3\": {\n\t\t\t\"accelerated_amd64\": \"ami-0bbe7685e20d96d41\",\n\t\t\t\"cpu_amd64\": \"ami-09b5be21ab02366ed\",\n\t\t\t\"cpu_arm64\": \"ami-03412067ed8e25694\"\n\t\t},\n\t\t\"ap-south-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0a373620de38ec67d\",\n\t\t\t\"cpu_amd64\": \"ami-02c0a86da73ff476d\",\n\t\t\t\"cpu_arm64\": \"ami-055c892f309c8766d\"\n\t\t},\n\t\t\"ap-southeast-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-006821bdff8e4bcb6\",\n\t\t\t\"cpu_amd64\": \"ami-09d066ada84539c8b\",\n\t\t\t\"cpu_arm64\": \"ami-06b35c1ca0ddc31c2\"\n\t\t},\n\t\t\"ap-southeast-2\": {\n\t\t\t\"accelerated_amd64\": \"ami-04ab748d420d0c991\",\n\t\t\t\"cpu_amd64\": \"ami-0471c8c731aff64c4\",\n\t\t\t\"cpu_arm64\": \"ami-0bf89cf1628a1835f\"\n\t\t},\n\t\t\"ca-central-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0cf22a8efb72853d5\",\n\t\t\t\"cpu_amd64\": \"ami-03dc7e20ad86871af\",\n\t\t\t\"cpu_arm64\": \"ami-0cc7b246694710794\"\n\t\t},\n\t\t\"eu-central-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0f6759a0eb110c0b1\",\n\t\t\t\"cpu_amd64\": \"ami-0f0bee6c186e51fd3\",\n\t\t\t\"cpu_arm64\": \"ami-024f7b2f337648f01\"\n\t\t},\n\t\t\"eu-north-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0b00a6e259d3bb3f0\",\n\t\t\t\"cpu_amd64\": \"ami-0cb92e7e44afc2fe2\",\n\t\t\t\"cpu_arm64\": \"ami-0559329495f812c1d\"\n\t\t},\n\t\t\"eu-south-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0a07fabae484c3acb\",\n\t\t\t\"cpu_amd64\": \"ami-076e155512c7ccd51\",\n\t\t\t\"cpu_arm64\": \"ami-0e21d55220a1ce340\"\n\t\t},\n\t\t\"eu-west-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0ce94487e6828454f\",\n\t\t\t\"cpu_amd64\": \"ami-03ba20706dab77da5\",\n\t\t\t\"cpu_arm64\": \"ami-04f28ffa44ceadaa6\"\n\t\t},\n\t\t\"eu-west-2\": {\n\t\t\t\"accelerated_amd64\": \"ami-09ba6f26d6f9bafb1\",\n\t\t\t\"cpu_amd64\": \"ami-08b4681872300c3d1\",\n\t\t\t\"cpu_arm64\": \"ami-07dd0d8548066837c\"\n\t\t},\n\t\t\"eu-west-3\": {\n\t\t\t\"accelerated_amd64\": \"ami-018ab5c8ff8b25e33\",\n\t\t\t\"cpu_amd64\": \"ami-02ec360db9d1c78aa\",\n\t\t\t\"cpu_arm64\": \"ami-030f5a65291bee0cf\"\n\t\t},\n\t\t\"me-south-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-04af060694de1bde4\",\n\t\t\t\"cpu_amd64\": \"ami-0da6a4f209e783387\",\n\t\t\t\"cpu_arm64\": \"ami-037082471a912a223\"\n\t\t},\n\t\t\"sa-east-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-022752810a1fe5dbb\",\n\t\t\t\"cpu_amd64\": \"ami-0ce71a979799fcfe1\",\n\t\t\t\"cpu_arm64\": \"ami-04ba15cc4ce9eec0c\"\n\t\t},\n\t\t\"us-east-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-05ed50f0d1f4bcf18\",\n\t\t\t\"cpu_amd64\": \"ami-0d5cbb67678bc879c\",\n\t\t\t\"cpu_arm64\": \"ami-013845c4dfca498f6\"\n\t\t},\n\t\t\"us-east-2\": {\n\t\t\t\"accelerated_amd64\": \"ami-00bc47f895d9384a0\",\n\t\t\t\"cpu_amd64\": \"ami-0c0c07bda9db344ec\",\n\t\t\t\"cpu_arm64\": \"ami-01c1233f0498cd796\"\n\t\t},\n\t\t\"us-gov-east-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0a1f4110c37454db8\",\n\t\t\t\"cpu_amd64\": \"ami-03a40d00b52a4b7d4\",\n\t\t\t\"cpu_arm64\": \"ami-01d8133657b1e33e5\"\n\t\t},\n\t\t\"us-gov-west-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-03d3d9405fd000a27\",\n\t\t\t\"cpu_amd64\": \"ami-05067e9caa5a2b27b\",\n\t\t\t\"cpu_arm64\": \"ami-0c775d2df44eafd24\"\n\t\t},\n\t\t\"us-west-1\": {\n\t\t\t\"accelerated_amd64\": \"ami-0dff3ce23d6001ee4\",\n\t\t\t\"cpu_amd64\": \"ami-0c68a0cdb2f0ae2b7\",\n\t\t\t\"cpu_arm64\": \"ami-0fe9484a6f1e7b2f1\"\n\t\t},\n\t\t\"us-west-2\": {\n\t\t\t\"accelerated_amd64\": \"ami-077be0d48d8abb41b\",\n\t\t\t\"cpu_amd64\": \"ami-045759229c0add028\",\n\t\t\t\"cpu_arm64\": \"ami-08850c88dbabfc43b\"\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "manager/manifests/apis.yaml.j2",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: networking.istio.io/v1beta1\nkind: Gateway\nmetadata:\n  name: apis-gateway\n  namespace: default\nspec:\n  selector:\n    istio: ingressgateway-apis\n  servers:\n    - port:\n        number: 80\n        name: http\n        protocol: HTTP\n      hosts:\n        - \"*\"\n    {% if config.get('ssl_certificate_arn', '') == '' %}\n    - port:\n        number: 443\n        name: https\n        protocol: HTTPS\n      hosts:\n        - \"*\"\n      tls:\n        mode: SIMPLE\n        serverCertificate: /etc/istio/customgateway-certs/tls.crt\n        privateKey: /etc/istio/customgateway-certs/tls.key\n    {% else %}\n    - port:\n        number: 443\n        name: https\n        protocol: HTTP\n      hosts:\n        - \"*\"\n    {% endif %}\n"
  },
  {
    "path": "manager/manifests/async-gateway.yaml.j2",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: async-gateway\n  namespace: default\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: async-gateway\n  namespace: default\nspec:\n  selector:\n    matchLabels:\n      app: async-gateway\n  strategy:\n    rollingUpdate:\n      maxSurge: 25%\n      maxUnavailable: 25%\n    type: RollingUpdate\n  template:\n    metadata:\n      name: async-gateway\n      labels:\n        app: async-gateway\n    spec:\n      serviceAccountName: async-gateway\n      containers:\n        - name: gateway\n          image: {{ config[\"image_async_gateway\"] }}\n          imagePullPolicy: Always\n          args:\n            - --port\n            - \"8888\"\n            - --cluster-uid\n            - \"{{ config[\"cluster_uid\"] }}\"\n            - --bucket\n            - \"{{ config[\"bucket\"] }}\"\n          envFrom:\n            - configMapRef:\n                name: env-vars\n          ports:\n            - containerPort: 8888\n          readinessProbe:\n            httpGet:\n              path: /healthz\n              port: 8888\n              scheme: HTTP\n          livenessProbe:\n            httpGet:\n              path: /healthz\n              port: 8888\n              scheme: HTTP\n          resources:\n            requests:\n              cpu: 400m\n              memory: 512Mi\n            limits:\n              cpu: 400m\n---\napiVersion: v1\nkind: Service\nmetadata:\n  name: async-gateway\nspec:\n  type: ClusterIP\n  selector:\n    app: async-gateway\n  ports:\n    - port: 8888\n---\napiVersion: autoscaling/v2beta2\nkind: HorizontalPodAutoscaler\nmetadata:\n  name: async-gateway\nspec:\n  maxReplicas: 20\n  minReplicas: 1\n  scaleTargetRef:\n    apiVersion: apps/v1\n    kind: Deployment\n    name: async-gateway\n  metrics:\n    - type: Resource\n      resource:\n        name: cpu\n        target:\n          type: Utilization\n          averageUtilization: 90\n    - type: Resource\n      resource:\n        name: memory\n        target:\n          type: Utilization\n          averageUtilization: 90\n"
  },
  {
    "path": "manager/manifests/autoscaler.yaml.j2",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: autoscaler\n  namespace: default\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: autoscaler-role\nrules:\n- apiGroups:\n  - \"networking.istio.io\"\n  resources:\n  - virtualservices\n  verbs:\n  - get\n  - list\n  - watch\n  - update\n- apiGroups:\n  - \"apps\"\n  resources:\n  - deployments\n  verbs:\n  - get\n  - update\n  - watch\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: autoscaler-rolebinding\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: autoscaler-role\nsubjects:\n- kind: ServiceAccount\n  name: autoscaler\n  namespace: default\n\n---\n\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: autoscaler\nspec:\n  selector:\n    matchLabels:\n      app: autoscaler\n  template:\n    metadata:\n      name: autoscaler\n      labels:\n        app: autoscaler\n    spec:\n      serviceAccountName: autoscaler\n      containers:\n        - name: autoscaler\n          imagePullPolicy: Always\n          image: {{ config['image_autoscaler'] }}\n          args:\n            - \"--in-cluster\"\n            - \"--port=8000\"\n            - \"--prometheus-url=http://prometheus.prometheus:9090\"\n            - \"--namespace=default\"\n          ports:\n            - containerPort: 8000\n          livenessProbe:\n            httpGet:\n              port: 8000\n              path: /healthz\n          readinessProbe:\n            httpGet:\n              port: 8000\n              path: /healthz\n          resources:\n            requests:\n              cpu: 100m\n              memory: 100Mi\n          envFrom:\n          - configMapRef:\n              name: env-vars\n---\n\napiVersion: v1\nkind: Service\nmetadata:\n  name: autoscaler\nspec:\n  type: ClusterIP\n  selector:\n    app: autoscaler\n  ports:\n    - port: 8000\n"
  },
  {
    "path": "manager/manifests/cluster-autoscaler.yaml.j2",
    "content": "# Copyright 2016 The Kubernetes Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n# Source: https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.22.2/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  labels:\n    k8s-addon: cluster-autoscaler.addons.k8s.io\n    k8s-app: cluster-autoscaler\n  name: cluster-autoscaler\n  namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: cluster-autoscaler\n  labels:\n    k8s-addon: cluster-autoscaler.addons.k8s.io\n    k8s-app: cluster-autoscaler\nrules:\n  - apiGroups: [\"\"]\n    resources: [\"events\", \"endpoints\"]\n    verbs: [\"create\", \"patch\"]\n  - apiGroups: [\"\"]\n    resources: [\"pods/eviction\"]\n    verbs: [\"create\"]\n  - apiGroups: [\"\"]\n    resources: [\"pods/status\"]\n    verbs: [\"update\"]\n  - apiGroups: [\"\"]\n    resources: [\"endpoints\"]\n    resourceNames: [\"cluster-autoscaler\"]\n    verbs: [\"get\", \"update\"]\n  - apiGroups: [\"\"]\n    resources: [\"nodes\"]\n    verbs: [\"watch\", \"list\", \"get\", \"update\"]\n  - apiGroups: [\"\"]\n    resources:\n      - \"pods\"\n      - \"services\"\n      - \"replicationcontrollers\"\n      - \"persistentvolumeclaims\"\n      - \"persistentvolumes\"\n    verbs: [\"watch\", \"list\", \"get\"]\n  - apiGroups: [\"extensions\"]\n    resources: [\"replicasets\", \"daemonsets\"]\n    verbs: [\"watch\", \"list\", \"get\"]\n  - apiGroups: [\"policy\"]\n    resources: [\"poddisruptionbudgets\"]\n    verbs: [\"watch\", \"list\"]\n  - apiGroups: [\"apps\"]\n    resources: [\"statefulsets\", \"replicasets\", \"daemonsets\"]\n    verbs: [\"watch\", \"list\", \"get\"]\n  - apiGroups: [\"storage.k8s.io\"]\n    resources: [\"storageclasses\", \"csinodes\"]\n    verbs: [\"watch\", \"list\", \"get\"]\n  - apiGroups: [\"batch\", \"extensions\"]\n    resources: [\"jobs\"]\n    verbs: [\"get\", \"list\", \"watch\", \"patch\"]\n  - apiGroups: [\"coordination.k8s.io\"]\n    resources: [\"leases\"]\n    verbs: [\"create\"]\n  - apiGroups: [\"coordination.k8s.io\"]\n    resourceNames: [\"cluster-autoscaler\"]\n    resources: [\"leases\"]\n    verbs: [\"get\", \"update\"]\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n  name: cluster-autoscaler\n  namespace: kube-system\n  labels:\n    k8s-addon: cluster-autoscaler.addons.k8s.io\n    k8s-app: cluster-autoscaler\nrules:\n  - apiGroups: [\"\"]\n    resources: [\"configmaps\"]\n    verbs: [\"create\",\"list\",\"watch\"]\n  - apiGroups: [\"\"]\n    resources: [\"configmaps\"]\n    resourceNames: [\"cluster-autoscaler-status\", \"cluster-autoscaler-priority-expander\"]\n    verbs: [\"delete\", \"get\", \"update\", \"watch\"]\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: cluster-autoscaler\n  labels:\n    k8s-addon: cluster-autoscaler.addons.k8s.io\n    k8s-app: cluster-autoscaler\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: cluster-autoscaler\nsubjects:\n  - kind: ServiceAccount\n    name: cluster-autoscaler\n    namespace: kube-system\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n  name: cluster-autoscaler\n  namespace: kube-system\n  labels:\n    k8s-addon: cluster-autoscaler.addons.k8s.io\n    k8s-app: cluster-autoscaler\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: Role\n  name: cluster-autoscaler\nsubjects:\n  - kind: ServiceAccount\n    name: cluster-autoscaler\n    namespace: kube-system\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: cluster-autoscaler-priority-expander\n  namespace: kube-system\ndata:\n  priorities: |-\n  {% for p in range(1, 101) %}\n    {% set found = {'priority': False} %}\n    {% for ng in config['node_groups'] %}\n      {% if ng['priority'] == p %}\n        {%- if found.update({'priority':True}) %} {%- endif %}\n      {% endif %}\n    {% endfor %}\n    {% if found['priority'] %}\n    {{ p }}:\n    {% endif %}\n    {% for ng in config['node_groups'] %}\n      {% if ng['priority'] == p %}\n        {% if ng['spot'] %}\n      - .*{{ 'cx-ws-' + ng['name'] }}.*\n        {% else %}\n      - .*{{ 'cx-wd-' + ng['name'] }}.*\n        {% endif %}\n      {% endif %}\n    {% endfor %}\n  {% endfor %}\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: cluster-autoscaler\n  namespace: kube-system\n  labels:\n    app: cluster-autoscaler\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: cluster-autoscaler\n  template:\n    metadata:\n      labels:\n        app: cluster-autoscaler\n    spec:\n      serviceAccountName: cluster-autoscaler\n      priorityClassName: system-cluster-critical\n      containers:\n        - image: {{ config['image_cluster_autoscaler'] }}\n          name: cluster-autoscaler\n          resources:\n            limits:\n              cpu: 300m\n            requests:\n              cpu: 100m\n              memory: 400Mi\n          command:\n            - ./cluster-autoscaler\n            - --v=4\n            - --stderrthreshold=info\n            - --cloud-provider=aws\n            - --skip-nodes-with-local-storage=false\n            - --expander=priority\n            - --max-total-unready-percentage=5\n            - --ok-total-unready-count=30\n            - --max-node-provision-time=8m\n            - --scan-interval=20s\n            - --scale-up-rate-limit-enabled=true\n            - --scale-up-max-number-nodes-per-min=50\n            - --scale-up-burst-number-nodes-per-min=75\n            - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/{{ config['cluster_name'] }}\n          volumeMounts:\n            - name: ssl-certs\n              mountPath: /etc/ssl/certs/ca-certificates.crt #/etc/ssl/certs/ca-bundle.crt for Amazon Linux Worker Nodes\n              readOnly: true\n          imagePullPolicy: \"Always\"\n      volumes:\n        - name: ssl-certs\n          hostPath:\n            path: \"/etc/ssl/certs/ca-bundle.crt\"\n  strategy:\n    type: RollingUpdate\n    rollingUpdate:\n      maxSurge: 0  # necessary because there may not be enough room on the operator node for a rolling update\n"
  },
  {
    "path": "manager/manifests/default_cortex_cli_config.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\ndefault_environment: default\nenvironments:\n  - name: default\n    operator_endpoint: http://operator.default.svc.cluster.local:8888\n"
  },
  {
    "path": "manager/manifests/event-exporter.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  namespace: logging\n  name: event-exporter\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: event-exporter\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: view\nsubjects:\n  - kind: ServiceAccount\n    namespace: logging\n    name: event-exporter\n\n---\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: event-exporter-config\n  namespace: logging\ndata:\n  config.yaml: |\n    logLevel: error\n    logFormat: json\n    route:\n      routes:\n        - match:\n            - receiver: \"stdout\"\n              labels:\n                cortex.dev/api: true\n    receivers:\n      - name: \"stdout\"\n        file:\n          path: \"/dev/stdout\"\n\n---\n\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: event-exporter\n  namespace: logging\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app: event-exporter\n  template:\n    metadata:\n      labels:\n        app: event-exporter\n    spec:\n      serviceAccountName: event-exporter\n      containers:\n        - name: event-exporter\n          image: $CORTEX_IMAGE_EVENT_EXPORTER\n          imagePullPolicy: IfNotPresent\n          args:\n            - -conf=/data/config.yaml\n          volumeMounts:\n            - mountPath: /data\n              name: event-exporter-config\n          resources:\n            requests:\n              cpu: 20m\n              memory: 50Mi\n      volumes:\n        - name: event-exporter-config\n          configMap:\n            name: event-exporter-config\n"
  },
  {
    "path": "manager/manifests/fluent-bit.yaml.j2",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: fluent-bit\n  namespace: logging\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: fluent-bit-read\nrules:\n  - apiGroups: [\"\"]\n    resources:\n      - namespaces\n      - pods\n    verbs: [\"get\", \"list\", \"watch\"]\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: fluent-bit-read\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: fluent-bit-read\nsubjects:\n  - kind: ServiceAccount\n    name: fluent-bit\n    namespace: logging\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: fluent-bit-config\n  namespace: logging\n  labels:\n    k8s-app: fluent-bit\ndata:\n  # Configuration files: server, input, filters and output\n  # ======================================================\n  fluent-bit.conf: |\n    [SERVICE]\n        Flush         1\n        Grace         30\n        Log_Level     info\n        Daemon        off\n        Parsers_File  parsers.conf\n        HTTP_Server   Off\n        Config_Watch  Off\n\n    @INCLUDE input-kubernetes.conf\n    @INCLUDE filter-kubernetes.conf\n    @INCLUDE filter-k8s-events.conf\n    @INCLUDE filter-stackdriver-format.conf\n    @INCLUDE output.conf\n\n  input-kubernetes.conf: |\n    [INPUT]\n        Name              tail\n        Tag               kube.*\n        Path              /var/log/containers/*.log\n        Parser            docker\n        DB                /var/log/flb_kube.db\n        Mem_Buf_Limit     5MB\n        Skip_Long_Lines   On\n        Refresh_Interval  10\n\n  filter-kubernetes.conf: |\n    [FILTER]\n        Name                kubernetes\n        Match               kube.var.log.containers.*\n        Kube_URL            https://kubernetes.default.svc:443\n        Kube_Tag_Prefix     kube.var.log.containers.\n        Merge_Log           On\n\n    # this retagging helps stackdriver and it doesn't matter for cloudwatch\n    # https://docs.fluentbit.io/manual/pipeline/outputs/stackdriver#configuration-file\n    [FILTER]\n        Name rewrite_tag\n        Match kube.var.log.containers.*\n        Rule $log ^(.*)$ k8s_container.$kubernetes['namespace_name'].$kubernetes['pod_name'].$kubernetes['container_name'] false\n\n    [FILTER]\n        Name                modify\n        Match               k8s_container.*\n        Condition           Key_Exists message\n        Hard_rename         message log\n\n    [FILTER]\n        Name                modify\n        Match               k8s_container.*\n        Condition           Key_Exists msg\n        Hard_rename         msg log\n\n    [FILTER]\n        Name                nest\n        Match               k8s_container.*\n        Operation           lift\n        Nested_under        kubernetes\n        Add_prefix          k8s.\n\n    [FILTER]\n        Name                modify\n        Match               k8s_container.*\n        Condition           Key_Does_Not_Exist cortex.labels\n        Rename              k8s.labels cortex.labels\n\n    [FILTER]\n        Name                modify\n        Match               k8s_container.*\n        Remove_wildcard     k8s.\n\n  filter-k8s-events.conf: |\n    [FILTER]\n        Name                nest\n        Match               k8s_container.*.event-exporter-*\n        Operation           lift\n        Nested_under        involvedObject\n        Add_prefix          involvedObject.\n\n    [FILTER]\n        Name                modify\n        Match               k8s_container.*.event-exporter-*\n        Condition           Key_exists labels\n        Rename              labels k8s.labels\n\n    [FILTER]\n        Name                modify\n        Match               k8s_container.*.event-exporter-*\n        Condition           Key_exists involvedObject.labels\n        Hard_copy           involvedObject.labels cortex.labels\n\n    [FILTER]\n        Name                nest\n        Match               k8s_container.*.event-exporter-*\n        Operation           nest\n        Wildcard            involvedObject.*\n        Nest_under          involvedObject\n        Remove_prefix       involvedObject.\n\n  filter-stackdriver-format.conf: |\n    [FILTER]\n        Name                modify\n        Match               k8s_container.*\n        Condition           Key_exists log\n        Rename              log message\n\n    [FILTER]\n        Name                modify\n        Match               k8s_container.*\n        Condition           Key_exists levelname\n        Rename              levelname level\n\n  output.conf: |\n    [OUTPUT]\n        Name              cloudwatch\n        Match             k8s_container.*\n        region            {{ config[\"region\"] }}\n        log_group_name    {{ config[\"cluster_name\"] }}\n        log_stream_prefix kube.\n        auto_create_group true\n\n  parsers.conf: |\n    [PARSER]\n        Name        docker\n        Format      json\n        Time_Key    time\n        Time_Format %Y-%m-%dT%H:%M:%S.%L\n        Time_Keep   On\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: fluent-bit\n  namespace: logging\nspec:\n  selector:\n    matchLabels:\n      k8s-app: fluent-bit-logging\n  template:\n    metadata:\n      labels:\n        app: fluent-bit\n        k8s-app: fluent-bit-logging\n        version: v1\n        kubernetes.io/cluster-service: \"true\"\n    spec:\n      containers:\n        - name: fluent-bit\n          image: {{ config[\"image_fluent_bit\"] }}\n          imagePullPolicy: Always\n          resources:\n            requests:\n              cpu: 100m\n              memory: 150Mi\n            limits:\n              cpu: 150m\n              memory: 150Mi\n          ports:\n            - containerPort: 2020\n          volumeMounts:\n            - name: varlog\n              mountPath: /var/log\n            - name: varlibdockercontainers\n              mountPath: /var/lib/docker/containers\n              readOnly: true\n            - name: fluent-bit-config\n              mountPath: /fluent-bit/etc/\n      terminationGracePeriodSeconds: 60\n      volumes:\n        - name: varlog\n          hostPath:\n            path: /var/log\n        - name: varlibdockercontainers\n          hostPath:\n            path: /var/lib/docker/containers\n        - name: fluent-bit-config\n          configMap:\n            name: fluent-bit-config\n      serviceAccountName: fluent-bit\n      tolerations:\n        - key: node-role.kubernetes.io/master\n          operator: Exists\n          effect: NoSchedule\n        - operator: \"Exists\"\n          effect: \"NoExecute\"\n        - operator: \"Exists\"\n          effect: \"NoSchedule\"\n        - key: aws.amazon.com/neuron\n          operator: Exists\n          effect: NoSchedule\n        - key: nvidia.com/gpu\n          operator: Exists\n          effect: NoSchedule\n        - key: workload\n          operator: Exists\n          effect: NoSchedule\n        - key: prometheus\n          operator: Exists\n          effect: NoSchedule\n"
  },
  {
    "path": "manager/manifests/grafana/grafana-dashboard-async.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-dashboard-async\n  namespace: default\ndata:\n  async.json: |-\n    {\n      \"annotations\": {\n        \"list\": [\n          {\n            \"builtIn\": 1,\n            \"datasource\": \"prometheus\",\n            \"enable\": true,\n            \"hide\": true,\n            \"iconColor\": \"rgba(0, 211, 255, 1)\",\n            \"name\": \"Annotations & Alerts\",\n            \"type\": \"dashboard\"\n          }\n        ]\n      },\n      \"editable\": true,\n      \"gnetId\": null,\n      \"graphTooltip\": 0,\n      \"iteration\": 1625805144458,\n      \"links\": [],\n      \"panels\": [\n        {\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 2,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"id\": 15,\n          \"options\": {\n            \"content\": \"<h1 style=\\\"text-align: center\\\">AsyncAPI</h1>\",\n            \"mode\": \"markdown\"\n          },\n          \"pluginVersion\": \"8.0.4\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"transparent\": true,\n          \"type\": \"text\"\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 22,\n          \"panels\": [],\n          \"title\": \"API Stats\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Request rate, computed over every minute, of an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(cortex_async_request_count{api_name=\\\"$api_name\\\"}[1m])) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"Request Rate\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request Rate\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"reqps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"In-flight requests for an API.\\n\\nNote: In-flight requests are recorded every 10 seconds, which will correspond to the minimum resolution.\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 4,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(cortex_async_active{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\"}) by (api_name)\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"active\",\n              \"refId\": \"Active\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(cortex_async_queued{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\"}) by (api_name)\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"queued\",\n              \"refId\": \"Queued\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(cortex_async_in_flight{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\"}) by (api_name)\",\n              \"hide\": true,\n              \"interval\": \"\",\n              \"legendFormat\": \"in flight\",\n              \"refId\": \"In Flight\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"In-Flight Requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Request rate, computed over every minute, for responses with status code 2XX of an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"hiddenSeries\": false,\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(cortex_async_request_count{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\",status_code=~\\\"2.+\\\"}[1m])) by (api_name, status_code)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"2XX\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"2XX Responses\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"reqps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"hiddenSeries\": false,\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(kube_deployment_status_replicas_available{deployment=\\\"api-$api_name\\\"}) by (deployment)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{deployment}}\",\n              \"refId\": \"Active Replicas\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Active Replicas\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Request rate, computed over every minute, for responses with status code 4XX of an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"hiddenSeries\": false,\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(cortex_async_request_count{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\",status_code=~\\\"4.+\\\"}[1m])) by (api_name, status_code)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"4XX\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"4XX Responses\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"reqps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Request rate, computed over every minute, for responses with status code 5XX of an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"hiddenSeries\": false,\n          \"id\": 10,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(cortex_async_request_count{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\",status_code=~\\\"5.+\\\"}[1m])) by (api_name, status_code)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"5XX\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"5XX Responses\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"reqps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"99th percentile latency, computed over a minute, for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"hiddenSeries\": false,\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum by (api_name, le) (rate(cortex_async_latency_bucket{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\"}[1m])))\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"p99 Latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"90th percentile latency, computed over a minute, for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"hiddenSeries\": false,\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.90, sum by (api_name, le) (rate(cortex_async_latency_bucket{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\"}[1m])))\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"p90 Latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"50th percentile latency, computed over a minute, for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 39\n          },\n          \"hiddenSeries\": false,\n          \"id\": 16,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.50, sum by (api_name, le) (rate(cortex_async_latency_bucket{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\"}[1m])))\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"p50 Latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Average latency, computed over a minute, for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 39\n          },\n          \"hiddenSeries\": false,\n          \"id\": 12,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(cortex_async_latency_sum{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\"}[1m])) by (api_name) / \\nsum(rate(cortex_async_latency_count{api_kind=\\\"AsyncAPI\\\",api_name=\\\"$api_name\\\"}[1m])) by (api_name)\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Average Latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 48\n          },\n          \"id\": 20,\n          \"panels\": [],\n          \"title\": \"Aggregate Usage\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Total CPU Request\": \"semi-dark-orange\",\n            \"Total CPU Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total CPU usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 49\n          },\n          \"hiddenSeries\": false,\n          \"id\": 24,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(rate(container_cpu_usage_seconds_total{pod=~\\\"api-$api_name.+\\\", container!=\\\"POD\\\", name!=\\\"\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total CPU Usage\",\n              \"refId\": \"CPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"api-$api_name.+\\\", resource=\\\"cpu\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total CPU Request\",\n              \"refId\": \"CPU Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"core\",\n              \"label\": \"cpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total Memory Request\": \"semi-dark-orange\",\n            \"Total Memory Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total memory usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 49\n          },\n          \"hiddenSeries\": false,\n          \"id\": 26,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(sum_over_time(container_memory_working_set_bytes{pod=~\\\"api-$api_name.+\\\", name!=\\\"\\\", container!=\\\"POD\\\"}[1m])) /\\navg(count_over_time(container_memory_working_set_bytes{pod=~\\\"api-$api_name.+\\\", name!=\\\"\\\", container=\\\"api\\\"}[1m])) / 1024^2\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Memory Usage\",\n              \"refId\": \"Memory Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"api-$api_name.+\\\", resource=\\\"memory\\\"}) / 1024^2\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Memory Request\",\n              \"refId\": \"Memory Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total GPU Capacity\": \"semi-dark-orange\",\n            \"Total GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Utilization\": \"light-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total GPU core usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 57\n          },\n          \"hiddenSeries\": false,\n          \"id\": 28,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"}) / 100\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total GPU Usage\",\n              \"refId\": \"GPU Usage\"\n            },\n            {\n              \"expr\": \"count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total GPU Capacity\",\n              \"refId\": \"GPU Capacity\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total GPU Core Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"gpuCore\",\n              \"label\": \"gpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Total Used GPU Memory\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total GPU memory usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 57\n          },\n          \"hiddenSeries\": false,\n          \"id\": 29,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Used GPU Memory\",\n              \"refId\": \"GPU Used Memory\"\n            },\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\\\"api-$api_name.+\\\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Capacity GPU Memory\",\n              \"refId\": \"GPU Capacity Memory\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total GPU Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 65\n          },\n          \"id\": 18,\n          \"panels\": [],\n          \"title\": \"Average Replica Usage\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Avg CPU Request\": \"semi-dark-orange\",\n            \"Avg CPU Usage\": \"semi-dark-green\",\n            \"Total CPU Request\": \"semi-dark-orange\",\n            \"Total CPU Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg CPU usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 66\n          },\n          \"hiddenSeries\": false,\n          \"id\": 30,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(rate(container_cpu_usage_seconds_total{pod=~\\\"api-$api_name.+\\\", container!=\\\"POD\\\", name!=\\\"\\\"}[1m]))\\n/\\nsum(kube_pod_info{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg CPU Usage\",\n              \"refId\": \"CPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"api-$api_name.+\\\", resource=\\\"cpu\\\"})\\n/\\nsum(kube_pod_info{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg CPU Request\",\n              \"refId\": \"CPU Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"core\",\n              \"label\": \"cpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg Memory Request\": \"semi-dark-orange\",\n            \"Avg Memory Usage\": \"semi-dark-green\",\n            \"Total Memory Request\": \"semi-dark-orange\",\n            \"Total Memory Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg memory usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 66\n          },\n          \"hiddenSeries\": false,\n          \"id\": 31,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(sum_over_time(container_memory_working_set_bytes{pod=~\\\"api-$api_name.+\\\", name!=\\\"\\\", container!=\\\"POD\\\"}[1m]))\\n/\\navg(count_over_time(container_memory_working_set_bytes{pod=~\\\"api-$api_name.+\\\", name!=\\\"\\\", container=\\\"api\\\"}[1m])) / 1024^2\\n/\\nsum(kube_pod_info{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Memory Usage\",\n              \"refId\": \"Memory Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"api-$api_name.+\\\", resource=\\\"memory\\\"}) / 1024^2\\n/\\nsum(kube_pod_info{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Memory Request\",\n              \"refId\": \"Memory Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg GPU Capacity\": \"semi-dark-orange\",\n            \"Avg GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Capacity\": \"semi-dark-orange\",\n            \"Total GPU Utilization\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg GPU core usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 74\n          },\n          \"hiddenSeries\": false,\n          \"id\": 32,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"}) / 100\\n/\\ncount(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg GPU Usage\",\n              \"refId\": \"GPU Usage\"\n            },\n            {\n              \"expr\": \"count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"})\\n/\\ncount(count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"}) by (exported_pod))\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg GPU Capacity\",\n              \"refId\": \"GPU Capacity\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg GPU Core Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"gpuCore\",\n              \"label\": \"gpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Avg Used GPU Memory\": \"semi-dark-green\",\n            \"Total Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Total Used GPU Memory\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg GPU memory usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 74\n          },\n          \"hiddenSeries\": false,\n          \"id\": 33,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\\n/\\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Used GPU Memory\",\n              \"refId\": \"GPU Used Memory\"\n            },\n            {\n              \"exemplar\": false,\n              \"expr\": \"(sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\\\"api-$api_name.+\\\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"}))\\n/\\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Capacity GPU Memory\",\n              \"refId\": \"GPU Capacity Memory\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg GPU Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"refresh\": \"30s\",\n      \"schemaVersion\": 30,\n      \"style\": \"dark\",\n      \"tags\": [],\n      \"templating\": {\n        \"list\": [\n          {\n            \"allValue\": null,\n            \"current\": {\n              \"selected\": false,\n              \"text\": \"None\",\n              \"value\": \"None\"\n            },\n            \"datasource\": null,\n            \"definition\": \"label_values(cortex_async_queue_length{api_kind=\\\"AsyncAPI\\\"}, api_name)\",\n            \"description\": null,\n            \"error\": null,\n            \"hide\": 0,\n            \"includeAll\": false,\n            \"label\": \"API Name\",\n            \"multi\": true,\n            \"name\": \"api_name\",\n            \"options\": [],\n            \"query\": {\n              \"query\": \"label_values(cortex_async_queue_length{api_kind=\\\"AsyncAPI\\\"}, api_name)\",\n              \"refId\": \"StandardVariableQuery\"\n            },\n            \"refresh\": 1,\n            \"regex\": \"\",\n            \"skipUrlSync\": false,\n            \"sort\": 1,\n            \"tagValuesQuery\": \"\",\n            \"tagsQuery\": \"\",\n            \"type\": \"query\",\n            \"useTags\": false\n          }\n        ]\n      },\n      \"time\": {\n        \"from\": \"now-1h\",\n        \"to\": \"now\"\n      },\n      \"timepicker\": {},\n      \"timezone\": \"\",\n      \"title\": \"AsyncAPI\",\n      \"uid\": \"asyncapi\",\n      \"version\": 1\n    }\n"
  },
  {
    "path": "manager/manifests/grafana/grafana-dashboard-batch.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-dashboard-batch\n  namespace: default\ndata:\n  batch.json: |-\n    {\n      \"annotations\": {\n        \"list\": [\n          {\n            \"builtIn\": 1,\n            \"datasource\": \"prometheus\",\n            \"enable\": true,\n            \"hide\": true,\n            \"iconColor\": \"rgba(0, 211, 255, 1)\",\n            \"name\": \"Annotations & Alerts\",\n            \"type\": \"dashboard\"\n          }\n        ]\n      },\n      \"editable\": true,\n      \"gnetId\": null,\n      \"graphTooltip\": 0,\n      \"iteration\": 1625169092506,\n      \"links\": [],\n      \"panels\": [\n        {\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 2,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"id\": 7,\n          \"options\": {\n            \"content\": \"<h1 style=\\\"text-align: center\\\">BatchAPI</h1>\\n\",\n            \"mode\": \"markdown\"\n          },\n          \"pluginVersion\": \"8.0.4\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"transparent\": true,\n          \"type\": \"text\"\n        },\n        {\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {},\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 22,\n          \"title\": \"API Stats\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Number of succeeded batches for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(cortex_batch_succeeded{api_name=~\\\"$api_name\\\"}) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"# Succeeded Batches\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:26\",\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:27\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Number of failed batches for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(cortex_batch_failed{api_name=~\\\"$api_name\\\"}) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"# Failed Batches\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:262\",\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:263\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Average time per batch for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"hiddenSeries\": false,\n          \"id\": 5,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(cortex_time_per_batch_sum{api_name=~\\\"$api_name\\\"}) by (api_name) / sum(cortex_time_per_batch_count{api_name=~\\\"$api_name\\\"}) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Average Time per Batch\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Active Jobs\": \"semi-dark-green\",\n            \"Active Workers\": \"semi-dark-orange\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Active jobs/workers\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"hiddenSeries\": false,\n          \"id\": 20,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"count(kube_job_status_active{job_name=~\\\"$api_name.+\\\"} != 0)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"Active Jobs\",\n              \"refId\": \"Active Batches\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_job_status_active{job_name=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Active Workers\",\n              \"refId\": \"Active Workers\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"# Active Jobs/Workers\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:128\",\n              \"decimals\": 0,\n              \"format\": \"count\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:129\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {},\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"id\": 11,\n          \"title\": \"Aggregate Worker Usage\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Total CPU Request\": \"semi-dark-orange\",\n            \"Total CPU Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total CPU usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"hiddenSeries\": false,\n          \"id\": 13,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(rate(container_cpu_usage_seconds_total{pod=~\\\"$api_name.+\\\", container!=\\\"POD\\\", name!=\\\"\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total CPU Usage\",\n              \"refId\": \"CPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"$api_name.+\\\", resource=\\\"cpu\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total CPU Request\",\n              \"refId\": \"CPU Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"core\",\n              \"label\": \"cpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total Memory Request\": \"semi-dark-orange\",\n            \"Total Memory Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total memory usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"hiddenSeries\": false,\n          \"id\": 15,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(sum_over_time(container_memory_working_set_bytes{pod=~\\\"$api_name.+\\\", name!=\\\"\\\", container!=\\\"POD\\\"}[1m]))\\n/\\navg(count_over_time(container_memory_working_set_bytes{pod=~\\\"$api_name.+\\\", name!=\\\"\\\", container=\\\"api\\\"}[1m])) / 1024^2\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Memory Usage\",\n              \"refId\": \"Memory Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"$api_name.+\\\", resource=\\\"memory\\\"}) / 1024^2\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Memory Request\",\n              \"refId\": \"Memory Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total GPU Capacity\": \"semi-dark-orange\",\n            \"Total GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Utilization\": \"light-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total GPU core usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 29\n          },\n          \"hiddenSeries\": false,\n          \"id\": 17,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"}) / 100\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total GPU Usage\",\n              \"refId\": \"GPU Usage\"\n            },\n            {\n              \"expr\": \"count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total GPU Capacity\",\n              \"refId\": \"GPU Capacity\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total GPU Core Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"gpuCore\",\n              \"label\": \"gpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Total Used GPU Memory\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total GPU memory usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 29\n          },\n          \"hiddenSeries\": false,\n          \"id\": 19,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Used GPU Memory\",\n              \"refId\": \"GPU Used Memory\"\n            },\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\\\"$api_name.+\\\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Capacity GPU Memory\",\n              \"refId\": \"GPU Capacity Memory\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total GPU Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {},\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 37\n          },\n          \"id\": 9,\n          \"panels\": [],\n          \"title\": \"Avg Worker Usage\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Avg CPU Request\": \"semi-dark-orange\",\n            \"Avg CPU Usage\": \"semi-dark-green\",\n            \"Total CPU Request\": \"semi-dark-orange\",\n            \"Total CPU Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg CPU usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 38\n          },\n          \"hiddenSeries\": false,\n          \"id\": 23,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(rate(container_cpu_usage_seconds_total{pod=~\\\"$api_name.+\\\", container!=\\\"POD\\\", name!=\\\"\\\"}[1m]))\\n/\\nsum(kube_pod_info{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg CPU Usage\",\n              \"refId\": \"CPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"$api_name.+\\\", resource=\\\"cpu\\\"})\\n/\\nsum(kube_pod_info{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg CPU Request\",\n              \"refId\": \"CPU Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"core\",\n              \"label\": \"cpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg Memory Request\": \"semi-dark-orange\",\n            \"Avg Memory Usage\": \"semi-dark-green\",\n            \"Total Memory Request\": \"semi-dark-orange\",\n            \"Total Memory Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg memory usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 38\n          },\n          \"hiddenSeries\": false,\n          \"id\": 24,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(sum_over_time(container_memory_working_set_bytes{pod=~\\\"$api_name.+\\\", name!=\\\"\\\", container!=\\\"POD\\\"}[1m]))\\n/\\navg(count_over_time(container_memory_working_set_bytes{pod=~\\\"$api_name.+\\\", name!=\\\"\\\", container=\\\"api\\\"}[1m])) / 1024^2\\n/\\nsum(kube_pod_info{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Memory Usage\",\n              \"refId\": \"Memory Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource{exported_pod=~\\\"$api_name.+\\\", resource=\\\"memory\\\"}) / 1024^2\\n/\\nsum(kube_pod_info{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Memory Request\",\n              \"refId\": \"Memory Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg GPU Capacity\": \"semi-dark-orange\",\n            \"Avg GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Capacity\": \"semi-dark-orange\",\n            \"Total GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Utilization\": \"light-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg GPU core usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 46\n          },\n          \"hiddenSeries\": false,\n          \"id\": 25,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"}) / 100\\n/\\ncount(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg GPU Usage\",\n              \"refId\": \"GPU Usage\"\n            },\n            {\n              \"expr\": \"count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"})\\n/\\ncount(count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"}) by (exported_pod))\",\n              \"hide\": false,\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg GPU Capacity\",\n              \"refId\": \"GPU Capacity\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg GPU Core Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"gpuCore\",\n              \"label\": \"gpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Avg Used GPU Memory\": \"semi-dark-green\",\n            \"Total Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Total Used GPU Memory\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg GPU memory usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 46\n          },\n          \"hiddenSeries\": false,\n          \"id\": 26,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\\n/\\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Used GPU Memory\",\n              \"refId\": \"GPU Used Memory\"\n            },\n            {\n              \"exemplar\": false,\n              \"expr\": \"(sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\\\"$api_name.+\\\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"}))\\n/\\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Capacity GPU Memory\",\n              \"refId\": \"GPU Capacity Memory\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg GPU Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"refresh\": \"30s\",\n      \"schemaVersion\": 30,\n      \"style\": \"dark\",\n      \"tags\": [],\n      \"templating\": {\n        \"list\": [\n          {\n            \"allValue\": null,\n            \"current\": {\n              \"selected\": false,\n              \"text\": \"None\",\n              \"value\": \"None\"\n            },\n            \"datasource\": null,\n            \"definition\": \"label_values({__name__=~\\\"cortex_batch_.+\\\"}, api_name)\",\n            \"description\": null,\n            \"error\": null,\n            \"hide\": 0,\n            \"includeAll\": false,\n            \"label\": \"API Name\",\n            \"multi\": true,\n            \"name\": \"api_name\",\n            \"options\": [],\n            \"query\": {\n              \"query\": \"label_values({__name__=~\\\"cortex_batch_.+\\\"}, api_name)\",\n              \"refId\": \"StandardVariableQuery\"\n            },\n            \"refresh\": 2,\n            \"regex\": \"\",\n            \"skipUrlSync\": false,\n            \"sort\": 0,\n            \"tagValuesQuery\": \"\",\n            \"tagsQuery\": \"\",\n            \"type\": \"query\",\n            \"useTags\": false\n          }\n        ]\n      },\n      \"time\": {\n        \"from\": \"now-1h\",\n        \"to\": \"now\"\n      },\n      \"timepicker\": {},\n      \"timezone\": \"\",\n      \"title\": \"BatchAPI\",\n      \"uid\": \"batchapi\",\n      \"version\": 1\n    }\n"
  },
  {
    "path": "manager/manifests/grafana/grafana-dashboard-cluster.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-dashboard-cluster\n  namespace: default\ndata:\n  cluster.json: |\n    {\n      \"annotations\": {\n        \"list\": [\n          {\n            \"builtIn\": 1,\n            \"datasource\": \"prometheus\",\n            \"enable\": true,\n            \"hide\": true,\n            \"iconColor\": \"rgba(0, 211, 255, 1)\",\n            \"name\": \"Annotations & Alerts\",\n            \"type\": \"dashboard\"\n          }\n        ]\n      },\n      \"editable\": true,\n      \"gnetId\": null,\n      \"graphTooltip\": 0,\n      \"links\": [],\n      \"panels\": [\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"id\": 23,\n          \"panels\": [],\n          \"title\": \"Size\",\n          \"type\": \"row\"\n        },\n        {\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 2,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 16,\n          \"options\": {\n            \"content\": \"<h1 style=\\\"text-align: center\\\">Cluster</h1>\\n\",\n            \"mode\": \"html\"\n          },\n          \"pluginVersion\": \"8.0.4\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"transparent\": true,\n          \"type\": \"text\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 20,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"count(instance:node_cpu_utilisation:rate1m)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Total Nodes\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Nodes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:117\",\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:118\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 21,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_info{exported_pod!~\\\"(aws-node|grafana|autoscaler|cluster-autoscaler|coredns|event-exporter|fluent-bit|kube-proxy|k8s-neuron-scheduler|kube-state-metrics|metrics-server|node-exporter|operator|operator-controller-manager|prometheus-operator|prometheus-prometheus|prometheus-statsd-exporter|dcgm-exporter|ingressgateway|istiod|activator|enqueuer|gateway|nvidia-device-plugin-daemonset|neuron-device-plugin-daemonset|async-gateway)-(.+)\\\"})\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"User Workload Pods\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Pods\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:142\",\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:143\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 18,\n          \"panels\": [],\n          \"title\": \"Costs\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Cortex System Costs\": \"light-red\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total cluster costs include the EKS control plane, the system/user nodes, and the EBS volumes.\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"unit\": \"currencyUSD\"\n            },\n            \"overrides\": []\n          },\n          \"fill\": 0,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"hiddenSeries\": false,\n          \"id\": 25,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cortex_cluster_cost{api=\\\"false\\\", kind=\\\"false\\\", component=\\\"cluster-costs\\\"}\",\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Cluster Costs\",\n              \"refId\": \"Total Cluster Costs\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"cortex_cluster_cost{api=\\\"false\\\", kind=\\\"false\\\", component=\\\"cortex-system-costs\\\"}\",\n              \"hide\": false,\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Cortex System Costs\",\n              \"refId\": \"Cortex System Costs\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Cluster Costs / hr\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:165\",\n              \"format\": \"currencyUSD\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:166\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Cortex System Costs\": \"light-red\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Compute formula: for each \\\"API kind\\\" pod, see how many replicas of a given pod can fit on a specific node and then take the total cost of running the node (which can be on-demand or spot) and divide it by the maximum number of pods that can be run. Then, add the cost of each pod to an accumulator for each \\\"API kind\\\". Finally, once the cost for every \\\"API kind\\\" is computed, renormalize the sum of all API kinds to the actual cost of the cluster (minus the cortex system costs) so that all API kinds summed will give you the cost of the cluster.\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"unit\": \"currencyUSD\"\n            },\n            \"overrides\": []\n          },\n          \"fill\": 0,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"hiddenSeries\": false,\n          \"id\": 28,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum by (component) (cortex_cluster_cost{api=\\\"false\\\", kind=\\\"true\\\"})\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{component}}\",\n              \"refId\": \"Workload Costs by API Kind\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Workload Costs by API Kind / hr\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:165\",\n              \"format\": \"currencyUSD\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:166\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Cortex System Costs\": \"light-red\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Compute formula: for each \\\"API name\\\" pod, see how many replicas of a given pod can fit on a specific node and then take the total cost of running the node (which can be on-demand or spot) and divide it by the maximum number of pods that can be run. Then, add the cost of each pod to an accumulator for each \\\"API name\\\". Finally, once the cost for every \\\"API name\\\" is computed, renormalize the sum of all API names to the actual cost of the cluster (minus the cortex system costs) so that all API names summed will give you the cost of the cluster.\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"unit\": \"currencyUSD\"\n            },\n            \"overrides\": []\n          },\n          \"fill\": 2,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 19\n          },\n          \"hiddenSeries\": false,\n          \"id\": 26,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum by (component) (cortex_cluster_cost{api=\\\"true\\\", kind=\\\"false\\\"})\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{component}}\",\n              \"refId\": \"Workload Costs by API Name\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Workload Costs by API Name / hr\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:165\",\n              \"format\": \"currencyUSD\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:166\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 10,\n          \"panels\": [],\n          \"repeat\": null,\n          \"title\": \"CPU\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 28\n          },\n          \"hiddenSeries\": false,\n          \"id\": 1,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:cpu_utilization:ratio{}\\n\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU Utilisation\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:156\",\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": 1,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:157\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 28\n          },\n          \"hiddenSeries\": false,\n          \"id\": 2,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:load1:ratio{}\\n\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU Saturation (load1 per CPU)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:181\",\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": 1,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:182\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 35\n          },\n          \"id\": 11,\n          \"panels\": [],\n          \"repeat\": null,\n          \"title\": \"Memory\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 36\n          },\n          \"hiddenSeries\": false,\n          \"id\": 3,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:memory_utilization:ratio{}\\n\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory Utilisation\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:210\",\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": 1,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:211\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 36\n          },\n          \"hiddenSeries\": false,\n          \"id\": 4,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:vmstat_pgmajfault:rate1m{}\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory Saturation (Major Page Faults)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:398\",\n              \"format\": \"rps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:399\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 43\n          },\n          \"id\": 12,\n          \"panels\": [],\n          \"repeat\": null,\n          \"title\": \"Network\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 44\n          },\n          \"hiddenSeries\": false,\n          \"id\": 5,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"/ Receive/\",\n              \"stack\": \"A\"\n            },\n            {\n              \"alias\": \"/ Transmit/\",\n              \"stack\": \"B\",\n              \"transform\": \"negative-Y\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:network_receive_bytes_excluding_low:rate1m{}\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{some-non-existent-label}} Receive\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:network_transmit_bytes_excluding_lo:rate1m{}\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{some-non-existent-label}} Transmit\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Net Utilisation (Bytes Receive/Transmit)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:235\",\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:236\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 44\n          },\n          \"hiddenSeries\": false,\n          \"id\": 6,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"/ Receive/\",\n              \"stack\": \"A\"\n            },\n            {\n              \"alias\": \"/ Transmit/\",\n              \"stack\": \"B\",\n              \"transform\": \"negative-Y\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:network_receive_drop_excluding_lo:rate1m{}\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{some-non-existent-label}} Receive\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:network_transmit_drop_excluding_lo:rate1m{}\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{some-non-existent-label}} Transmit\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Net Saturation (Drops Receive/Transmit)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:521\",\n              \"format\": \"rps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:522\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 51\n          },\n          \"id\": 13,\n          \"panels\": [],\n          \"repeat\": null,\n          \"title\": \"Disk IO\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 52\n          },\n          \"hiddenSeries\": false,\n          \"id\": 7,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:disk_io_utilization:ratio{}\\n\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk IO Utilisation\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:446\",\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": 1,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:447\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 52\n          },\n          \"hiddenSeries\": false,\n          \"id\": 8,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:disk_io_saturation:ratio{}\\n\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk IO Saturation\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:471\",\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": 1,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:472\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 59\n          },\n          \"id\": 14,\n          \"panels\": [],\n          \"repeat\": null,\n          \"title\": \"Disk Space\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 10,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 60\n          },\n          \"hiddenSeries\": false,\n          \"id\": 9,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"cluster:disk_space_utilization:ratio{}\\n\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"legendLink\": \"/dashboard/file/node-rsrc-use.json\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Space Utilisation\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:496\",\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": 1,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:497\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"refresh\": \"1m\",\n      \"schemaVersion\": 30,\n      \"style\": \"dark\",\n      \"tags\": [],\n      \"templating\": {\n        \"list\": []\n      },\n      \"time\": {\n        \"from\": \"now-12h\",\n        \"to\": \"now\"\n      },\n      \"timepicker\": {\n        \"refresh_intervals\": [\n          \"5s\",\n          \"10s\",\n          \"30s\",\n          \"1m\",\n          \"5m\",\n          \"15m\",\n          \"30m\",\n          \"1h\",\n          \"2h\",\n          \"1d\"\n        ],\n        \"time_options\": [\n          \"5m\",\n          \"15m\",\n          \"1h\",\n          \"6h\",\n          \"12h\",\n          \"24h\",\n          \"2d\",\n          \"7d\",\n          \"30d\"\n        ]\n      },\n      \"timezone\": \"\",\n      \"title\": \"Cluster\",\n      \"uid\": \"cluster\",\n      \"version\": 4\n    }\n"
  },
  {
    "path": "manager/manifests/grafana/grafana-dashboard-control-plane.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-dashboard-control-plane\n  namespace: default\ndata:\n  control-plane.json: |\n    {\n      \"annotations\": {\n        \"list\": [\n          {\n            \"builtIn\": 1,\n            \"enable\": true,\n            \"hide\": true,\n            \"iconColor\": \"rgba(0, 211, 255, 1)\",\n            \"name\": \"Annotations & Alerts\",\n            \"type\": \"dashboard\"\n          }\n        ]\n      },\n      \"description\": \"Metrics for API Server, Controller and Scheduler.\",\n      \"editable\": true,\n      \"gnetId\": 10907,\n      \"graphTooltip\": 0,\n      \"links\": [],\n      \"panels\": [\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"id\": 18,\n          \"panels\": [],\n          \"title\": \"API Server Summary\",\n          \"type\": \"row\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"color\": {\n                \"mode\": \"thresholds\"\n              },\n              \"decimals\": null,\n              \"mappings\": [],\n              \"max\": 100,\n              \"min\": 0,\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  },\n                  {\n                    \"color\": \"red\",\n                    \"value\": 80\n                  }\n                ]\n              },\n              \"unit\": \"none\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 3,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 6,\n          \"links\": [],\n          \"options\": {\n            \"orientation\": \"auto\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"last\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\n            \"showThresholdLabels\": false,\n            \"showThresholdMarkers\": true,\n            \"text\": {}\n          },\n          \"pluginVersion\": \"8.0.4\",\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(apiserver_request_total[5m]))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{verb}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"API Server RPS\",\n          \"type\": \"gauge\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 5,\n            \"x\": 3,\n            \"y\": 1\n          },\n          \"hiddenSeries\": false,\n          \"id\": 47,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(apiserver_current_inflight_requests) by (requestKind)\",\n              \"legendFormat\": \"{{requestKind}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Current Inflight Requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 5,\n            \"x\": 8,\n            \"y\": 1\n          },\n          \"hiddenSeries\": false,\n          \"id\": 32,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": false,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\\\"WATCH|CONNECT\\\"}[5m])) by (le) )\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Latency - p99\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:45\",\n              \"decimals\": 2,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:46\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"decimals\": 2,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 5,\n            \"x\": 13,\n            \"y\": 1\n          },\n          \"hiddenSeries\": false,\n          \"id\": 21,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(apiserver_request_duration_seconds_sum{verb!~\\\"WATCH|CONNECT\\\"}[5m])) \",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"seconds\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"API Server Response Duration (Seconds)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:101\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:102\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"decimals\": 0,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 5,\n            \"x\": 18,\n            \"y\": 1\n          },\n          \"hiddenSeries\": false,\n          \"id\": 15,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(apiserver_request_duration_seconds_bucket{code=~\\\"^(?:4..)$|^(?:5..)$\\\"}[5m])) by (code)>0\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{code}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"API Server Errors\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:157\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:158\",\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 24,\n          \"panels\": [],\n          \"title\": \"API Server Details\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"decimals\": 0,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 23,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"hiddenSeries\": false,\n          \"id\": 12,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(apiserver_request_total[5m])) by (verb)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{verb}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"API Server Requests (by Verb)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 23,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"hiddenSeries\": false,\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(apiserver_request_total[5m])) by (resource)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{resource}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"API Requests (by Resource)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"decimals\": 0,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 23,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"hiddenSeries\": false,\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(apiserver_request_total{client!~\\\"kubelet.*|kube-scheduler.*|kube-controller.*|kube-apiserver.*|kube-proxy.*\\\"}[5m])) by (client)>0\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{client}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"API Requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 23,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"hiddenSeries\": false,\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\\\"WATCH|CONNECT\\\"}[5m])) by (le, resource) )\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{resource}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Latency by resource\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:252\",\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:253\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 23,\n            \"x\": 0,\n            \"y\": 38\n          },\n          \"hiddenSeries\": false,\n          \"id\": 49,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"histogram_quantile(0.90, sum(rate(apiserver_request_duration_seconds_bucket{verb!~\\\"WATCH|CONNECT\\\"}[5m])) by (le, verb) )\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{resource}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Latency by verb\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:194\",\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:195\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"decimals\": 0,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"links\": []\n            },\n            \"overrides\": []\n          },\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 23,\n            \"x\": 0,\n            \"y\": 46\n          },\n          \"hiddenSeries\": false,\n          \"id\": 10,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"aggregation\": \"Last\",\n              \"decimals\": 2,\n              \"displayAliasType\": \"Warning / Critical\",\n              \"displayType\": \"Regular\",\n              \"displayValueWithAlias\": \"Never\",\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(apiserver_request_total[5m])) by (instance)>0\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} \",\n              \"refId\": \"A\",\n              \"units\": \"none\",\n              \"valueHandler\": \"Number Threshold\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"API Server Requests by server\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:223\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:224\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"refresh\": \"30s\",\n      \"schemaVersion\": 30,\n      \"style\": \"dark\",\n      \"tags\": [],\n      \"templating\": {\n        \"list\": []\n      },\n      \"time\": {\n        \"from\": \"now-1h\",\n        \"to\": \"now\"\n      },\n      \"timepicker\": {\n        \"refresh_intervals\": [\n          \"5s\",\n          \"10s\",\n          \"30s\",\n          \"1m\",\n          \"5m\",\n          \"15m\",\n          \"30m\",\n          \"1h\",\n          \"2h\",\n          \"1d\"\n        ],\n        \"time_options\": [\n          \"5m\",\n          \"15m\",\n          \"1h\",\n          \"6h\",\n          \"12h\",\n          \"24h\",\n          \"2d\",\n          \"7d\",\n          \"30d\"\n        ]\n      },\n      \"timezone\": \"\",\n      \"title\": \"K8S Control Plane\",\n      \"uid\": \"k8scontrolplane\",\n      \"version\": 2\n    }\n"
  },
  {
    "path": "manager/manifests/grafana/grafana-dashboard-nodes.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-dashboard-nodes\n  namespace: default\ndata:\n  nodes.json: |\n    {\n      \"annotations\": {\n        \"list\": [\n          {\n            \"builtIn\": 1,\n            \"datasource\": \"prometheus\",\n            \"enable\": true,\n            \"hide\": true,\n            \"iconColor\": \"rgba(0, 211, 255, 1)\",\n            \"name\": \"Annotations & Alerts\",\n            \"type\": \"dashboard\"\n          }\n        ]\n      },\n      \"editable\": true,\n      \"gnetId\": null,\n      \"graphTooltip\": 0,\n      \"iteration\": 1625177052451,\n      \"links\": [],\n      \"panels\": [\n        {\n          \"datasource\": null,\n          \"description\": \"\",\n          \"gridPos\": {\n            \"h\": 2,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"id\": 11,\n          \"options\": {\n            \"content\": \"<h1 style=\\\"text-align: center\\\">Nodes</h1>\\n\",\n            \"mode\": \"html\"\n          },\n          \"pluginVersion\": \"8.0.4\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"transparent\": true,\n          \"type\": \"text\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"hiddenSeries\": false,\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(\\n  (1 - rate(node_cpu_seconds_total{job=\\\"node-exporter\\\", mode=\\\"idle\\\", instance=\\\"$instance\\\"}[$__interval]))\\n/ ignoring(cpu) group_left\\n  count without (cpu)( node_cpu_seconds_total{job=\\\"node-exporter\\\", mode=\\\"idle\\\", instance=\\\"$instance\\\"})\\n)\\n\",\n              \"format\": \"time_series\",\n              \"interval\": \"1m\",\n              \"intervalFactor\": 5,\n              \"legendFormat\": \"{{cpu}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": 1,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": 1,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 0,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 2\n          },\n          \"hiddenSeries\": false,\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_load1{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"1m load average\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_load5{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"5m load average\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"node_load15{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"15m load average\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"count(node_cpu_seconds_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", mode=\\\"idle\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"logical cores\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load Average\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 18,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"hiddenSeries\": false,\n          \"id\": 4,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(\\n  node_memory_MemTotal_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\\n-\\n  node_memory_MemFree_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\\n-\\n  node_memory_Buffers_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\\n-\\n  node_memory_Cached_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\\n)\\n\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"memory used\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_memory_Buffers_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"memory buffers\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"node_memory_Cached_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"memory cached\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"node_memory_MemFree_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"memory free\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cacheTimeout\": null,\n          \"datasource\": null,\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"color\": {\n                \"mode\": \"thresholds\"\n              },\n              \"mappings\": [\n                {\n                  \"options\": {\n                    \"match\": \"null\",\n                    \"result\": {\n                      \"text\": \"N/A\"\n                    }\n                  },\n                  \"type\": \"special\"\n                }\n              ],\n              \"max\": 100,\n              \"min\": 0,\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"rgba(50, 172, 45, 0.97)\",\n                    \"value\": null\n                  },\n                  {\n                    \"color\": \"rgba(237, 129, 40, 0.89)\",\n                    \"value\": 80\n                  },\n                  {\n                    \"color\": \"rgba(245, 54, 54, 0.9)\",\n                    \"value\": 90\n                  }\n                ]\n              },\n              \"unit\": \"percent\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 6,\n            \"x\": 18,\n            \"y\": 9\n          },\n          \"id\": 5,\n          \"interval\": null,\n          \"links\": [],\n          \"maxDataPoints\": 100,\n          \"options\": {\n            \"orientation\": \"horizontal\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"lastNotNull\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\n            \"showThresholdLabels\": false,\n            \"showThresholdMarkers\": true,\n            \"text\": {}\n          },\n          \"pluginVersion\": \"8.0.4\",\n          \"targets\": [\n            {\n              \"expr\": \"100 -\\n(\\n  avg(node_memory_MemAvailable_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"})\\n/\\n  avg(node_memory_MemTotal_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\"})\\n* 100\\n)\\n\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"title\": \"Memory Usage\",\n          \"type\": \"gauge\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 0,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"hiddenSeries\": false,\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"/ read| written/\",\n              \"yaxis\": 1\n            },\n            {\n              \"alias\": \"/ io time/\",\n              \"yaxis\": 2\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_read_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", device=~\\\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\\\"}[$__interval])\",\n              \"format\": \"time_series\",\n              \"interval\": \"1m\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{device}} read\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(node_disk_written_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", device=~\\\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\\\"}[$__interval])\",\n              \"format\": \"time_series\",\n              \"interval\": \"1m\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{device}} written\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", device=~\\\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\\\"}[$__interval])\",\n              \"format\": \"time_series\",\n              \"interval\": \"1m\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{device}} io time\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk I/O\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 16\n          },\n          \"hiddenSeries\": false,\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"used\",\n              \"color\": \"#E0B400\"\n            },\n            {\n              \"alias\": \"available\",\n              \"color\": \"#73BF69\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(\\n  max by (device) (\\n    node_filesystem_size_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", fstype!=\\\"\\\"}\\n  -\\n    node_filesystem_avail_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", fstype!=\\\"\\\"}\\n  )\\n)\\n\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"used\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(\\n  max by (device) (\\n    node_filesystem_avail_bytes{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", fstype!=\\\"\\\"}\\n  )\\n)\\n\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"available\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Space Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 0,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"hiddenSeries\": false,\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_network_receive_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", device!=\\\"lo\\\"}[$__interval])\",\n              \"format\": \"time_series\",\n              \"interval\": \"1m\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{device}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network Received\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 0,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"hiddenSeries\": false,\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_network_transmit_bytes_total{job=\\\"node-exporter\\\", instance=\\\"$instance\\\", device!=\\\"lo\\\"}[$__interval])\",\n              \"format\": \"time_series\",\n              \"interval\": \"1m\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{device}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network Transmitted\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"refresh\": \"1m\",\n      \"schemaVersion\": 30,\n      \"style\": \"dark\",\n      \"tags\": [],\n      \"templating\": {\n        \"list\": [\n          {\n            \"allValue\": null,\n            \"current\": {},\n            \"datasource\": null,\n            \"definition\": \"\",\n            \"description\": null,\n            \"error\": null,\n            \"hide\": 0,\n            \"includeAll\": false,\n            \"label\": null,\n            \"multi\": false,\n            \"name\": \"instance\",\n            \"options\": [],\n            \"query\": {\n              \"query\": \"label_values(node_exporter_build_info{job=\\\"node-exporter\\\"}, instance)\",\n              \"refId\": \"prometheus-instance-Variable-Query\"\n            },\n            \"refresh\": 2,\n            \"regex\": \"\",\n            \"skipUrlSync\": false,\n            \"sort\": 0,\n            \"tagValuesQuery\": \"\",\n            \"tagsQuery\": \"\",\n            \"type\": \"query\",\n            \"useTags\": false\n          }\n        ]\n      },\n      \"time\": {\n        \"from\": \"now-12h\",\n        \"to\": \"now\"\n      },\n      \"timepicker\": {\n        \"refresh_intervals\": [\n          \"5s\",\n          \"10s\",\n          \"30s\",\n          \"1m\",\n          \"5m\",\n          \"15m\",\n          \"30m\",\n          \"1h\",\n          \"2h\",\n          \"1d\"\n        ],\n        \"time_options\": [\n          \"5m\",\n          \"15m\",\n          \"1h\",\n          \"6h\",\n          \"12h\",\n          \"24h\",\n          \"2d\",\n          \"7d\",\n          \"30d\"\n        ]\n      },\n      \"timezone\": \"\",\n      \"title\": \"Nodes\",\n      \"uid\": \"nodes\",\n      \"version\": 1\n    }\n"
  },
  {
    "path": "manager/manifests/grafana/grafana-dashboard-realtime.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-dashboard-realtime\n  namespace: default\ndata:\n  realtime.json: |-\n    {\n      \"annotations\": {\n        \"list\": [\n          {\n            \"builtIn\": 1,\n            \"datasource\": \"prometheus\",\n            \"enable\": true,\n            \"hide\": true,\n            \"iconColor\": \"rgba(0, 211, 255, 1)\",\n            \"name\": \"Annotations & Alerts\",\n            \"type\": \"dashboard\"\n          }\n        ]\n      },\n      \"editable\": true,\n      \"gnetId\": null,\n      \"graphTooltip\": 0,\n      \"iteration\": 1625175811197,\n      \"links\": [],\n      \"panels\": [\n        {\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 2,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"id\": 15,\n          \"options\": {\n            \"content\": \"<h1 style=\\\"text-align: center\\\">RealtimeAPI</h1>\",\n            \"mode\": \"markdown\"\n          },\n          \"pluginVersion\": \"8.0.4\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"transparent\": true,\n          \"type\": \"text\"\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 22,\n          \"panels\": [],\n          \"title\": \"API Stats\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Request rate, computed over every minute, of an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum (rate(istio_requests_total{destination_service=~\\\"api-$api_name.+\\\"}[1m])) by (destination_service)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{destination_service}}\",\n              \"refId\": \"Request Rate\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request Rate\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"([^\\\\.]+)\\\\..+\",\n                \"renamePattern\": \"$1\"\n              }\n            },\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"reqps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Active in-flight requests for an API.\\n\\nNote: In-flight requests are recorded every 10 seconds, which will correspond to the minimum resolution.\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 4,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(cortex_in_flight_requests{api_name=~\\\"$api_name\\\"}) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"In-Flight Requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Request rate, computed over every minute, for responses with status code 2XX of an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"hiddenSeries\": false,\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(istio_requests_total{destination_service=~\\\"api-$api_name.+\\\", response_code=~\\\"2.+\\\"}[1m])) by (destination_service, response_code)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{destination_service}}\",\n              \"refId\": \"2XX\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"2XX Responses\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"([^\\\\.]+)\\\\..+\",\n                \"renamePattern\": \"$1\"\n              }\n            },\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1217\",\n              \"format\": \"reqps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1218\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"hiddenSeries\": false,\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"count(cortex_in_flight_requests{api_name=~\\\"$api_name\\\", container!=\\\"activator\\\"}) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"Active Replicas\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Active Replicas\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:236\",\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:237\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Request rate, computed over every minute, for responses with status code 4XX of an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"hiddenSeries\": false,\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(istio_requests_total{destination_service=~\\\"api-$api_name.+\\\", response_code=~\\\"4.+\\\"}[1m])) by (destination_service, response_code)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{destination_service}}\",\n              \"refId\": \"4XX\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"4XX Responses\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"([^\\\\.]+)\\\\..+\",\n                \"renamePattern\": \"$1\"\n              }\n            },\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"reqps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Request rate, computed over every minute, for responses with status code 5XX of an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"hiddenSeries\": false,\n          \"id\": 10,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(istio_requests_total{destination_service=~\\\"api-$api_name.+\\\", response_code=~\\\"5.+\\\"}[1m])) by (destination_service, response_code)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{destination_service}}\",\n              \"refId\": \"5XX\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"5XX Responses\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"([^\\\\.]+)\\\\..+\",\n                \"renamePattern\": \"$1\"\n              }\n            },\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"reqps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"99th percentile latency, computed over a minute, for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"hiddenSeries\": false,\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"histogram_quantile(0.99, sum by (destination_service, le) (rate(istio_request_duration_milliseconds_bucket{destination_service=~\\\"api-$api_name.+\\\"}[1m])))\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{destination_service}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"p99 Latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"([^\\\\.]+)\\\\..+\",\n                \"renamePattern\": \"$1\"\n              }\n            },\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1302\",\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1303\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"90th percentile latency, computed over a minute, for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"hiddenSeries\": false,\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"histogram_quantile(0.90, sum by (destination_service, le) (rate(istio_request_duration_milliseconds_bucket{destination_service=~\\\"api-$api_name.+\\\"}[1m])))\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"{{destination_service}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"p90 Latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"([^\\\\.]+)\\\\..+\",\n                \"renamePattern\": \"$1\"\n              }\n            },\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"50th percentile latency, computed over a minute, for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 39\n          },\n          \"hiddenSeries\": false,\n          \"id\": 16,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"histogram_quantile(0.50, sum by (destination_service, le) (rate(istio_request_duration_milliseconds_bucket{destination_service=~\\\"api-$api_name.+\\\"}[1m])))\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"{{destination_service}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"p50 Latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"([^\\\\.]+)\\\\..+\",\n                \"renamePattern\": \"$1\"\n              }\n            },\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Average latency, computed over a minute, for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 39\n          },\n          \"hiddenSeries\": false,\n          \"id\": 12,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(rate(istio_request_duration_milliseconds_sum{destination_service=~\\\"api-$api_name.+\\\"}[1m])) by (destination_service) / sum(rate(istio_request_duration_milliseconds_count{destination_service=~\\\"api-$api_name.+\\\"}[1m])) by (destination_service)\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"{{destination_service}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Average Latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transformations\": [\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"([^\\\\.]+)\\\\..+\",\n                \"renamePattern\": \"$1\"\n              }\n            },\n            {\n              \"id\": \"renameByRegex\",\n              \"options\": {\n                \"regex\": \"api-(.*)\",\n                \"renamePattern\": \"$1\"\n              }\n            }\n          ],\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 48\n          },\n          \"id\": 20,\n          \"panels\": [],\n          \"title\": \"Aggregate Usage\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Total CPU Request\": \"semi-dark-orange\",\n            \"Total CPU Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total CPU usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 49\n          },\n          \"hiddenSeries\": false,\n          \"id\": 24,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(rate(container_cpu_usage_seconds_total{pod=~\\\"api-$api_name.+\\\", container!=\\\"POD\\\", name!=\\\"\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total CPU Usage\",\n              \"refId\": \"CPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"api-$api_name.+\\\", resource=\\\"cpu\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total CPU Request\",\n              \"refId\": \"CPU Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"core\",\n              \"label\": \"cpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total Memory Request\": \"semi-dark-orange\",\n            \"Total Memory Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total memory usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 49\n          },\n          \"hiddenSeries\": false,\n          \"id\": 26,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(sum_over_time(container_memory_working_set_bytes{pod=~\\\"api-$api_name.+\\\", name!=\\\"\\\", container!=\\\"POD\\\"}[1m])) /\\navg(count_over_time(container_memory_working_set_bytes{pod=~\\\"api-$api_name.+\\\", name!=\\\"\\\", container=\\\"api\\\"}[1m])) / 1024^2\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Memory Usage\",\n              \"refId\": \"Memory Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"api-$api_name.+\\\", resource=\\\"memory\\\"}) / 1024^2\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Memory Request\",\n              \"refId\": \"Memory Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total GPU Capacity\": \"semi-dark-orange\",\n            \"Total GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Utilization\": \"light-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total GPU core usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 57\n          },\n          \"hiddenSeries\": false,\n          \"id\": 28,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"}) / 100\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total GPU Usage\",\n              \"refId\": \"GPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total GPU Capacity\",\n              \"refId\": \"GPU Capacity\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total GPU Core Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"gpuCore\",\n              \"label\": \"gpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Total Used GPU Memory\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total GPU memory usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 57\n          },\n          \"hiddenSeries\": false,\n          \"id\": 29,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Used GPU Memory\",\n              \"refId\": \"GPU Used Memory\"\n            },\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\\\"api-$api_name.+\\\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Capacity GPU Memory\",\n              \"refId\": \"GPU Capacity Memory\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total GPU Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 65\n          },\n          \"id\": 18,\n          \"panels\": [],\n          \"title\": \"Average Replica Usage\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Avg CPU Request\": \"semi-dark-orange\",\n            \"Avg CPU Usage\": \"semi-dark-green\",\n            \"Total CPU Request\": \"semi-dark-orange\",\n            \"Total CPU Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg CPU usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 66\n          },\n          \"hiddenSeries\": false,\n          \"id\": 30,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(rate(container_cpu_usage_seconds_total{pod=~\\\"api-$api_name.+\\\", container!=\\\"POD\\\", name!=\\\"\\\"}[1m]))\\n/\\nsum(kube_pod_info{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg CPU Usage\",\n              \"refId\": \"CPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"api-$api_name.+\\\", resource=\\\"cpu\\\"})\\n/\\nsum(kube_pod_info{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg CPU Request\",\n              \"refId\": \"CPU Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"core\",\n              \"label\": \"cpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg Memory Request\": \"semi-dark-orange\",\n            \"Avg Memory Usage\": \"semi-dark-green\",\n            \"Total Memory Request\": \"semi-dark-orange\",\n            \"Total Memory Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg memory usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 66\n          },\n          \"hiddenSeries\": false,\n          \"id\": 31,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(sum_over_time(container_memory_working_set_bytes{pod=~\\\"api-$api_name.+\\\", name!=\\\"\\\", container!=\\\"POD\\\"}[1m]))\\n/\\navg(count_over_time(container_memory_working_set_bytes{pod=~\\\"api-$api_name.+\\\", name!=\\\"\\\", container=\\\"api\\\"}[1m])) / 1024^2\\n/\\nsum(kube_pod_info{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Memory Usage\",\n              \"refId\": \"Memory Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"api-$api_name.+\\\", resource=\\\"memory\\\"}) / 1024^2\\n/\\nsum(kube_pod_info{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Memory Request\",\n              \"refId\": \"Memory Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg GPU Capacity\": \"semi-dark-orange\",\n            \"Avg GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Capacity\": \"semi-dark-orange\",\n            \"Total GPU Utilization\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg GPU core usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 74\n          },\n          \"hiddenSeries\": false,\n          \"id\": 32,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"}) / 100\\n/\\ncount(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg GPU Usage\",\n              \"refId\": \"GPU Usage\"\n            },\n            {\n              \"expr\": \"count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"})\\n/\\ncount(count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"api-$api_name.+\\\"}) by (exported_pod))\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg GPU Capacity\",\n              \"refId\": \"GPU Capacity\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg GPU Core Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"gpuCore\",\n              \"label\": \"gpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Avg Used GPU Memory\": \"semi-dark-green\",\n            \"Total Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Total Used GPU Memory\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg GPU memory usage across all replicas of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 74\n          },\n          \"hiddenSeries\": false,\n          \"id\": 33,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\\n/\\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Used GPU Memory\",\n              \"refId\": \"GPU Used Memory\"\n            },\n            {\n              \"exemplar\": false,\n              \"expr\": \"(sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\\\"api-$api_name.+\\\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"}))\\n/\\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"api-$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Capacity GPU Memory\",\n              \"refId\": \"GPU Capacity Memory\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg GPU Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"refresh\": \"30s\",\n      \"schemaVersion\": 30,\n      \"style\": \"dark\",\n      \"tags\": [],\n      \"templating\": {\n        \"list\": [\n          {\n            \"allValue\": null,\n            \"current\": {\n              \"selected\": false,\n              \"text\": \"None\",\n              \"value\": \"None\"\n            },\n            \"datasource\": null,\n            \"definition\": \"label_values(cortex_in_flight_requests, api_name)\",\n            \"description\": null,\n            \"error\": null,\n            \"hide\": 0,\n            \"includeAll\": false,\n            \"label\": \"API Name\",\n            \"multi\": true,\n            \"name\": \"api_name\",\n            \"options\": [],\n            \"query\": {\n              \"query\": \"label_values(cortex_in_flight_requests, api_name)\",\n              \"refId\": \"StandardVariableQuery\"\n            },\n            \"refresh\": 1,\n            \"regex\": \"\",\n            \"skipUrlSync\": false,\n            \"sort\": 0,\n            \"tagValuesQuery\": \"\",\n            \"tagsQuery\": \"\",\n            \"type\": \"query\",\n            \"useTags\": false\n          }\n        ]\n      },\n      \"time\": {\n        \"from\": \"now-1h\",\n        \"to\": \"now\"\n      },\n      \"timepicker\": {},\n      \"timezone\": \"\",\n      \"title\": \"RealtimeAPI\",\n      \"uid\": \"realtimeapi\",\n      \"version\": 3\n    }\n"
  },
  {
    "path": "manager/manifests/grafana/grafana-dashboard-task.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-dashboard-task\n  namespace: default\ndata:\n  task.json: |-\n    {\n      \"annotations\": {\n        \"list\": [\n          {\n            \"builtIn\": 1,\n            \"datasource\": \"prometheus\",\n            \"enable\": true,\n            \"hide\": true,\n            \"iconColor\": \"rgba(0, 211, 255, 1)\",\n            \"name\": \"Annotations & Alerts\",\n            \"type\": \"dashboard\"\n          }\n        ]\n      },\n      \"editable\": true,\n      \"gnetId\": null,\n      \"graphTooltip\": 0,\n      \"id\": 7,\n      \"iteration\": 1625281006508,\n      \"links\": [],\n      \"panels\": [\n        {\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 2,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 0\n          },\n          \"id\": 7,\n          \"options\": {\n            \"content\": \"<h1 style=\\\"text-align: center\\\">TaskAPI</h1>\\n\",\n            \"mode\": \"markdown\"\n          },\n          \"pluginVersion\": \"8.0.4\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"transparent\": true,\n          \"type\": \"text\"\n        },\n        {\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 22,\n          \"title\": \"API Stats\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Number of succeeded tasks for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(cortex_task_succeeded{api_name=~\\\"$api_name\\\"}) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"# Succeeded Tasks\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:26\",\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:27\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Number of failed tasks for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"hiddenSeries\": false,\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(cortex_task_failed{api_name=~\\\"$api_name\\\"}) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"# Failed Tasks\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:262\",\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:263\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Average time per task for an API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"hiddenSeries\": false,\n          \"id\": 5,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(cortex_time_per_task_sum{api_name=~\\\"$api_name\\\"}) by (api_name) / sum(cortex_time_per_task_count{api_name=~\\\"$api_name\\\"}) by (api_name)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{api_name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Average Time per Task\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:161\",\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:162\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Active Jobs\": \"semi-dark-green\",\n            \"Active Workers\": \"semi-dark-orange\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Active tasks\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"hiddenSeries\": false,\n          \"id\": 20,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"count(kube_job_status_active{job_name=~\\\"$api_name.+\\\"} != 0)\",\n              \"interval\": \"\",\n              \"legendFormat\": \"Active Tasks\",\n              \"refId\": \"Active Tasks\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"# Active Tasks\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:128\",\n              \"decimals\": 0,\n              \"format\": \"count\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:129\",\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"id\": 11,\n          \"title\": \"Aggregate Worker Usage\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Total CPU Request\": \"semi-dark-orange\",\n            \"Total CPU Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total CPU usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"hiddenSeries\": false,\n          \"id\": 13,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(rate(container_cpu_usage_seconds_total{pod=~\\\"$api_name.+\\\", container!=\\\"POD\\\", name!=\\\"\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total CPU Usage\",\n              \"refId\": \"CPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"$api_name.+\\\", resource=\\\"cpu\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total CPU Request\",\n              \"refId\": \"CPU Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"core\",\n              \"label\": \"cpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total Memory Request\": \"semi-dark-orange\",\n            \"Total Memory Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total memory usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"hiddenSeries\": false,\n          \"id\": 15,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(sum_over_time(container_memory_working_set_bytes{pod=~\\\"$api_name.+\\\", name!=\\\"\\\", container!=\\\"\\\"}[1m]))\\n/\\navg(count_over_time(container_memory_working_set_bytes{pod=~\\\"$api_name.+\\\", name!=\\\"\\\", container!=\\\"\\\"}[1m])) / 1024^2\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Memory Usage\",\n              \"refId\": \"Memory Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"$api_name.+\\\", resource=\\\"memory\\\"}) / 1024^2\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Memory Request\",\n              \"refId\": \"Memory Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total GPU Capacity\": \"semi-dark-orange\",\n            \"Total GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Utilization\": \"light-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total GPU core usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 29\n          },\n          \"hiddenSeries\": false,\n          \"id\": 17,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"}) / 100\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total GPU Usage\",\n              \"refId\": \"GPU Usage\"\n            },\n            {\n              \"expr\": \"count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total GPU Capacity\",\n              \"refId\": \"GPU Capacity\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total GPU Core Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"gpuCore\",\n              \"label\": \"gpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Total Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Total Used GPU Memory\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Total GPU memory usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 29\n          },\n          \"hiddenSeries\": false,\n          \"id\": 19,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Used GPU Memory\",\n              \"refId\": \"GPU Used Memory\"\n            },\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\\\"$api_name.+\\\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Total Capacity GPU Memory\",\n              \"refId\": \"GPU Capacity Memory\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total GPU Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"collapsed\": false,\n          \"datasource\": null,\n          \"gridPos\": {\n            \"h\": 1,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 37\n          },\n          \"id\": 9,\n          \"panels\": [],\n          \"title\": \"Avg Worker Usage\",\n          \"type\": \"row\"\n        },\n        {\n          \"aliasColors\": {\n            \"Avg CPU Request\": \"semi-dark-orange\",\n            \"Avg CPU Usage\": \"semi-dark-green\",\n            \"Total CPU Request\": \"semi-dark-orange\",\n            \"Total CPU Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg CPU usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 38\n          },\n          \"hiddenSeries\": false,\n          \"id\": 23,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(rate(container_cpu_usage_seconds_total{pod=~\\\"$api_name.+\\\", container!=\\\"POD\\\", name!=\\\"\\\"}[1m]))\\n/\\nsum(kube_pod_info{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg CPU Usage\",\n              \"refId\": \"CPU Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource_requests{exported_pod=~\\\"$api_name.+\\\", resource=\\\"cpu\\\"})\\n/\\nsum(kube_pod_info{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg CPU Request\",\n              \"refId\": \"CPU Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"core\",\n              \"label\": \"cpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg Memory Request\": \"semi-dark-orange\",\n            \"Avg Memory Usage\": \"semi-dark-green\",\n            \"Total Memory Request\": \"semi-dark-orange\",\n            \"Total Memory Usage\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg memory usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 38\n          },\n          \"hiddenSeries\": false,\n          \"id\": 24,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"exemplar\": false,\n              \"expr\": \"sum(sum_over_time(container_memory_working_set_bytes{pod=~\\\"$api_name.+\\\", name!=\\\"\\\", container!=\\\"\\\"}[1m]))\\n/\\navg(count_over_time(container_memory_working_set_bytes{pod=~\\\"$api_name.+\\\", name!=\\\"\\\", container!=\\\"\\\"}[1m])) / 1024^2\\n/\\nsum(kube_pod_info{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Memory Usage\",\n              \"refId\": \"Memory Usage\"\n            },\n            {\n              \"exemplar\": true,\n              \"expr\": \"sum(kube_pod_container_resource{exported_pod=~\\\"$api_name.+\\\", resource=\\\"memory\\\"}) / 1024^2\\n/\\nsum(kube_pod_info{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Memory Request\",\n              \"refId\": \"Memory Request\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg GPU Capacity\": \"semi-dark-orange\",\n            \"Avg GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Capacity\": \"semi-dark-orange\",\n            \"Total GPU Usage\": \"semi-dark-green\",\n            \"Total GPU Utilization\": \"light-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg GPU core usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 46\n          },\n          \"hiddenSeries\": false,\n          \"id\": 25,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"}) / 100\\n/\\ncount(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg GPU Usage\",\n              \"refId\": \"GPU Usage\"\n            },\n            {\n              \"expr\": \"count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"})\\n/\\ncount(count(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\\\"$api_name.+\\\"}) by (exported_pod))\",\n              \"hide\": false,\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg GPU Capacity\",\n              \"refId\": \"GPU Capacity\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg GPU Core Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"gpuCore\",\n              \"label\": \"gpu\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {\n            \"Avg Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Avg Used GPU Memory\": \"semi-dark-green\",\n            \"Total Capacity GPU Memory\": \"semi-dark-orange\",\n            \"Total Used GPU Memory\": \"semi-dark-green\"\n          },\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": null,\n          \"description\": \"Avg GPU memory usage across all workers of the API\",\n          \"fill\": 1,\n          \"fillGradient\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 46\n          },\n          \"hiddenSeries\": false,\n          \"id\": 26,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"nullPointMode\": \"null\",\n          \"options\": {\n            \"alertThreshold\": true\n          },\n          \"percentage\": false,\n          \"pluginVersion\": \"8.0.4\",\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\\n/\\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Used GPU Memory\",\n              \"refId\": \"GPU Used Memory\"\n            },\n            {\n              \"exemplar\": false,\n              \"expr\": \"(sum(DCGM_FI_DEV_FB_FREE{exported_pod=~\\\"$api_name.+\\\"}) + sum(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"}))\\n/\\ncount(DCGM_FI_DEV_FB_USED{exported_pod=~\\\"$api_name.+\\\"})\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"legendFormat\": \"Avg Capacity GPU Memory\",\n              \"refId\": \"GPU Capacity Memory\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Avg GPU Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"$$hashKey\": \"object:1404\",\n              \"format\": \"MiB\",\n              \"label\": \"memory\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"$$hashKey\": \"object:1405\",\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"refresh\": \"30s\",\n      \"schemaVersion\": 30,\n      \"style\": \"dark\",\n      \"tags\": [],\n      \"templating\": {\n        \"list\": [\n          {\n            \"allValue\": null,\n            \"current\": {\n              \"selected\": false,\n              \"text\": \"None\",\n              \"value\": \"None\"\n            },\n            \"datasource\": null,\n            \"definition\": \"label_values({__name__=~\\\"cortex_task_.+\\\"}, api_name)\",\n            \"description\": null,\n            \"error\": null,\n            \"hide\": 0,\n            \"includeAll\": false,\n            \"label\": \"API Name\",\n            \"multi\": true,\n            \"name\": \"api_name\",\n            \"options\": [],\n            \"query\": {\n              \"query\": \"label_values({__name__=~\\\"cortex_task_.+\\\"}, api_name)\",\n              \"refId\": \"StandardVariableQuery\"\n            },\n            \"refresh\": 2,\n            \"regex\": \"\",\n            \"skipUrlSync\": false,\n            \"sort\": 0,\n            \"tagValuesQuery\": \"\",\n            \"tagsQuery\": \"\",\n            \"type\": \"query\",\n            \"useTags\": false\n          }\n        ]\n      },\n      \"time\": {\n        \"from\": \"now-1h\",\n        \"to\": \"now\"\n      },\n      \"timepicker\": {},\n      \"timezone\": \"\",\n      \"title\": \"TaskAPI\",\n      \"uid\": \"taskapi\",\n      \"version\": 6\n    }\n"
  },
  {
    "path": "manager/manifests/grafana/grafana.yaml.j2",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-datasources\n  namespace: default\ndata:\n  datasources.yaml: |\n    {\n        \"apiVersion\": 1,\n        \"datasources\": [\n            {\n                \"access\": \"proxy\",\n                \"editable\": false,\n                \"name\": \"prometheus\",\n                \"orgId\": 1,\n                \"type\": \"prometheus\",\n                \"url\": \"http://prometheus.prometheus:9090\",\n                \"version\": 1,\n                \"isDefault\": true\n            }\n        ]\n    }\n\n---\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: grafana-dashboards\n  namespace: default\ndata:\n  dashboards.yaml: |-\n    {\n        \"apiVersion\": 1,\n        \"providers\": [\n            {\n                \"folder\": \"Cortex\",\n                \"name\": \"Cortex\",\n                \"options\": {\n                    \"path\": \"/grafana-dashboard-definitions/cortex\"\n                },\n                \"disableDeletion\": true,\n                \"orgId\": 1,\n                \"type\": \"file\"\n            }\n        ]\n    }\n\n---\n\napiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n  name: grafana-storage\n  namespace: default\nspec:\n  storageClassName: ssd\n  accessModes:\n    - ReadWriteOnce\n  resources:\n    requests:\n      storage: 2Gi\n\n---\n\napiVersion: apps/v1\nkind: StatefulSet\nmetadata:\n  labels:\n    app: grafana\n  name: grafana\n  namespace: default\nspec:\n  serviceName: grafana\n  replicas: 1\n  selector:\n    matchLabels:\n      app: grafana\n  template:\n    metadata:\n      labels:\n        app: grafana\n    spec:\n      containers:\n        - image: {{ config['image_grafana'] }}\n          name: grafana\n          ports:\n            - containerPort: 3000\n              name: http\n          readinessProbe:\n            httpGet:\n              path: /api/health\n              port: http\n          resources:\n            limits:\n              cpu: 200m\n              memory: 200Mi\n            requests:\n              cpu: 100m\n              memory: 100Mi\n          env:\n            - name: GF_SERVER_ROOT_URL\n              value: \"%(protocol)s://%(domain)s:%(http_port)s/dashboard\"\n            - name: GF_SERVER_SERVE_FROM_SUB_PATH\n              value: \"true\"\n            - name: GF_USERS_DEFAULT_THEME\n              value: \"light\"\n          volumeMounts:\n            - mountPath: /var/lib/grafana\n              name: grafana-storage\n              readOnly: false\n            - mountPath: /etc/grafana/provisioning/datasources\n              name: grafana-datasources\n              readOnly: false\n            - mountPath: /etc/grafana/provisioning/dashboards\n              name: grafana-dashboards\n              readOnly: false\n            - mountPath: /grafana-dashboard-definitions/cortex/realtime\n              name: grafana-dashboard-realtime\n              readOnly: false\n            - mountPath: /grafana-dashboard-definitions/cortex/async\n              name: grafana-dashboard-async\n              readOnly: false\n            - mountPath: /grafana-dashboard-definitions/cortex/batch\n              name: grafana-dashboard-batch\n              readOnly: false\n            - mountPath: /grafana-dashboard-definitions/cortex/task\n              name: grafana-dashboard-task\n              readOnly: false\n            - mountPath: /grafana-dashboard-definitions/cortex/cluster\n              name: grafana-dashboard-cluster\n              readOnly: false\n            - mountPath: /grafana-dashboard-definitions/cortex/nodes\n              name: grafana-dashboard-nodes\n              readOnly: false\n            {% if env.get(\"CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD\") == \"true\" %}\n            - mountPath: /grafana-dashboard-definitions/cortex/control-plane\n              name: grafana-dashboard-control-plane\n              readOnly: false\n            {% endif %}\n      securityContext:\n        fsGroup: 65534\n        runAsNonRoot: true\n        runAsUser: 65534\n      volumes:\n        - name: grafana-storage\n          persistentVolumeClaim:\n            claimName: grafana-storage\n        - name: grafana-datasources\n          configMap:\n            name: grafana-datasources\n        - name: grafana-dashboards\n          configMap:\n            name: grafana-dashboards\n        - name: grafana-dashboard-realtime\n          configMap:\n            name: grafana-dashboard-realtime\n        - name: grafana-dashboard-async\n          configMap:\n            name: grafana-dashboard-async\n        - name: grafana-dashboard-batch\n          configMap:\n            name: grafana-dashboard-batch\n        - name: grafana-dashboard-task\n          configMap:\n            name: grafana-dashboard-task\n        - name: grafana-dashboard-cluster\n          configMap:\n            name: grafana-dashboard-cluster\n        - name: grafana-dashboard-nodes\n          configMap:\n            name: grafana-dashboard-nodes\n        {% if env.get(\"CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD\") == \"true\" %}\n        - name: grafana-dashboard-control-plane\n          configMap:\n            name: grafana-dashboard-control-plane\n        {% endif %}\n      nodeSelector:\n        prometheus: \"true\"\n      tolerations:\n        - key: prometheus\n          operator: Exists\n          effect: NoSchedule\n      affinity:\n        podAffinity:\n          preferredDuringSchedulingIgnoredDuringExecution:\n            - podAffinityTerm:\n                labelSelector:\n                  matchLabels:\n                    prometheus: prometheus\n                topologyKey: kubernetes.io/hostname\n              weight: 100\n\n---\n\napiVersion: v1\nkind: Service\nmetadata:\n  labels:\n    app: grafana\n  name: grafana\n  namespace: default\nspec:\n  type: ClusterIP\n  ports:\n    - name: http\n      port: 3000\n      targetPort: http\n  selector:\n    app: grafana\n\n---\n\napiVersion: networking.istio.io/v1beta1\nkind: VirtualService\nmetadata:\n  name: grafana\n  namespace: default\nspec:\n  hosts:\n    - \"*\"\n  gateways:\n    - operator-gateway\n  http:\n    - name: grafana\n      match:\n        - uri:\n            prefix: \"/dashboard\"\n        - uri:\n            prefix: \"/grafana\"\n      rewrite:\n        uri: \"/dashboard\"\n      route:\n        - destination:\n            host: grafana\n            port:\n              number: 3000\n"
  },
  {
    "path": "manager/manifests/inferentia.yaml",
    "content": "# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy of this\n# software and associated documentation files (the \"Software\"), to deal in the Software\n# without restriction, including without limitation the rights to use, copy, modify,\n# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to\n# permit persons to whom the Software is furnished to do so.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n# Source: https://github.com/aws/aws-neuron-sdk/blob/master/src/k8/k8s-neuron-*\n\napiVersion: v1\ndata:\n  policy.cfg: |\n    {\n      \"kind\": \"Policy\",\n      \"apiVersion\": \"v1\",\n      \"extenders\": [\n        {\n          \"urlPrefix\": \"http://127.0.0.1:32700\",\n          \"filterVerb\": \"filter\",\n          \"bindVerb\":   \"bind\",\n          \"enableHttps\": false,\n          \"nodeCacheCapable\": true,\n          \"managedResources\": [\n            {\n              \"name\": \"aws.amazon.com/neuron\",\n              \"ignoredByScheduler\": false\n            }\n          ],\n          \"ignorable\": false\n        }\n      ]\n    }\nkind: ConfigMap\nmetadata:\n  name: scheduler-policy\n  namespace: kube-system\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  name: k8s-neuron-scheduler\nrules:\n  - apiGroups:\n      - \"\"\n    resources:\n      - nodes\n    verbs:\n      - get\n      - list\n      - watch\n  - apiGroups:\n      - \"\"\n    resources:\n      - events\n    verbs:\n      - create\n      - patch\n  - apiGroups:\n      - \"\"\n    resources:\n      - pods\n    verbs:\n      - update\n      - patch\n      - get\n      - list\n      - watch\n  - apiGroups:\n      - \"\"\n    resources:\n      - bindings\n      - pods/binding\n    verbs:\n      - create\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: k8s-neuron-scheduler\n  namespace: kube-system\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  name: k8s-neuron-scheduler\n  namespace: kube-system\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: k8s-neuron-scheduler\nsubjects:\n  - kind: ServiceAccount\n    name: k8s-neuron-scheduler\n    namespace: kube-system\n\n---\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n  name: k8s-neuron-scheduler\n  namespace: kube-system\nspec:\n  replicas: 1\n  strategy:\n    type: Recreate\n  selector:\n    matchLabels:\n      app: neuron-scheduler\n      component: k8s-neuron-scheduler\n  template:\n    metadata:\n      labels:\n        app: neuron-scheduler\n        component: k8s-neuron-scheduler\n    spec:\n      hostNetwork: true\n      tolerations:\n        - effect: NoSchedule\n          operator: Exists\n          key: node-role.kubernetes.io/master\n        - effect: NoSchedule\n          operator: Exists\n          key: node.cloudprovider.kubernetes.io/uninitialized\n      serviceAccount: k8s-neuron-scheduler\n      priorityClassName: \"system-node-critical\"\n      containers:\n        - name: neuron-scheduler\n          image: $CORTEX_IMAGE_NEURON_SCHEDULER\n          env:\n            - name: PORT\n              value: \"12345\"\n          resources:\n            requests:\n              cpu: 50m\n              memory: 100Mi\n\n---\napiVersion: v1\nkind: Service\nmetadata:\n  name: k8s-neuron-scheduler\n  namespace: kube-system\n  labels:\n    app: neuron-scheduler\n    component: k8s-neuron-scheduler\nspec:\n  type: NodePort\n  ports:\n    - port: 12345\n      name: http\n      targetPort: 12345\n      nodePort: 32700\n  selector:\n    # select app=ingress-nginx pods\n    app: neuron-scheduler\n    component: k8s-neuron-scheduler\n\n---\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  name: neuron-device-plugin\nrules:\n  - apiGroups:\n      - \"\"\n    resources:\n      - nodes\n    verbs:\n      - get\n      - list\n      - watch\n  - apiGroups:\n      - \"\"\n    resources:\n      - events\n    verbs:\n      - create\n      - patch\n  - apiGroups:\n      - \"\"\n    resources:\n      - pods\n    verbs:\n      - update\n      - patch\n      - get\n      - list\n      - watch\n  - apiGroups:\n      - \"\"\n    resources:\n      - nodes/status\n    verbs:\n      - patch\n      - update\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: neuron-device-plugin\n  namespace: kube-system\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n  name: neuron-device-plugin\n  namespace: kube-system\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: neuron-device-plugin\nsubjects:\n  - kind: ServiceAccount\n    name: neuron-device-plugin\n    namespace: kube-system\n\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: neuron-device-plugin-daemonset\n  namespace: kube-system\nspec:\n  selector:\n    matchLabels:\n      name: neuron-device-plugin-ds\n  updateStrategy:\n    type: RollingUpdate\n  template:\n    metadata:\n      labels:\n        name: neuron-device-plugin-ds\n    spec:\n      serviceAccount: neuron-device-plugin\n      tolerations:\n        - key: CriticalAddonsOnly\n          operator: Exists\n        - key: aws.amazon.com/neuron\n          operator: Exists\n          effect: NoSchedule\n        - key: workload\n          operator: Exists\n          effect: NoSchedule\n      # Mark this pod as a critical add-on; when enabled, the critical add-on\n      # scheduler reserves resources for critical add-on pods so that they can\n      # be rescheduled after a failure.\n      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/\n      priorityClassName: \"system-node-critical\"\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n              - matchExpressions:\n                  - key: \"node.kubernetes.io/instance-type\"\n                    operator: In\n                    values:\n                      - inf1.xlarge\n                      - inf1.2xlarge\n                      - inf1.6xlarge\n                      - inf1.24xlarge\n      containers:\n        #Device Plugin containers are available both in us-east and us-west ecr\n        #repos\n        - image: $CORTEX_IMAGE_NEURON_DEVICE_PLUGIN\n          imagePullPolicy: Always\n          name: neuron-device-plugin\n          env:\n            - name: KUBECONFIG\n              value: /etc/kubernetes/kubelet.conf\n            - name: NODE_NAME\n              valueFrom:\n                fieldRef:\n                  fieldPath: spec.nodeName\n          securityContext:\n            allowPrivilegeEscalation: false\n            capabilities:\n              drop: [\"ALL\"]\n          volumeMounts:\n            - name: device-plugin\n              mountPath: /var/lib/kubelet/device-plugins\n            - name: infa-map\n              mountPath: /run\n          resources:\n            requests:\n              cpu: 100m\n              memory: 100Mi\n      nodeSelector:\n        workload: \"true\"\n        aws.amazon.com/neuron: \"true\"\n      volumes:\n        - name: device-plugin\n          hostPath:\n            path: /var/lib/kubelet/device-plugins\n        - name: infa-map\n          hostPath:\n            path: /run\n"
  },
  {
    "path": "manager/manifests/istio.yaml.j2",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: install.istio.io/v1alpha1\nkind: IstioOperator\nspec:\n  profile: minimal\n  hub: {{ env['CORTEX_IMAGE_ISTIO_PROXY_HUB'] }}  # this is only used by proxy, since pilot overrides it (proxy doesn't have dedicated hub config)\n  tag: {{ env['CORTEX_IMAGE_ISTIO_PROXY_TAG'] }}  # this is only used by proxy, since pilot overrides it (proxy doesn't have dedicated tag config)\n  meshConfig:\n    discoverySelectors:\n      - matchLabels:\n          istio-discovery: enabled\n  components:\n    pilot:  # \"pilot\" refers to the istiod container\n      hub: {{ env['CORTEX_IMAGE_ISTIO_PILOT_HUB'] }}\n      tag: {{ env['CORTEX_IMAGE_ISTIO_PILOT_TAG'] }}\n      k8s:\n        resources:\n          requests:\n            cpu: 100m  # default is 500m\n            memory: 700Mi  # default is 2048Mi == 2Gi\n        hpaSpec:\n          minReplicas: 1\n          maxReplicas: 5\n          metrics:\n            - type: Resource\n              resource:\n                name: cpu\n                targetAverageUtilization: 90\n            - type: Resource\n              resource:\n                name: memory\n                targetAverageUtilization: 90\n          scaleTargetRef:\n            apiVersion: apps/v1\n            kind: Deployment\n            name: istiod\n    cni:\n      enabled: false\n    ingressGateways:\n      - name: istio-ingressgateway\n        enabled: false\n      - name: ingressgateway-operator\n        enabled: true\n        namespace: istio-system\n        label:\n          app: operator-istio-gateway\n          istio: ingressgateway-operator\n        k8s:\n          serviceAnnotations:\n            service.beta.kubernetes.io/aws-load-balancer-type: \"nlb\"\n            service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: \"true\"\n            service.beta.kubernetes.io/aws-load-balancer-backend-protocol: \"tcp\"\n            service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: \"{{ env['CORTEX_OPERATOR_LOAD_BALANCER_TAGS'] }}\"\n            {% if config.get('operator_load_balancer_scheme') == 'internal' %}\n            service.beta.kubernetes.io/aws-load-balancer-internal: \"true\"\n            {% endif %}\n          service:\n            type: LoadBalancer\n            externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies\n            loadBalancerSourceRanges: {{ config.get('operator_load_balancer_cidr_white_list', ['0.0.0.0/0']) }}\n            selector:\n              app: operator-istio-gateway\n              istio: ingressgateway-operator\n            ports:\n              - name: http2\n                port: 80\n                targetPort: 80\n              - name: https\n                port: 443\n                targetPort: 443\n          resources:\n            requests:\n              cpu: 100m\n              memory: 128Mi\n            limits:\n              cpu: 1000m\n              memory: 1024Mi\n          replicaCount: 1\n          hpaSpec:\n            minReplicas: 1\n            maxReplicas: 1\n            metrics:\n              - type: Resource\n                resource:\n                  name: cpu\n                  targetAverageUtilization: 80\n            scaleTargetRef:\n              apiVersion: apps/v1\n              kind: Deployment\n              name: ingressgateway-operator\n      - name: ingressgateway-apis\n        enabled: true\n        namespace: istio-system\n        label:\n          app: apis-istio-gateway\n          istio: ingressgateway-apis\n        k8s:\n          serviceAnnotations:\n            service.beta.kubernetes.io/aws-load-balancer-type: \"{{ env['CORTEX_API_LOAD_BALANCER_TYPE'] }}\"\n            service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: \"true\"\n            service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: \"{{ env['CORTEX_API_LOAD_BALANCER_TAGS'] }}\"\n            service.beta.kubernetes.io/aws-load-balancer-backend-protocol: \"tcp\"\n            service.beta.kubernetes.io/aws-load-balancer-ssl-ports: \"https\"  # \"https\" is the name of the https port below\n            {% if config.get('api_load_balancer_scheme') == 'internal' %}\n            service.beta.kubernetes.io/aws-load-balancer-internal: \"true\"\n            {% endif %}\n            {% if config.get('ssl_certificate_arn', '') != '' %}\n            service.beta.kubernetes.io/aws-load-balancer-ssl-cert: \"{{ config['ssl_certificate_arn'] }}\"\n            {% endif %}\n          service:\n            type: LoadBalancer\n            loadBalancerSourceRanges: {{ config.get('api_load_balancer_cidr_white_list', ['0.0.0.0/0']) }}\n            externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies\n            selector:\n              app: apis-istio-gateway\n              istio: ingressgateway-apis\n            ports:\n              - name: http2\n                port: 80\n                targetPort: 80\n              - name: https\n                port: 443\n                targetPort: 443\n          resources:\n            requests:\n              cpu: 400m\n              memory: 512Mi\n            limits:\n              cpu: 1500m\n              memory: 1024Mi\n          replicaCount: 1\n          hpaSpec:\n            minReplicas: 1\n            maxReplicas: 100\n            metrics:\n              - type: Resource\n                resource:\n                  name: cpu\n                  targetAverageUtilization: 90\n              - type: Resource\n                resource:\n                  name: mem\n                  targetAverageUtilization: 90\n            scaleTargetRef:\n              apiVersion: apps/v1\n              kind: Deployment\n              name: ingressgateway-apis\n  values:\n    global:\n      defaultResources:\n        requests:\n          cpu: 10m\n      proxy:\n        autoInject: disabled\n        image: {{ env['CORTEX_IMAGE_ISTIO_PROXY_IMAGE'] }}\n    pilot:\n      image: {{ env['CORTEX_IMAGE_ISTIO_PILOT_IMAGE'] }}\n    gateways:\n      istio-ingressgateway:\n        runAsRoot: true\n        autoscaleEnabled: true\n        secretVolumes:\n        - name: customgateway-certs\n          secretName: istio-customgateway-certs\n          mountPath: /etc/istio/customgateway-certs\n        - name: customgateway-ca-certs\n          secretName: istio-customgateway-ca-certs\n          mountPath: /etc/istio/customgateway-ca-certs\n"
  },
  {
    "path": "manager/manifests/kube-proxy.patch.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# This is a patch that needs to be applied onto the daemonset that's added by eksctl.\n\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: kube-proxy\n  namespace: kube-system\nspec:\n  selector:\n    matchLabels:\n      k8s-app: kube-proxy\n  template:\n    spec:\n      containers:\n      - name: kube-proxy\n        command:\n        - kube-proxy\n        - --v=2\n        - --proxy-mode=ipvs\n        - --ipvs-scheduler=rr\n        - --config=/var/lib/kube-proxy/config\n        env:\n        - name: KUBE_PROXY_MODE\n          value: ipvs\n  updateStrategy:\n    rollingUpdate:\n      maxUnavailable: 20%\n    type: RollingUpdate\n"
  },
  {
    "path": "manager/manifests/metrics-server.yaml",
    "content": "# Copyright 2021 The Kubernetes Authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n# Source: https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.6.1/components.yaml\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  labels:\n    k8s-app: metrics-server\n  name: metrics-server\n  namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  labels:\n    k8s-app: metrics-server\n    rbac.authorization.k8s.io/aggregate-to-admin: \"true\"\n    rbac.authorization.k8s.io/aggregate-to-edit: \"true\"\n    rbac.authorization.k8s.io/aggregate-to-view: \"true\"\n  name: system:aggregated-metrics-reader\nrules:\n  - apiGroups:\n      - metrics.k8s.io\n    resources:\n      - pods\n      - nodes\n    verbs:\n      - get\n      - list\n      - watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  labels:\n    k8s-app: metrics-server\n  name: system:metrics-server\nrules:\n  - apiGroups:\n      - \"\"\n    resources:\n      - nodes/metrics\n    verbs:\n      - get\n  - apiGroups:\n      - \"\"\n    resources:\n      - pods\n      - nodes\n    verbs:\n      - get\n      - list\n      - watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n  labels:\n    k8s-app: metrics-server\n  name: metrics-server-auth-reader\n  namespace: kube-system\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: Role\n  name: extension-apiserver-authentication-reader\nsubjects:\n  - kind: ServiceAccount\n    name: metrics-server\n    namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  labels:\n    k8s-app: metrics-server\n  name: metrics-server:system:auth-delegator\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: system:auth-delegator\nsubjects:\n  - kind: ServiceAccount\n    name: metrics-server\n    namespace: kube-system\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  labels:\n    k8s-app: metrics-server\n  name: system:metrics-server\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: system:metrics-server\nsubjects:\n  - kind: ServiceAccount\n    name: metrics-server\n    namespace: kube-system\n---\napiVersion: v1\nkind: Service\nmetadata:\n  labels:\n    k8s-app: metrics-server\n  name: metrics-server\n  namespace: kube-system\nspec:\n  ports:\n    - name: https\n      port: 443\n      protocol: TCP\n      targetPort: https\n  selector:\n    k8s-app: metrics-server\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  labels:\n    k8s-app: metrics-server\n  name: metrics-server\n  namespace: kube-system\nspec:\n  selector:\n    matchLabels:\n      k8s-app: metrics-server\n  strategy:\n    rollingUpdate:\n      maxUnavailable: 0\n  template:\n    metadata:\n      labels:\n        k8s-app: metrics-server\n    spec:\n      containers:\n        - args:\n            - --cert-dir=/tmp\n            - --secure-port=4443\n            - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname\n            - --kubelet-use-node-status-port\n            - --metric-resolution=15s\n          image: $CORTEX_IMAGE_METRICS_SERVER\n          imagePullPolicy: Always\n          livenessProbe:\n            failureThreshold: 3\n            httpGet:\n              path: /livez\n              port: https\n              scheme: HTTPS\n            periodSeconds: 10\n          name: metrics-server\n          ports:\n            - containerPort: 4443\n              name: https\n              protocol: TCP\n          readinessProbe:\n            failureThreshold: 3\n            httpGet:\n              path: /readyz\n              port: https\n              scheme: HTTPS\n            initialDelaySeconds: 20\n            periodSeconds: 10\n          resources:\n            requests:\n              cpu: 50m\n              memory: 100Mi\n            limits:\n              cpu: 200m\n              memory: 500Mi\n          securityContext:\n            allowPrivilegeEscalation: false\n            readOnlyRootFilesystem: true\n            runAsNonRoot: true\n            runAsUser: 1000\n          volumeMounts:\n            - mountPath: /tmp\n              name: tmp-dir\n      nodeSelector:\n        kubernetes.io/os: linux\n      priorityClassName: system-cluster-critical\n      serviceAccountName: metrics-server\n      volumes:\n        - emptyDir: {}\n          name: tmp-dir\n---\napiVersion: apiregistration.k8s.io/v1\nkind: APIService\nmetadata:\n  labels:\n    k8s-app: metrics-server\n  name: v1beta1.metrics.k8s.io\nspec:\n  group: metrics.k8s.io\n  groupPriorityMinimum: 100\n  insecureSkipTLSVerify: true\n  service:\n    name: metrics-server\n    namespace: kube-system\n  version: v1beta1\n  versionPriority: 100\n"
  },
  {
    "path": "manager/manifests/namespaces.yaml",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: Namespace\nmetadata:\n  name: istio-system\n---\n\napiVersion: v1\nkind: Namespace\nmetadata:\n  name: logging\n---\n\napiVersion: v1\nkind: Namespace\nmetadata:\n  name: prometheus\n---\n"
  },
  {
    "path": "manager/manifests/nvidia.yaml",
    "content": "# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n# Source: https://github.com/NVIDIA/k8s-device-plugin/blob/v0.11.0/nvidia-device-plugin.yml\n\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: nvidia-device-plugin-daemonset\n  namespace: kube-system\nspec:\n  selector:\n    matchLabels:\n      name: nvidia-device-plugin-ds\n  updateStrategy:\n    type: RollingUpdate\n    rollingUpdate:\n      maxUnavailable: 1\n  template:\n    metadata:\n      labels:\n        name: nvidia-device-plugin-ds\n    spec:\n      tolerations:\n        # This toleration is deprecated. Kept here for backward compatibility\n        # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/\n        - key: CriticalAddonsOnly\n          operator: Exists\n        - key: nvidia.com/gpu\n          operator: Exists\n          effect: NoSchedule\n        - key: workload\n          operator: Exists\n          effect: NoSchedule\n      # Mark this pod as a critical add-on; when enabled, the critical add-on\n      # scheduler reserves resources for critical add-on pods so that they can\n      # be rescheduled after a failure.\n      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/\n      priorityClassName: \"system-node-critical\"\n      containers:\n        - image: $CORTEX_IMAGE_NVIDIA_DEVICE_PLUGIN\n          name: nvidia-device-plugin-ctr\n          args: [\"--fail-on-init-error=false\"]\n          securityContext:\n            allowPrivilegeEscalation: false\n            capabilities:\n              drop: [\"ALL\"]\n          volumeMounts:\n            - name: device-plugin\n              mountPath: /var/lib/kubelet/device-plugins\n          resources: # https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml#L44\n            requests:\n              cpu: 100m\n              memory: 100Mi\n            limits:\n              memory: 100Mi\n      nodeSelector:\n        workload: \"true\"\n        nvidia.com/gpu: \"true\"\n      volumes:\n        - name: device-plugin\n          hostPath:\n            path: /var/lib/kubelet/device-plugins\n"
  },
  {
    "path": "manager/manifests/operator.yaml.j2",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: operator\n  namespace: default\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: operator\n  namespace: default\nsubjects:\n  - kind: ServiceAccount\n    name: operator\n    namespace: default\nroleRef:\n  kind: ClusterRole\n  name: cluster-admin\n  apiGroup: rbac.authorization.k8s.io\n\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: operator\n  namespace: default\n  labels:\n    workloadID: operator\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      workloadID: operator\n  template:\n    metadata:\n      labels:\n        workloadID: operator\n    spec:\n      serviceAccountName: operator\n      containers:\n        - name: operator\n          image: {{ config['image_operator'] }}\n          imagePullPolicy: Always\n          resources:\n            requests:\n              cpu: 100m\n              memory: 128Mi\n            limits:\n              cpu: 1500m\n              memory: 2048Mi\n          ports:\n            - containerPort: 8888\n          envFrom:\n            - configMapRef:\n                name: env-vars\n          volumeMounts:\n            - name: cluster-config\n              mountPath: /configs/cluster\n            - name: docker-client\n              mountPath: /var/run/docker.sock\n      volumes:\n        - name: cluster-config\n          configMap:\n            name: cluster-config\n        - name: docker-client\n          hostPath:\n            path: /var/run/docker.sock\n            type: Socket\n\n---\napiVersion: v1\nkind: Service\nmetadata:\n  namespace: default\n  name: operator\n  labels:\n    cortex.dev/name: operator\nspec:\n  selector:\n    workloadID: operator\n  ports:\n    - port: 8888\n      name: http\n\n---\napiVersion: networking.istio.io/v1beta1\nkind: Gateway\nmetadata:\n  name: operator-gateway\n  namespace: default\nspec:\n  selector:\n    istio: ingressgateway-operator\n  servers:\n    - port:\n        number: 80\n        name: http\n        protocol: HTTP\n      hosts:\n        - \"*\"\n    - port:\n        number: 443\n        name: https\n        protocol: HTTPS\n      hosts:\n        - \"*\"\n      tls:\n        mode: SIMPLE\n        serverCertificate: /etc/istio/customgateway-certs/tls.crt\n        privateKey: /etc/istio/customgateway-certs/tls.key\n\n---\napiVersion: networking.istio.io/v1beta1\nkind: VirtualService\nmetadata:\n  name: operator\n  namespace: default\nspec:\n  hosts:\n    - \"*\"\n  gateways:\n    - operator-gateway\n  http:\n    - route:\n        - destination:\n            host: operator\n            port:\n              number: 8888\n"
  },
  {
    "path": "manager/manifests/prometheus-additional-scrape-configs.yaml.j2",
    "content": "# Copyright 2015 The Prometheus Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\n{% if env.get(\"CORTEX_DEV_ADD_CONTROL_PLANE_DASHBOARD\") == \"true\" %}\n- job_name: \"kubernetes-apiservers\"\n  kubernetes_sd_configs:\n    - role: endpoints\n\n  # Default to scraping over https. If required, just disable this or change to\n  # `http`.\n  scheme: https\n\n  # This TLS & authorization config is used to connect to the actual scrape\n  # endpoints for cluster components. This is separate to discovery auth\n  # configuration because discovery & scraping are two separate concerns in\n  # Prometheus. The discovery auth config is automatic if Prometheus runs inside\n  # the cluster. Otherwise, more config options have to be provided within the\n  # <kubernetes_sd_config>.\n  tls_config:\n    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n    # If your node certificates are self-signed or use a different CA to the\n    # master CA, then disable certificate verification below. Note that\n    # certificate verification is an integral part of a secure infrastructure\n    # so this should only be disabled in a controlled environment. You can\n    # disable certificate verification by uncommenting the line below.\n    #\n    # insecure_skip_verify: true\n  authorization:\n    credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token\n\n  # Keep only the default/kubernetes service endpoints for the https port. This\n  # will add targets for each API server which Kubernetes adds an endpoint to\n  # the default/kubernetes service.\n  relabel_configs:\n    - source_labels:\n        [\n          __meta_kubernetes_namespace,\n          __meta_kubernetes_service_name,\n          __meta_kubernetes_endpoint_port_name,\n        ]\n      action: keep\n      regex: default;kubernetes;https\n{% endif %}\n"
  },
  {
    "path": "manager/manifests/prometheus-dcgm-exporter.yaml",
    "content": "# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: dcgm-exporter\n  namespace: prometheus\n  labels:\n    app.kubernetes.io/name: dcgm-exporter\n    app.kubernetes.io/instance: dcgm-exporter\n    app.kubernetes.io/component: dcgm-exporter\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  name: dcgm-exporter\n  namespace: prometheus\n  labels:\n    app.kubernetes.io/name: dcgm-exporter\n    app.kubernetes.io/instance: dcgm-exporter\n    app.kubernetes.io/component: dcgm-exporter\nspec:\n  updateStrategy:\n    type: RollingUpdate\n  selector:\n    matchLabels:\n      app.kubernetes.io/name: dcgm-exporter\n      app.kubernetes.io/instance: dcgm-exporter\n      app.kubernetes.io/component: dcgm-exporter\n  template:\n    metadata:\n      labels:\n        app.kubernetes.io/name: dcgm-exporter\n        app.kubernetes.io/instance: dcgm-exporter\n        app.kubernetes.io/component: dcgm-exporter\n    spec:\n      serviceAccountName: dcgm-exporter\n      affinity:\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n            - matchExpressions:\n              - key: nvidia.com/gpu\n                operator: Exists\n      containers:\n      - env:\n        - name: DCGM_EXPORTER_LISTEN\n          value: :9400\n        - name: DCGM_EXPORTER_KUBERNETES\n          value: \"true\"\n        image: $CORTEX_IMAGE_PROMETHEUS_DCGM_EXPORTER\n        imagePullPolicy: Always\n        name: dcgm-exporter\n        ports:\n        - containerPort: 9400\n          name: metrics\n          protocol: TCP\n        resources:\n          requests:\n            cpu: 50m\n            memory: 50Mi\n        securityContext:\n          capabilities:\n            add:\n            - SYS_ADMIN\n          runAsNonRoot: false\n          runAsUser: 0\n        volumeMounts:\n        - mountPath: /var/lib/kubelet/pod-resources\n          name: pod-gpu-resources\n          readOnly: true\n      tolerations:\n      - key: workload\n        effect: NoSchedule\n        operator: Exists\n      - key: nvidia.com/gpu\n        operator: Exists\n        effect: NoSchedule\n      volumes:\n      - hostPath:\n          path: /var/lib/kubelet/pod-resources\n          type: \"\"\n        name: pod-gpu-resources\n      nodeSelector:\n        workload: \"true\"\n        nvidia.com/gpu: \"$NVIDIA_COM_GPU_VALUE\"\n---\napiVersion: monitoring.coreos.com/v1\nkind: PodMonitor\nmetadata:\n  name: dcgm-exporter\n  namespace: prometheus\n  labels:\n    monitoring.cortex.dev: dcgm-exporter\n    app.kubernetes.io/name: dcgm-exporter\n    app.kubernetes.io/instance: dcgm-exporter\n    app.kubernetes.io/component: dcgm-exporter\n  annotations:\n    prometheus.io/scrape: 'true'\n    prometheus.io/port: '9400'\nspec:\n  jobLabel: \"dcgm-exporter\"\n  podMetricsEndpoints:\n    - port: metrics\n      path: /metrics\n      scheme: http\n      interval: 15s\n      metricRelabelings:\n      - action: keep\n        sourceLabels: [__name__]\n        regex: \"DCGM_FI_DEV_(\\\n          GPU_UTIL|\\\n          FB_USED|\\\n          FB_FREE\\\n          )\"\n      - action: labelkeep\n        regex: (__name__|exported_pod)\n  namespaceSelector:\n    any: true\n  selector:\n    matchLabels:\n      app.kubernetes.io/name: dcgm-exporter\n      app.kubernetes.io/instance: dcgm-exporter\n"
  },
  {
    "path": "manager/manifests/prometheus-kube-state-metrics.yaml",
    "content": "# Copyright 2021 The Kubernetes Authors All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  labels:\n    app.kubernetes.io/name: kube-state-metrics\n  name: kube-state-metrics\n  namespace: prometheus\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  labels:\n    app.kubernetes.io/name: kube-state-metrics\n  name: kube-state-metrics\nrules:\n\n- apiGroups: [\"certificates.k8s.io\"]\n  resources:\n  - certificatesigningrequests\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - configmaps\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"batch\"]\n  resources:\n  - cronjobs\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"extensions\", \"apps\"]\n  resources:\n  - daemonsets\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"extensions\", \"apps\"]\n  resources:\n  - deployments\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - endpoints\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"autoscaling\"]\n  resources:\n  - horizontalpodautoscalers\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"extensions\", \"networking.k8s.io\"]\n  resources:\n  - ingresses\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"batch\"]\n  resources:\n  - jobs\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - limitranges\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"admissionregistration.k8s.io\"]\n  resources:\n    - mutatingwebhookconfigurations\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - namespaces\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"networking.k8s.io\"]\n  resources:\n  - networkpolicies\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - nodes\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - persistentvolumeclaims\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - persistentvolumes\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"policy\"]\n  resources:\n    - poddisruptionbudgets\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - pods\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"extensions\", \"apps\"]\n  resources:\n  - replicasets\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - replicationcontrollers\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - resourcequotas\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - secrets\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"\"]\n  resources:\n  - services\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"apps\"]\n  resources:\n  - statefulsets\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"storage.k8s.io\"]\n  resources:\n    - storageclasses\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"admissionregistration.k8s.io\"]\n  resources:\n    - validatingwebhookconfigurations\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"storage.k8s.io\"]\n  resources:\n    - volumeattachments\n  verbs: [\"list\", \"watch\"]\n\n- apiGroups: [\"autoscaling.k8s.io\"]\n  resources:\n    - verticalpodautoscalers\n  verbs: [\"list\", \"watch\"]\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  labels:\n    app.kubernetes.io/name: kube-state-metrics\n  name: kube-state-metrics\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: kube-state-metrics\nsubjects:\n- kind: ServiceAccount\n  name: kube-state-metrics\n  namespace: prometheus\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: kube-state-metrics\n  namespace: prometheus\n  labels:\n    app.kubernetes.io/name: kube-state-metrics\n    app.kubernetes.io/version: \"2.1.0\"\nspec:\n  selector:\n    matchLabels:\n      app.kubernetes.io/name: kube-state-metrics\n  replicas: 1\n  template:\n    metadata:\n      labels:\n        app.kubernetes.io/name: kube-state-metrics\n    spec:\n      hostNetwork: false\n      serviceAccountName: kube-state-metrics\n      securityContext:\n        fsGroup: 65534\n        runAsGroup: 65534\n        runAsUser: 65534\n      containers:\n      - name: kube-state-metrics\n        resources:\n          requests:\n            cpu: 300m\n            memory: 400Mi\n        args:\n        - --port=8080\n        - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,verticalpodautoscalers,volumeattachments\n        - --telemetry-port=8081\n        imagePullPolicy: Always\n        image: $CORTEX_IMAGE_PROMETHEUS_KUBE_STATE_METRICS\n        ports:\n        - containerPort: 8080\n          name: metrics\n          protocol: TCP\n        livenessProbe:\n          httpGet:\n            path: /healthz\n            port: 8080\n          initialDelaySeconds: 5\n          timeoutSeconds: 5\n        readinessProbe:\n          httpGet:\n            path: /\n            port: 8080\n          initialDelaySeconds: 5\n          timeoutSeconds: 5\n      nodeSelector:\n        prometheus: \"true\"\n      tolerations:\n        - key: prometheus\n          operator: Exists\n          effect: NoSchedule\n---\napiVersion: monitoring.coreos.com/v1\nkind: PodMonitor\nmetadata:\n  name: kube-state-metrics\n  namespace: prometheus\n  labels:\n    name: kube-state-metrics\n    monitoring.cortex.dev: kube-state-metrics\nspec:\n  jobLabel: \"kube-state-metrics\"\n  podMetricsEndpoints:\n    - port: metrics\n      scheme: http\n      path: /metrics\n      interval: 30s\n      metricRelabelings:\n      - action: keep\n        sourceLabels: [__name__]\n        regex: \"kube_(\\\n          pod_container_resource_requests|\\\n          pod_info|\\\n          deployment_status_replicas_available|\\\n          job_status_active\\\n          )\"\n      - action: labelkeep\n        regex: (__name__|exported_pod|exported_container|job_name|resource|deployment)\n  namespaceSelector:\n    any: true\n  selector:\n    matchLabels:\n      app.kubernetes.io/name: kube-state-metrics\n"
  },
  {
    "path": "manager/manifests/prometheus-kubelet-exporter.yaml",
    "content": "# Copyright 2016 The prometheus-operator Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n  labels:\n    k8s-app: kubelet\n    monitoring.cortex.dev: kubelet-exporter\n  name: kubelet\n  namespace: prometheus\nspec:\n  endpoints:\n  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\n    honorLabels: true\n    interval: 30s\n    metricRelabelings:\n    - action: drop\n      sourceLabels: [__name__]\n    port: https-metrics\n    relabelings:\n    - sourceLabels:\n      - __metrics_path__\n      targetLabel: metrics_path\n    scheme: https\n    tlsConfig:\n      insecureSkipVerify: true\n  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\n    honorLabels: true\n    honorTimestamps: false\n    interval: 30s\n    metricRelabelings:\n    - action: keep\n      sourceLabels: [__name__]\n      regex: \"container_(\\\n        cpu_usage_seconds_total|\\\n        memory_working_set_bytes\\\n        )\"\n    - action: labelkeep\n      regex: (__name__|pod|container|name)\n    path: /metrics/cadvisor\n    port: https-metrics\n    relabelings:\n    - sourceLabels:\n      - __metrics_path__\n      targetLabel: metrics_path\n    scheme: https\n    tlsConfig:\n      insecureSkipVerify: true\n  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\n    honorLabels: true\n    interval: 30s\n    path: /metrics/probes\n    port: https-metrics\n    relabelings:\n    - sourceLabels:\n      - __metrics_path__\n      targetLabel: metrics_path\n    metricRelabelings:\n    - action: drop\n      sourceLabels: [__name__]\n    scheme: https\n    tlsConfig:\n      insecureSkipVerify: true\n  jobLabel: k8s-app\n  namespaceSelector:\n    matchNames:\n    - kube-system\n  selector:\n    matchLabels:\n      k8s-app: kubelet\n"
  },
  {
    "path": "manager/manifests/prometheus-monitoring.yaml",
    "content": "# Copyright 2016 The prometheus-operator Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\napiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n  name: ssd\nvolumeBindingMode: WaitForFirstConsumer\nprovisioner: kubernetes.io/aws-ebs\nparameters:\n  type: gp2\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: Prometheus\nmetadata:\n  name: prometheus\n  namespace: prometheus\nspec:\n  image: $CORTEX_IMAGE_PROMETHEUS\n  serviceAccountName: prometheus\n  nodeSelector:\n    prometheus: \"true\"\n  tolerations:\n    - key: prometheus\n      operator: Exists\n      effect: NoSchedule\n  podMonitorSelector:\n    matchExpressions:\n      - key: \"monitoring.cortex.dev\"\n        operator: \"Exists\"\n  serviceMonitorSelector:\n    matchExpressions:\n      - key: \"monitoring.cortex.dev\"\n        operator: \"Exists\"\n  ruleSelector:\n    matchLabels:\n      prometheus: k8s\n  resources:\n    requests:\n      memory: 400Mi\n  enableAdminAPI: false\n  storage:\n    volumeClaimTemplate:\n      spec:\n        storageClassName: ssd\n        resources:\n          requests:\n            storage: 40Gi\n  additionalScrapeConfigs:\n    name: additional-scrape-configs\n    key: prometheus-additional-scrape-configs.yaml\n  retention: 2w\n  retentionSize: 35GB\n  securityContext:\n    fsGroup: 2000\n    runAsNonRoot: true\n    runAsUser: 1000\n---\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: prometheus\n  namespace: prometheus\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: prometheus\nrules:\n  - apiGroups: [ \"\" ]\n    resources:\n      - nodes\n      - nodes/metrics\n      - services\n      - endpoints\n      - pods\n    verbs: [ \"get\", \"list\", \"watch\" ]\n  - apiGroups: [ \"\" ]\n    resources:\n      - configmaps\n    verbs: [ \"get\" ]\n  - apiGroups:\n      - networking.k8s.io\n    resources:\n      - ingresses\n    verbs: [ \"get\", \"list\", \"watch\" ]\n  - nonResourceURLs: [ \"/metrics\" ]\n    verbs: [ \"get\" ]\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: prometheus\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: prometheus\nsubjects:\n  - kind: ServiceAccount\n    name: prometheus\n    namespace: prometheus\n\n---\n\napiVersion: v1\nkind: Service\nmetadata:\n  name: prometheus\n  namespace: prometheus\nspec:\n  type: ClusterIP\n  ports:\n    - port: 9090\n      targetPort: 9090\n  selector:\n    prometheus: prometheus\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: PodMonitor\nmetadata:\n  name: istio-ingress-stats\n  namespace: prometheus\n  labels:\n    monitoring.cortex.dev: \"istio\"\nspec:\n  selector:\n    matchExpressions:\n      - { key: prometheus-ignore, operator: DoesNotExist }\n      - { key: istio, operator: Exists }\n      - { key: release, operator: In, values: [ \"istio\" ]}\n  namespaceSelector:\n    any: true\n  jobLabel: istio-ingress-stats\n  podMetricsEndpoints:\n    - path: /stats/prometheus\n      interval: 15s\n      relabelings:\n        - action: keep\n          sourceLabels: [ __meta_kubernetes_pod_container_name ]\n          regex: \"istio-proxy\"\n        - action: keep\n          sourceLabels: [ __meta_kubernetes_pod_annotationpresent_prometheus_io_scrape ]\n        - sourceLabels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ]\n          action: replace\n          regex: ([^:]+)(?::\\d+)?;(\\d+)\n          replacement: $1:$2\n          targetLabel: __address__\n        - action: labeldrop\n          regex: \"__meta_kubernetes_pod_label_(.+)\"\n        - sourceLabels: [ __meta_kubernetes_namespace ]\n          action: replace\n          targetLabel: namespace\n        - sourceLabels: [ __meta_kubernetes_pod_name ]\n          action: replace\n          targetLabel: pod_name\n        - sourceLabels: [ __name__, __meta_kubernetes_pod_label_istio, __meta_kubernetes_pod_name ]\n          action: replace\n          regex: (istio_requests_total)?;(ingressgateway-apis)?;(.+)\n          replacement: $3\n          targetLabel: istioingress_podname\n      metricRelabelings:\n        - action: keep\n          sourceLabels: [__name__]\n          regex: \"istio_(\\\n            requests_total|\\\n            request_duration_milliseconds_bucket|\\\n            request_duration_milliseconds_sum|\\\n            request_duration_milliseconds_count\\\n            )\"\n        - action: labelkeep\n          regex: (__name__|destination_service|response_code|le|istioingress_podname)\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: PodMonitor\nmetadata:\n  name: proxy-stats\n  namespace: prometheus\n  labels:\n    monitoring.cortex.dev: \"proxy\"\nspec:\n  selector:\n    matchLabels:\n      apiKind: RealtimeAPI\n    matchExpressions:\n      - { key: prometheus-ignore, operator: DoesNotExist }\n  namespaceSelector:\n    any: true\n  jobLabel: proxy-stats\n  podMetricsEndpoints:\n    - path: /metrics\n      scheme: http\n      interval: 10s\n      port: admin\n      relabelings:\n        - action: keep\n          sourceLabels: [ __meta_kubernetes_pod_container_name ]\n          regex: \"proxy\"\n        - sourceLabels: [ __meta_kubernetes_pod_label_apiName ]\n          action: replace\n          targetLabel: api_name\n        - sourceLabels: [ __meta_kubernetes_pod_label_apiKind ]\n          action: replace\n          targetLabel: api_kind\n        - sourceLabels: [ __meta_kubernetes_pod_label_apiID ]\n          action: replace\n          targetLabel: api_id\n        - sourceLabels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ]\n          action: replace\n          regex: ([^:]+)(?::\\d+)?;(\\d+)\n          replacement: $1:$2\n          targetLabel: __address__\n        - action: labeldrop\n          regex: \"__meta_kubernetes_pod_label_(.+)\"\n        - sourceLabels: [ __meta_kubernetes_namespace ]\n          action: replace\n          targetLabel: namespace\n        - sourceLabels: [ __meta_kubernetes_pod_name ]\n          action: replace\n          targetLabel: pod_name\n      metricRelabelings:\n        - action: keep\n          sourceLabels: [__name__]\n          regex: \"cortex_(.+)\"\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: PodMonitor\nmetadata:\n  name: async-stats\n  namespace: prometheus\n  labels:\n    monitoring.cortex.dev: \"dequeuer-async\"\nspec:\n  selector:\n    matchLabels:\n      apiKind: AsyncAPI\n    matchExpressions:\n      - { key: prometheus-ignore, operator: DoesNotExist }\n  namespaceSelector:\n    any: true\n  jobLabel: async-stats\n  podMetricsEndpoints:\n    - path: /metrics\n      scheme: http\n      interval: 10s\n      port: admin\n      relabelings:\n        - action: keep\n          sourceLabels: [ __meta_kubernetes_pod_container_name ]\n          regex: \"dequeuer\"\n        - sourceLabels: [ __meta_kubernetes_pod_label_apiName ]\n          action: replace\n          targetLabel: api_name\n        - sourceLabels: [ __meta_kubernetes_pod_label_apiKind ]\n          action: replace\n          targetLabel: api_kind\n        - sourceLabels: [ __meta_kubernetes_pod_label_apiID ]\n          action: replace\n          targetLabel: api_id\n        - sourceLabels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ]\n          action: replace\n          regex: ([^:]+)(?::\\d+)?;(\\d+)\n          replacement: $1:$2\n          targetLabel: __address__\n        - action: labeldrop\n          regex: \"__meta_kubernetes_pod_label_(.+)\"\n        - sourceLabels: [ __meta_kubernetes_namespace ]\n          action: replace\n          targetLabel: namespace\n        - sourceLabels: [ __meta_kubernetes_pod_name ]\n          action: replace\n          targetLabel: pod_name\n      metricRelabelings:\n        - action: keep\n          sourceLabels: [__name__]\n          regex: \"cortex_(.+)\"\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: PodMonitor\nmetadata:\n  name: prometheus-statsd-exporter\n  namespace: prometheus\n  labels:\n    name: prometheus-statsd-exporter\n    monitoring.cortex.dev: \"statsd-exporter\"\nspec:\n  jobLabel: \"statsd-exporter\"\n  podMetricsEndpoints:\n    - port: metrics\n      scheme: http\n      path: /metrics\n      interval: 20s\n      metricRelabelings:\n        - action: keep\n          sourceLabels: [__name__]\n          regex: \"cortex_(.+)\"\n  namespaceSelector:\n    any: true\n  selector:\n    matchLabels:\n      name: prometheus-statsd-exporter\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n  name: operator\n  namespace: prometheus\n  labels:\n    name: operator\n    monitoring.cortex.dev: \"operator\"\nspec:\n  jobLabel: \"operator\"\n  endpoints:\n    - port: http\n      scheme: http\n      path: /metrics\n      interval: 10s\n      metricRelabelings:\n        - action: keep\n          sourceLabels: [__name__]\n          regex: \"cortex_(.+)\"\n        - action: labeldrop\n          regex: (container|endpoint|instance|job|namespace|pod|service)\n  namespaceSelector:\n    any: true\n  selector:\n    matchLabels:\n      cortex.dev/name: operator\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: PodMonitor\nmetadata:\n  name: activator-stats\n  labels:\n    monitoring.cortex.dev: \"activator\"\nspec:\n  selector:\n    matchLabels:\n      app: activator\n    matchExpressions:\n      - { key: prometheus-ignore, operator: DoesNotExist }\n  namespaceSelector:\n    any: true\n  jobLabel: activator-stats\n  podMetricsEndpoints:\n    - path: /metrics\n      scheme: http\n      interval: 10s\n      port: admin\n      relabelings:\n        - action: keep\n          sourceLabels: [ __meta_kubernetes_pod_container_name ]\n          regex: \"activator\"\n        - sourceLabels: [ __address__, __meta_kubernetes_pod_annotation_prometheus_io_port ]\n          action: replace\n          regex: ([^:]+)(?::\\d+)?;(\\d+)\n          replacement: $1:$2\n          targetLabel: __address__\n        - action: labeldrop\n          regex: \"__meta_kubernetes_pod_label_(.+)\"\n        - sourceLabels: [ __meta_kubernetes_namespace ]\n          action: replace\n          targetLabel: namespace\n        - sourceLabels: [ __meta_kubernetes_pod_name ]\n          action: replace\n          targetLabel: pod_name\n      metricRelabelings:\n        - action: keep\n          sourceLabels: [__name__]\n          regex: \"cortex_(.+)\"\n"
  },
  {
    "path": "manager/manifests/prometheus-node-exporter.yaml",
    "content": "# Copyright 2016 The prometheus-operator Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  labels:\n    app.kubernetes.io/version: v1.3.1\n  name: node-exporter\n  namespace: prometheus\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  labels:\n    app.kubernetes.io/version: v1.3.1\n  name: node-exporter\nrules:\n  - apiGroups:\n      - authentication.k8s.io\n    resources:\n      - tokenreviews\n    verbs:\n      - create\n  - apiGroups:\n      - authorization.k8s.io\n    resources:\n      - subjectaccessreviews\n    verbs:\n      - create\n\n---\n\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  labels:\n    app.kubernetes.io/version: v1.3.1\n  name: node-exporter\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: node-exporter\nsubjects:\n  - kind: ServiceAccount\n    name: node-exporter\n    namespace: prometheus\n\n---\n\napiVersion: v1\nkind: Service\nmetadata:\n  labels:\n    app.kubernetes.io/name: node-exporter\n    app.kubernetes.io/version: v1.3.1\n  name: node-exporter\n  namespace: prometheus\nspec:\n  clusterIP: None\n  ports:\n    - name: https\n      port: 9100\n      targetPort: https\n  selector:\n    app.kubernetes.io/name: node-exporter\n\n---\n\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n  labels:\n    app.kubernetes.io/name: node-exporter\n    app.kubernetes.io/version: v1.3.1\n  name: node-exporter\n  namespace: prometheus\nspec:\n  selector:\n    matchLabels:\n      app.kubernetes.io/name: node-exporter\n  template:\n    metadata:\n      labels:\n        app.kubernetes.io/name: node-exporter\n        app.kubernetes.io/version: v1.3.1\n    spec:\n      containers:\n        - args:\n            - --web.listen-address=127.0.0.1:9100\n            - --path.sysfs=/host/sys\n            - --path.rootfs=/host/root\n            - --no-collector.wifi\n            - --no-collector.hwmon\n            - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)\n            - --collector.netclass.ignored-devices=^(veth.*)$\n            - --collector.netdev.device-exclude=^(veth.*)$\n          image: $CORTEX_IMAGE_PROMETHEUS_NODE_EXPORTER\n          name: node-exporter\n          resources:\n            limits:\n              cpu: 250m\n              memory: 180Mi\n            requests:\n              cpu: 40m\n              memory: 180Mi\n          volumeMounts:\n            - mountPath: /host/sys\n              mountPropagation: HostToContainer\n              name: sys\n              readOnly: true\n            - mountPath: /host/root\n              mountPropagation: HostToContainer\n              name: root\n              readOnly: true\n        - args:\n            - --logtostderr\n            - --secure-listen-address=[$(IP)]:9100\n            - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305\n            - --upstream=http://127.0.0.1:9100/\n          env:\n            - name: IP\n              valueFrom:\n                fieldRef:\n                  fieldPath: status.podIP\n          image: $CORTEX_IMAGE_KUBE_RBAC_PROXY\n          name: kube-rbac-proxy\n          ports:\n            - containerPort: 9100\n              hostPort: 9100\n              name: https\n          resources:\n            limits:\n              cpu: 20m\n              memory: 40Mi\n            requests:\n              cpu: 10m\n              memory: 20Mi\n      hostNetwork: true\n      hostPID: true\n      nodeSelector:\n        kubernetes.io/os: linux\n      securityContext:\n        runAsNonRoot: true\n        runAsUser: 65534\n      serviceAccountName: node-exporter\n      tolerations:\n        - operator: Exists\n      volumes:\n        - hostPath:\n            path: /sys\n          name: sys\n        - hostPath:\n            path: /\n          name: root\n  updateStrategy:\n    rollingUpdate:\n      maxUnavailable: 10%\n    type: RollingUpdate\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n  labels:\n    app.kubernetes.io/name: node-exporter\n    app.kubernetes.io/version: v1.3.1\n    monitoring.cortex.dev: node-exporter\n  name: node-exporter\n  namespace: prometheus\nspec:\n  endpoints:\n    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\n      interval: 15s\n      port: https\n      relabelings:\n        - action: replace\n          regex: (.*)\n          replacement: $1\n          sourceLabels:\n            - __meta_kubernetes_pod_node_name\n          targetLabel: instance\n      metricRelabelings:\n        - action: keep\n          sourceLabels: [__name__]\n          regex: \"node_(\\\n            cpu_seconds_total|\\\n            load1|\\\n            load5|\\\n            load15|\\\n            exporter_build_info|\\\n            memory_MemTotal_bytes|\\\n            memory_MemFree_bytes|\\\n            memory_Buffers_bytes|\\\n            memory_Cached_bytes|\\\n            memory_MemAvailable_bytes|\\\n            disk_read_bytes_total|\\\n            disk_written_bytes_total|\\\n            disk_io_time_seconds_total|\\\n            disk_io_time_weighted_seconds_total|\\\n            filesystem_size_bytes|\\\n            filesystem_avail_bytes|\\\n            network_receive_bytes_total|\\\n            network_transmit_bytes_total|\\\n            network_receive_drop_total|\\\n            network_transmit_drop_total|\\\n            vmstat_pgmajfault\\\n            )\"\n        - action: labelkeep\n          regex: (__name__|instance|job|device|fstype|mountpoint|mode)\n      scheme: https\n      tlsConfig:\n        insecureSkipVerify: true\n  jobLabel: app.kubernetes.io/name\n  selector:\n    matchLabels:\n      app.kubernetes.io/name: node-exporter\n\n---\n\napiVersion: monitoring.coreos.com/v1\nkind: PrometheusRule\nmetadata:\n  labels:\n    app.kubernetes.io/name: node-exporter\n    app.kubernetes.io/version: 1.1.2\n    prometheus: k8s\n  name: node-exporter-rules\n  namespace: prometheus\nspec:\n  groups:\n    - name: node-exporter.rules\n      rules:\n        - expr: |\n            count without (cpu) (\n              count without (mode) (\n                node_cpu_seconds_total{job=\"node-exporter\"}\n              )\n            )\n          record: instance:node_num_cpu:sum\n        - expr: |\n            1 - avg without (cpu, mode) (\n              rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[1m])\n            )\n          record: instance:node_cpu_utilisation:rate1m\n        - expr: |\n            (\n              node_load1{job=\"node-exporter\"}\n            /\n              instance:node_num_cpu:sum{job=\"node-exporter\"}\n            )\n          record: instance:node_load1_per_cpu:ratio\n        - expr: |\n            1 - (\n              node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n            /\n              node_memory_MemTotal_bytes{job=\"node-exporter\"}\n            )\n          record: instance:node_memory_utilisation:ratio\n        - expr: |\n            rate(node_vmstat_pgmajfault{job=\"node-exporter\"}[1m])\n          record: instance:node_vmstat_pgmajfault:rate1m\n        - expr: |\n            rate(node_disk_io_time_seconds_total{job=\"node-exporter\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[1m])\n          record: instance_device:node_disk_io_time_seconds:rate1m\n        - expr: |\n            rate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[1m])\n          record: instance_device:node_disk_io_time_weighted_seconds:rate1m\n        - expr: |\n            sum without (device) (\n              rate(node_network_receive_bytes_total{job=\"node-exporter\", device!=\"lo\"}[1m])\n            )\n          record: instance:node_network_receive_bytes_excluding_lo:rate1m\n        - expr: |\n            sum without (device) (\n              rate(node_network_transmit_bytes_total{job=\"node-exporter\", device!=\"lo\"}[1m])\n            )\n          record: instance:node_network_transmit_bytes_excluding_lo:rate1m\n        - expr: |\n            sum without (device) (\n              rate(node_network_receive_drop_total{job=\"node-exporter\", device!=\"lo\"}[1m])\n            )\n          record: instance:node_network_receive_drop_excluding_lo:rate1m\n        - expr: |\n            sum without (device) (\n              rate(node_network_transmit_drop_total{job=\"node-exporter\", device!=\"lo\"}[1m])\n            )\n          record: instance:node_network_transmit_drop_excluding_lo:rate1m\n        - expr: |\n            sum (\n              (\n                instance:node_cpu_utilisation:rate1m{job=\"node-exporter\"}\n              *\n                instance:node_num_cpu:sum{job=\"node-exporter\"}\n              )\n              / scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\"}))\n            )\n          record: cluster:cpu_utilization:ratio\n        - expr: |\n            sum (\n              instance:node_load1_per_cpu:ratio{job=\"node-exporter\"}\n              / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\"}))\n            )\n          record: cluster:load1:ratio\n        - expr: |\n            sum (\n              instance:node_memory_utilisation:ratio{job=\"node-exporter\"}\n              / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\"}))\n            )\n          record: cluster:memory_utilization:ratio\n        - expr: |\n            sum (\n              instance:node_vmstat_pgmajfault:rate1m{job=\"node-exporter\"}\n            )\n          record: cluster:vmstat_pgmajfault:rate1m\n        - expr: |\n            sum (instance:node_network_receive_bytes_excluding_lo:rate1m{job=\"node-exporter\"})\n          record: cluster:network_receive_bytes_excluding_low:rate1m\n        - expr: |\n            sum (instance:node_network_transmit_bytes_excluding_lo:rate1m{job=\"node-exporter\"})\n          record: cluster:network_transmit_bytes_excluding_lo:rate1m\n        - expr: |\n            sum (instance:node_network_receive_drop_excluding_lo:rate1m{job=\"node-exporter\"})\n          record: cluster:network_receive_drop_excluding_lo:rate1m\n        - expr: |\n            sum (instance:node_network_transmit_drop_excluding_lo:rate1m{job=\"node-exporter\"})\n          record: cluster:network_transmit_drop_excluding_lo:rate1m\n        - expr: |\n            sum (\n              instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\"}\n              / scalar(count(instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\"}))\n            )\n          record: cluster:disk_io_utilization:ratio\n        - expr: |\n            sum (\n              instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\"}\n              / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\"}))\n            )\n          record: cluster:disk_io_saturation:ratio\n        - expr: |\n            sum (\n              max without (fstype, mountpoint) (\n                node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\"} - node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\"}\n              )\n            )\n            / scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\"})))\n          record: cluster:disk_space_utilization:ratio\n"
  },
  {
    "path": "manager/manifests/prometheus-operator.yaml",
    "content": "# Copyright 2016 The prometheus-operator Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n# Source: https://github.com/prometheus-operator/prometheus-operator/blob/release-0.44/bundle.yaml\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: alertmanagerconfigs.monitoring.coreos.com\nspec:\n  group: monitoring.coreos.com\n  names:\n    categories:\n    - prometheus-operator\n    kind: AlertmanagerConfig\n    listKind: AlertmanagerConfigList\n    plural: alertmanagerconfigs\n    singular: alertmanagerconfig\n  scope: Namespaced\n  versions:\n  - name: v1alpha1\n    schema:\n      openAPIV3Schema:\n        description: AlertmanagerConfig defines a namespaced AlertmanagerConfig to be aggregated across multiple namespaces configuring one Alertmanager cluster.\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: AlertmanagerConfigSpec is a specification of the desired behavior of the Alertmanager configuration. By definition, the Alertmanager configuration only applies to alerts for which the `namespace` label is equal to the namespace of the AlertmanagerConfig resource.\n            properties:\n              inhibitRules:\n                description: List of inhibition rules. The rules will only apply to alerts matching the resource’s namespace.\n                items:\n                  description: InhibitRule defines an inhibition rule that allows to mute alerts when other alerts are already firing. See https://prometheus.io/docs/alerting/latest/configuration/#inhibit_rule\n                  properties:\n                    equal:\n                      description: Labels that must have an equal value in the source and target alert for the inhibition to take effect.\n                      items:\n                        type: string\n                      type: array\n                    sourceMatch:\n                      description: Matchers for which one or more alerts have to exist for the inhibition to take effect. The operator enforces that the alert matches the resource’s namespace.\n                      items:\n                        description: Matcher defines how to match on alert's labels.\n                        properties:\n                          name:\n                            description: Label to match.\n                            minLength: 1\n                            type: string\n                          regex:\n                            description: Whether to match on equality (false) or regular-expression (true).\n                            type: boolean\n                          value:\n                            description: Label value to match.\n                            type: string\n                        required:\n                        - name\n                        type: object\n                      type: array\n                    targetMatch:\n                      description: Matchers that have to be fulfilled in the alerts to be muted. The operator enforces that the alert matches the resource’s namespace.\n                      items:\n                        description: Matcher defines how to match on alert's labels.\n                        properties:\n                          name:\n                            description: Label to match.\n                            minLength: 1\n                            type: string\n                          regex:\n                            description: Whether to match on equality (false) or regular-expression (true).\n                            type: boolean\n                          value:\n                            description: Label value to match.\n                            type: string\n                        required:\n                        - name\n                        type: object\n                      type: array\n                  type: object\n                type: array\n              receivers:\n                description: List of receivers.\n                items:\n                  description: Receiver defines one or more notification integrations.\n                  properties:\n                    emailConfigs:\n                      description: List of Email configurations.\n                      items:\n                        description: EmailConfig configures notifications via Email.\n                        properties:\n                          authIdentity:\n                            description: The identity to use for authentication.\n                            type: string\n                          authPassword:\n                            description: The secret's key that contains the password to use for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          authSecret:\n                            description: The secret's key that contains the CRAM-MD5 secret. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          authUsername:\n                            description: The username to use for authentication.\n                            type: string\n                          from:\n                            description: The sender address.\n                            type: string\n                          headers:\n                            description: Further headers email header key/value pairs. Overrides any headers previously set by the notification implementation.\n                            items:\n                              description: KeyValue defines a (key, value) tuple.\n                              properties:\n                                key:\n                                  description: Key of the tuple.\n                                  minLength: 1\n                                  type: string\n                                value:\n                                  description: Value of the tuple.\n                                  type: string\n                              required:\n                              - key\n                              - value\n                              type: object\n                            type: array\n                          hello:\n                            description: The hostname to identify to the SMTP server.\n                            type: string\n                          html:\n                            description: The HTML body of the email notification.\n                            type: string\n                          requireTLS:\n                            description: The SMTP TLS requirement. Note that Go does not support unencrypted connections to remote SMTP endpoints.\n                            type: boolean\n                          sendResolved:\n                            description: Whether or not to notify about resolved alerts.\n                            type: boolean\n                          smarthost:\n                            description: The SMTP host through which emails are sent.\n                            type: string\n                          text:\n                            description: The text body of the email notification.\n                            type: string\n                          tlsConfig:\n                            description: TLS configuration\n                            properties:\n                              ca:\n                                description: Struct containing the CA cert to use for the targets.\n                                properties:\n                                  configMap:\n                                    description: ConfigMap containing data to use for the targets.\n                                    properties:\n                                      key:\n                                        description: The key to select.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the ConfigMap or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  secret:\n                                    description: Secret containing data to use for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              cert:\n                                description: Struct containing the client cert file for the targets.\n                                properties:\n                                  configMap:\n                                    description: ConfigMap containing data to use for the targets.\n                                    properties:\n                                      key:\n                                        description: The key to select.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the ConfigMap or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  secret:\n                                    description: Secret containing data to use for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              insecureSkipVerify:\n                                description: Disable target certificate validation.\n                                type: boolean\n                              keySecret:\n                                description: Secret containing the client key file for the targets.\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              serverName:\n                                description: Used to verify the hostname for the targets.\n                                type: string\n                            type: object\n                          to:\n                            description: The email address to send notifications to.\n                            type: string\n                        type: object\n                      type: array\n                    name:\n                      description: Name of the receiver. Must be unique across all items from the list.\n                      minLength: 1\n                      type: string\n                    opsgenieConfigs:\n                      description: List of OpsGenie configurations.\n                      items:\n                        description: OpsGenieConfig configures notifications via OpsGenie. See https://prometheus.io/docs/alerting/latest/configuration/#opsgenie_config\n                        properties:\n                          apiKey:\n                            description: The secret's key that contains the OpsGenie API key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          apiURL:\n                            description: The URL to send OpsGenie API requests to.\n                            type: string\n                          description:\n                            description: Description of the incident.\n                            type: string\n                          details:\n                            description: A set of arbitrary key/value pairs that provide further detail about the incident.\n                            items:\n                              description: KeyValue defines a (key, value) tuple.\n                              properties:\n                                key:\n                                  description: Key of the tuple.\n                                  minLength: 1\n                                  type: string\n                                value:\n                                  description: Value of the tuple.\n                                  type: string\n                              required:\n                              - key\n                              - value\n                              type: object\n                            type: array\n                          httpConfig:\n                            description: HTTP client configuration.\n                            properties:\n                              basicAuth:\n                                description: BasicAuth for the client.\n                                properties:\n                                  password:\n                                    description: The secret in the service monitor namespace that contains the password for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  username:\n                                    description: The secret in the service monitor namespace that contains the username for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              bearerTokenSecret:\n                                description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              proxyURL:\n                                description: Optional proxy URL.\n                                type: string\n                              tlsConfig:\n                                description: TLS configuration for the client.\n                                properties:\n                                  ca:\n                                    description: Struct containing the CA cert to use for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  cert:\n                                    description: Struct containing the client cert file for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  insecureSkipVerify:\n                                    description: Disable target certificate validation.\n                                    type: boolean\n                                  keySecret:\n                                    description: Secret containing the client key file for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  serverName:\n                                    description: Used to verify the hostname for the targets.\n                                    type: string\n                                type: object\n                            type: object\n                          message:\n                            description: Alert text limited to 130 characters.\n                            type: string\n                          note:\n                            description: Additional alert note.\n                            type: string\n                          priority:\n                            description: Priority level of alert. Possible values are P1, P2, P3, P4, and P5.\n                            type: string\n                          responders:\n                            description: List of responders responsible for notifications.\n                            items:\n                              description: OpsGenieConfigResponder defines a responder to an incident. One of `id`, `name` or `username` has to be defined.\n                              properties:\n                                id:\n                                  description: ID of the responder.\n                                  type: string\n                                name:\n                                  description: Name of the responder.\n                                  type: string\n                                type:\n                                  description: Type of responder.\n                                  minLength: 1\n                                  type: string\n                                username:\n                                  description: Username of the responder.\n                                  type: string\n                              required:\n                              - type\n                              type: object\n                            type: array\n                          sendResolved:\n                            description: Whether or not to notify about resolved alerts.\n                            type: boolean\n                          source:\n                            description: Backlink to the sender of the notification.\n                            type: string\n                          tags:\n                            description: Comma separated list of tags attached to the notifications.\n                            type: string\n                        type: object\n                      type: array\n                    pagerdutyConfigs:\n                      description: List of PagerDuty configurations.\n                      items:\n                        description: PagerDutyConfig configures notifications via PagerDuty. See https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config\n                        properties:\n                          class:\n                            description: The class/type of the event.\n                            type: string\n                          client:\n                            description: Client identification.\n                            type: string\n                          clientURL:\n                            description: Backlink to the sender of notification.\n                            type: string\n                          component:\n                            description: The part or component of the affected system that is broken.\n                            type: string\n                          description:\n                            description: Description of the incident.\n                            type: string\n                          details:\n                            description: Arbitrary key/value pairs that provide further detail about the incident.\n                            items:\n                              description: KeyValue defines a (key, value) tuple.\n                              properties:\n                                key:\n                                  description: Key of the tuple.\n                                  minLength: 1\n                                  type: string\n                                value:\n                                  description: Value of the tuple.\n                                  type: string\n                              required:\n                              - key\n                              - value\n                              type: object\n                            type: array\n                          group:\n                            description: A cluster or grouping of sources.\n                            type: string\n                          httpConfig:\n                            description: HTTP client configuration.\n                            properties:\n                              basicAuth:\n                                description: BasicAuth for the client.\n                                properties:\n                                  password:\n                                    description: The secret in the service monitor namespace that contains the password for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  username:\n                                    description: The secret in the service monitor namespace that contains the username for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              bearerTokenSecret:\n                                description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              proxyURL:\n                                description: Optional proxy URL.\n                                type: string\n                              tlsConfig:\n                                description: TLS configuration for the client.\n                                properties:\n                                  ca:\n                                    description: Struct containing the CA cert to use for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  cert:\n                                    description: Struct containing the client cert file for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  insecureSkipVerify:\n                                    description: Disable target certificate validation.\n                                    type: boolean\n                                  keySecret:\n                                    description: Secret containing the client key file for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  serverName:\n                                    description: Used to verify the hostname for the targets.\n                                    type: string\n                                type: object\n                            type: object\n                          routingKey:\n                            description: The secret's key that contains the PagerDuty integration key (when using Events API v2). Either this field or `serviceKey` needs to be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          sendResolved:\n                            description: Whether or not to notify about resolved alerts.\n                            type: boolean\n                          serviceKey:\n                            description: The secret's key that contains the PagerDuty service key (when using integration type \"Prometheus\"). Either this field or `routingKey` needs to be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          severity:\n                            description: Severity of the incident.\n                            type: string\n                          url:\n                            description: The URL to send requests to.\n                            type: string\n                        type: object\n                      type: array\n                    pushoverConfigs:\n                      description: List of Pushover configurations.\n                      items:\n                        description: PushoverConfig configures notifications via Pushover. See https://prometheus.io/docs/alerting/latest/configuration/#pushover_config\n                        properties:\n                          expire:\n                            description: How long your notification will continue to be retried for, unless the user acknowledges the notification.\n                            type: string\n                          html:\n                            description: Whether notification message is HTML or plain text.\n                            type: boolean\n                          httpConfig:\n                            description: HTTP client configuration.\n                            properties:\n                              basicAuth:\n                                description: BasicAuth for the client.\n                                properties:\n                                  password:\n                                    description: The secret in the service monitor namespace that contains the password for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  username:\n                                    description: The secret in the service monitor namespace that contains the username for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              bearerTokenSecret:\n                                description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              proxyURL:\n                                description: Optional proxy URL.\n                                type: string\n                              tlsConfig:\n                                description: TLS configuration for the client.\n                                properties:\n                                  ca:\n                                    description: Struct containing the CA cert to use for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  cert:\n                                    description: Struct containing the client cert file for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  insecureSkipVerify:\n                                    description: Disable target certificate validation.\n                                    type: boolean\n                                  keySecret:\n                                    description: Secret containing the client key file for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  serverName:\n                                    description: Used to verify the hostname for the targets.\n                                    type: string\n                                type: object\n                            type: object\n                          message:\n                            description: Notification message.\n                            type: string\n                          priority:\n                            description: Priority, see https://pushover.net/api#priority\n                            type: string\n                          retry:\n                            description: How often the Pushover servers will send the same notification to the user. Must be at least 30 seconds.\n                            type: string\n                          sendResolved:\n                            description: Whether or not to notify about resolved alerts.\n                            type: boolean\n                          sound:\n                            description: The name of one of the sounds supported by device clients to override the user's default sound choice\n                            type: string\n                          title:\n                            description: Notification title.\n                            type: string\n                          token:\n                            description: The secret's key that contains the registered application’s API token, see https://pushover.net/apps. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          url:\n                            description: A supplementary URL shown alongside the message.\n                            type: string\n                          urlTitle:\n                            description: A title for supplementary URL, otherwise just the URL is shown\n                            type: string\n                          userKey:\n                            description: The secret's key that contains the recipient user’s user key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                        type: object\n                      type: array\n                    slackConfigs:\n                      description: List of Slack configurations.\n                      items:\n                        description: SlackConfig configures notifications via Slack. See https://prometheus.io/docs/alerting/latest/configuration/#slack_config\n                        properties:\n                          actions:\n                            description: A list of Slack actions that are sent with each notification.\n                            items:\n                              description: SlackAction configures a single Slack action that is sent with each notification. See https://api.slack.com/docs/message-attachments#action_fields and https://api.slack.com/docs/message-buttons for more information.\n                              properties:\n                                confirm:\n                                  description: SlackConfirmationField protect users from destructive actions or particularly distinguished decisions by asking them to confirm their button click one more time. See https://api.slack.com/docs/interactive-message-field-guide#confirmation_fields for more information.\n                                  properties:\n                                    dismissText:\n                                      type: string\n                                    okText:\n                                      type: string\n                                    text:\n                                      minLength: 1\n                                      type: string\n                                    title:\n                                      type: string\n                                  required:\n                                  - text\n                                  type: object\n                                name:\n                                  type: string\n                                style:\n                                  type: string\n                                text:\n                                  minLength: 1\n                                  type: string\n                                type:\n                                  minLength: 1\n                                  type: string\n                                url:\n                                  type: string\n                                value:\n                                  type: string\n                              required:\n                              - text\n                              - type\n                              type: object\n                            type: array\n                          apiURL:\n                            description: The secret's key that contains the Slack webhook URL. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          callbackId:\n                            type: string\n                          channel:\n                            description: The channel or user to send notifications to.\n                            type: string\n                          color:\n                            type: string\n                          fallback:\n                            type: string\n                          fields:\n                            description: A list of Slack fields that are sent with each notification.\n                            items:\n                              description: SlackField configures a single Slack field that is sent with each notification. Each field must contain a title, value, and optionally, a boolean value to indicate if the field is short enough to be displayed next to other fields designated as short. See https://api.slack.com/docs/message-attachments#fields for more information.\n                              properties:\n                                short:\n                                  type: boolean\n                                title:\n                                  minLength: 1\n                                  type: string\n                                value:\n                                  minLength: 1\n                                  type: string\n                              required:\n                              - title\n                              - value\n                              type: object\n                            type: array\n                          footer:\n                            type: string\n                          httpConfig:\n                            description: HTTP client configuration.\n                            properties:\n                              basicAuth:\n                                description: BasicAuth for the client.\n                                properties:\n                                  password:\n                                    description: The secret in the service monitor namespace that contains the password for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  username:\n                                    description: The secret in the service monitor namespace that contains the username for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              bearerTokenSecret:\n                                description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              proxyURL:\n                                description: Optional proxy URL.\n                                type: string\n                              tlsConfig:\n                                description: TLS configuration for the client.\n                                properties:\n                                  ca:\n                                    description: Struct containing the CA cert to use for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  cert:\n                                    description: Struct containing the client cert file for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  insecureSkipVerify:\n                                    description: Disable target certificate validation.\n                                    type: boolean\n                                  keySecret:\n                                    description: Secret containing the client key file for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  serverName:\n                                    description: Used to verify the hostname for the targets.\n                                    type: string\n                                type: object\n                            type: object\n                          iconEmoji:\n                            type: string\n                          iconURL:\n                            type: string\n                          imageURL:\n                            type: string\n                          linkNames:\n                            type: boolean\n                          mrkdwnIn:\n                            items:\n                              type: string\n                            type: array\n                          pretext:\n                            type: string\n                          sendResolved:\n                            description: Whether or not to notify about resolved alerts.\n                            type: boolean\n                          shortFields:\n                            type: boolean\n                          text:\n                            type: string\n                          thumbURL:\n                            type: string\n                          title:\n                            type: string\n                          titleLink:\n                            type: string\n                          username:\n                            type: string\n                        type: object\n                      type: array\n                    victoropsConfigs:\n                      description: List of VictorOps configurations.\n                      items:\n                        description: VictorOpsConfig configures notifications via VictorOps. See https://prometheus.io/docs/alerting/latest/configuration/#victorops_config\n                        properties:\n                          apiKey:\n                            description: The secret's key that contains the API key to use when talking to the VictorOps API. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          apiUrl:\n                            description: The VictorOps API URL.\n                            type: string\n                          customFields:\n                            description: Additional custom fields for notification.\n                            items:\n                              description: KeyValue defines a (key, value) tuple.\n                              properties:\n                                key:\n                                  description: Key of the tuple.\n                                  minLength: 1\n                                  type: string\n                                value:\n                                  description: Value of the tuple.\n                                  type: string\n                              required:\n                              - key\n                              - value\n                              type: object\n                            type: array\n                          entityDisplayName:\n                            description: Contains summary of the alerted problem.\n                            type: string\n                          httpConfig:\n                            description: The HTTP client's configuration.\n                            properties:\n                              basicAuth:\n                                description: BasicAuth for the client.\n                                properties:\n                                  password:\n                                    description: The secret in the service monitor namespace that contains the password for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  username:\n                                    description: The secret in the service monitor namespace that contains the username for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              bearerTokenSecret:\n                                description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              proxyURL:\n                                description: Optional proxy URL.\n                                type: string\n                              tlsConfig:\n                                description: TLS configuration for the client.\n                                properties:\n                                  ca:\n                                    description: Struct containing the CA cert to use for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  cert:\n                                    description: Struct containing the client cert file for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  insecureSkipVerify:\n                                    description: Disable target certificate validation.\n                                    type: boolean\n                                  keySecret:\n                                    description: Secret containing the client key file for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  serverName:\n                                    description: Used to verify the hostname for the targets.\n                                    type: string\n                                type: object\n                            type: object\n                          messageType:\n                            description: Describes the behavior of the alert (CRITICAL, WARNING, INFO).\n                            type: string\n                          monitoringTool:\n                            description: The monitoring tool the state message is from.\n                            type: string\n                          routingKey:\n                            description: A key used to map the alert to a team.\n                            type: string\n                          sendResolved:\n                            description: Whether or not to notify about resolved alerts.\n                            type: boolean\n                          stateMessage:\n                            description: Contains long explanation of the alerted problem.\n                            type: string\n                        type: object\n                      type: array\n                    webhookConfigs:\n                      description: List of webhook configurations.\n                      items:\n                        description: WebhookConfig configures notifications via a generic receiver supporting the webhook payload. See https://prometheus.io/docs/alerting/latest/configuration/#webhook_config\n                        properties:\n                          httpConfig:\n                            description: HTTP client configuration.\n                            properties:\n                              basicAuth:\n                                description: BasicAuth for the client.\n                                properties:\n                                  password:\n                                    description: The secret in the service monitor namespace that contains the password for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  username:\n                                    description: The secret in the service monitor namespace that contains the username for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              bearerTokenSecret:\n                                description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              proxyURL:\n                                description: Optional proxy URL.\n                                type: string\n                              tlsConfig:\n                                description: TLS configuration for the client.\n                                properties:\n                                  ca:\n                                    description: Struct containing the CA cert to use for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  cert:\n                                    description: Struct containing the client cert file for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  insecureSkipVerify:\n                                    description: Disable target certificate validation.\n                                    type: boolean\n                                  keySecret:\n                                    description: Secret containing the client key file for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  serverName:\n                                    description: Used to verify the hostname for the targets.\n                                    type: string\n                                type: object\n                            type: object\n                          maxAlerts:\n                            description: Maximum number of alerts to be sent per webhook message. When 0, all alerts are included.\n                            format: int32\n                            minimum: 0\n                            type: integer\n                          sendResolved:\n                            description: Whether or not to notify about resolved alerts.\n                            type: boolean\n                          url:\n                            description: The URL to send HTTP POST requests to. `urlSecret` takes precedence over `url`. One of `urlSecret` and `url` should be defined.\n                            type: string\n                          urlSecret:\n                            description: The secret's key that contains the webhook URL to send HTTP requests to. `urlSecret` takes precedence over `url`. One of `urlSecret` and `url` should be defined. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                        type: object\n                      type: array\n                    wechatConfigs:\n                      description: List of WeChat configurations.\n                      items:\n                        description: WeChatConfig configures notifications via WeChat. See https://prometheus.io/docs/alerting/latest/configuration/#wechat_config\n                        properties:\n                          agentID:\n                            type: string\n                          apiSecret:\n                            description: The secret's key that contains the WeChat API key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          apiURL:\n                            description: The WeChat API URL.\n                            type: string\n                          corpID:\n                            description: The corp id for authentication.\n                            type: string\n                          httpConfig:\n                            description: HTTP client configuration.\n                            properties:\n                              basicAuth:\n                                description: BasicAuth for the client.\n                                properties:\n                                  password:\n                                    description: The secret in the service monitor namespace that contains the password for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  username:\n                                    description: The secret in the service monitor namespace that contains the username for authentication.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                type: object\n                              bearerTokenSecret:\n                                description: The secret's key that contains the bearer token to be used by the client for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator.\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              proxyURL:\n                                description: Optional proxy URL.\n                                type: string\n                              tlsConfig:\n                                description: TLS configuration for the client.\n                                properties:\n                                  ca:\n                                    description: Struct containing the CA cert to use for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  cert:\n                                    description: Struct containing the client cert file for the targets.\n                                    properties:\n                                      configMap:\n                                        description: ConfigMap containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key to select.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the ConfigMap or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                      secret:\n                                        description: Secret containing data to use for the targets.\n                                        properties:\n                                          key:\n                                            description: The key of the secret to select from.  Must be a valid secret key.\n                                            type: string\n                                          name:\n                                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                            type: string\n                                          optional:\n                                            description: Specify whether the Secret or its key must be defined\n                                            type: boolean\n                                        required:\n                                        - key\n                                        type: object\n                                    type: object\n                                  insecureSkipVerify:\n                                    description: Disable target certificate validation.\n                                    type: boolean\n                                  keySecret:\n                                    description: Secret containing the client key file for the targets.\n                                    properties:\n                                      key:\n                                        description: The key of the secret to select from.  Must be a valid secret key.\n                                        type: string\n                                      name:\n                                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                        type: string\n                                      optional:\n                                        description: Specify whether the Secret or its key must be defined\n                                        type: boolean\n                                    required:\n                                    - key\n                                    type: object\n                                  serverName:\n                                    description: Used to verify the hostname for the targets.\n                                    type: string\n                                type: object\n                            type: object\n                          message:\n                            description: API request data as defined by the WeChat API.\n                            type: string\n                          messageType:\n                            type: string\n                          sendResolved:\n                            description: Whether or not to notify about resolved alerts.\n                            type: boolean\n                          toParty:\n                            type: string\n                          toTag:\n                            type: string\n                          toUser:\n                            type: string\n                        type: object\n                      type: array\n                  required:\n                  - name\n                  type: object\n                type: array\n              route:\n                description: The Alertmanager route definition for alerts matching the resource’s namespace. If present, it will be added to the generated Alertmanager configuration as a first-level route.\n                properties:\n                  continue:\n                    description: Boolean indicating whether an alert should continue matching subsequent sibling nodes. It will always be overridden to true for the first-level route by the Prometheus operator.\n                    type: boolean\n                  groupBy:\n                    description: List of labels to group by.\n                    items:\n                      type: string\n                    type: array\n                  groupInterval:\n                    description: How long to wait before sending an updated notification. Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours).\n                    type: string\n                  groupWait:\n                    description: How long to wait before sending the initial notification. Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours).\n                    type: string\n                  matchers:\n                    description: 'List of matchers that the alert’s labels should match. For the first level route, the operator removes any existing equality and regexp matcher on the `namespace` label and adds a `namespace: <object namespace>` matcher.'\n                    items:\n                      description: Matcher defines how to match on alert's labels.\n                      properties:\n                        name:\n                          description: Label to match.\n                          minLength: 1\n                          type: string\n                        regex:\n                          description: Whether to match on equality (false) or regular-expression (true).\n                          type: boolean\n                        value:\n                          description: Label value to match.\n                          type: string\n                      required:\n                      - name\n                      type: object\n                    type: array\n                  receiver:\n                    description: Name of the receiver for this route. If not empty, it should be listed in the `receivers` field.\n                    type: string\n                  repeatInterval:\n                    description: How long to wait before repeating the last notification. Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours).\n                    type: string\n                  routes:\n                    description: Child routes.\n                    items:\n                      x-kubernetes-preserve-unknown-fields: true\n                    type: array\n                type: object\n            type: object\n        required:\n        - spec\n        type: object\n    served: true\n    storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n---\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: alertmanagers.monitoring.coreos.com\nspec:\n  group: monitoring.coreos.com\n  names:\n    categories:\n    - prometheus-operator\n    kind: Alertmanager\n    listKind: AlertmanagerList\n    plural: alertmanagers\n    singular: alertmanager\n  scope: Namespaced\n  versions:\n  - additionalPrinterColumns:\n    - description: The version of Alertmanager\n      jsonPath: .spec.version\n      name: Version\n      type: string\n    - description: The desired replicas number of Alertmanagers\n      jsonPath: .spec.replicas\n      name: Replicas\n      type: integer\n    - jsonPath: .metadata.creationTimestamp\n      name: Age\n      type: date\n    name: v1\n    schema:\n      openAPIV3Schema:\n        description: Alertmanager describes an Alertmanager cluster.\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: 'Specification of the desired behavior of the Alertmanager cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'\n            properties:\n              additionalPeers:\n                description: AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster.\n                items:\n                  type: string\n                type: array\n              affinity:\n                description: If specified, the pod's scheduling constraints.\n                properties:\n                  nodeAffinity:\n                    description: Describes node affinity scheduling rules for the pod.\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\n                          properties:\n                            preference:\n                              description: A node selector term, associated with the corresponding weight.\n                              properties:\n                                matchExpressions:\n                                  description: A list of node selector requirements by node's labels.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchFields:\n                                  description: A list of node selector requirements by node's fields.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                              type: object\n                            weight:\n                              description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - preference\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.\n                        properties:\n                          nodeSelectorTerms:\n                            description: Required. A list of node selector terms. The terms are ORed.\n                            items:\n                              description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\n                              properties:\n                                matchExpressions:\n                                  description: A list of node selector requirements by node's labels.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchFields:\n                                  description: A list of node selector requirements by node's fields.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                              type: object\n                            type: array\n                        required:\n                        - nodeSelectorTerms\n                        type: object\n                    type: object\n                  podAffinity:\n                    description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n                          properties:\n                            podAffinityTerm:\n                              description: Required. A pod affinity term, associated with the corresponding weight.\n                              properties:\n                                labelSelector:\n                                  description: A label query over a set of resources, in this case pods.\n                                  properties:\n                                    matchExpressions:\n                                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                      items:\n                                        description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                        properties:\n                                          key:\n                                            description: key is the label key that the selector applies to.\n                                            type: string\n                                          operator:\n                                            description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                            type: string\n                                          values:\n                                            description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                            items:\n                                              type: string\n                                            type: array\n                                        required:\n                                        - key\n                                        - operator\n                                        type: object\n                                      type: array\n                                    matchLabels:\n                                      additionalProperties:\n                                        type: string\n                                      description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                      type: object\n                                  type: object\n                                namespaces:\n                                  description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                                  items:\n                                    type: string\n                                  type: array\n                                topologyKey:\n                                  description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                                  type: string\n                              required:\n                              - topologyKey\n                              type: object\n                            weight:\n                              description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - podAffinityTerm\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\n                        items:\n                          description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key <topologyKey> matches that of any node on which a pod of the set of pods is running\n                          properties:\n                            labelSelector:\n                              description: A label query over a set of resources, in this case pods.\n                              properties:\n                                matchExpressions:\n                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                  items:\n                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: key is the label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                        type: string\n                                      values:\n                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchLabels:\n                                  additionalProperties:\n                                    type: string\n                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                  type: object\n                              type: object\n                            namespaces:\n                              description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                              items:\n                                type: string\n                              type: array\n                            topologyKey:\n                              description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                              type: string\n                          required:\n                          - topologyKey\n                          type: object\n                        type: array\n                    type: object\n                  podAntiAffinity:\n                    description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n                          properties:\n                            podAffinityTerm:\n                              description: Required. A pod affinity term, associated with the corresponding weight.\n                              properties:\n                                labelSelector:\n                                  description: A label query over a set of resources, in this case pods.\n                                  properties:\n                                    matchExpressions:\n                                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                      items:\n                                        description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                        properties:\n                                          key:\n                                            description: key is the label key that the selector applies to.\n                                            type: string\n                                          operator:\n                                            description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                            type: string\n                                          values:\n                                            description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                            items:\n                                              type: string\n                                            type: array\n                                        required:\n                                        - key\n                                        - operator\n                                        type: object\n                                      type: array\n                                    matchLabels:\n                                      additionalProperties:\n                                        type: string\n                                      description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                      type: object\n                                  type: object\n                                namespaces:\n                                  description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                                  items:\n                                    type: string\n                                  type: array\n                                topologyKey:\n                                  description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                                  type: string\n                              required:\n                              - topologyKey\n                              type: object\n                            weight:\n                              description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - podAffinityTerm\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\n                        items:\n                          description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key <topologyKey> matches that of any node on which a pod of the set of pods is running\n                          properties:\n                            labelSelector:\n                              description: A label query over a set of resources, in this case pods.\n                              properties:\n                                matchExpressions:\n                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                  items:\n                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: key is the label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                        type: string\n                                      values:\n                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchLabels:\n                                  additionalProperties:\n                                    type: string\n                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                  type: object\n                              type: object\n                            namespaces:\n                              description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                              items:\n                                type: string\n                              type: array\n                            topologyKey:\n                              description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                              type: string\n                          required:\n                          - topologyKey\n                          type: object\n                        type: array\n                    type: object\n                type: object\n              alertmanagerConfigNamespaceSelector:\n                description: Namespaces to be selected for AlertmanagerConfig discovery. If nil, only check own namespace.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              alertmanagerConfigSelector:\n                description: AlertmanagerConfigs to be selected for to merge and configure Alertmanager with.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              baseImage:\n                description: 'Base image that is used to deploy pods, without tag. Deprecated: use ''image'' instead'\n                type: string\n              clusterAdvertiseAddress:\n                description: 'ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918'\n                type: string\n              clusterGossipInterval:\n                description: Interval between gossip attempts.\n                type: string\n              clusterPeerTimeout:\n                description: Timeout for cluster peering.\n                type: string\n              clusterPushpullInterval:\n                description: Interval between pushpull attempts.\n                type: string\n              configMaps:\n                description: ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The ConfigMaps are mounted into /etc/alertmanager/configmaps/<configmap-name>.\n                items:\n                  type: string\n                type: array\n              configSecret:\n                description: ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for this Alertmanager instance. Defaults to 'alertmanager-<alertmanager-name>' The secret is mounted into /etc/alertmanager/config.\n                type: string\n              containers:\n                description: 'Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `alertmanager` and `config-reloader`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.'\n                items:\n                  description: A single application container that you want to run within a pod.\n                  properties:\n                    args:\n                      description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    command:\n                      description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    env:\n                      description: List of environment variables to set in the container. Cannot be updated.\n                      items:\n                        description: EnvVar represents an environment variable present in a Container.\n                        properties:\n                          name:\n                            description: Name of the environment variable. Must be a C_IDENTIFIER.\n                            type: string\n                          value:\n                            description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".'\n                            type: string\n                          valueFrom:\n                            description: Source for the environment variable's value. Cannot be used if value is not empty.\n                            properties:\n                              configMapKeyRef:\n                                description: Selects a key of a ConfigMap.\n                                properties:\n                                  key:\n                                    description: The key to select.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              fieldRef:\n                                description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                              secretKeyRef:\n                                description: Selects a key of a secret in the pod's namespace\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                            type: object\n                        required:\n                        - name\n                        type: object\n                      type: array\n                    envFrom:\n                      description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\n                      items:\n                        description: EnvFromSource represents the source of a set of ConfigMaps\n                        properties:\n                          configMapRef:\n                            description: The ConfigMap to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap must be defined\n                                type: boolean\n                            type: object\n                          prefix:\n                            description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\n                            type: string\n                          secretRef:\n                            description: The Secret to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret must be defined\n                                type: boolean\n                            type: object\n                        type: object\n                      type: array\n                    image:\n                      description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.'\n                      type: string\n                    imagePullPolicy:\n                      description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'\n                      type: string\n                    lifecycle:\n                      description: Actions that the management system should take in response to container lifecycle events. Cannot be updated.\n                      properties:\n                        postStart:\n                          description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                        preStop:\n                          description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                      type: object\n                    livenessProbe:\n                      description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    name:\n                      description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\n                      type: string\n                    ports:\n                      description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.\n                      items:\n                        description: ContainerPort represents a network port in a single container.\n                        properties:\n                          containerPort:\n                            description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.\n                            format: int32\n                            type: integer\n                          hostIP:\n                            description: What host IP to bind the external port to.\n                            type: string\n                          hostPort:\n                            description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\n                            format: int32\n                            type: integer\n                          name:\n                            description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\n                            type: string\n                          protocol:\n                            default: TCP\n                            description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".\n                            type: string\n                        required:\n                        - containerPort\n                        type: object\n                      type: array\n                      x-kubernetes-list-map-keys:\n                      - containerPort\n                      - protocol\n                      x-kubernetes-list-type: map\n                    readinessProbe:\n                      description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    resources:\n                      description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                      properties:\n                        limits:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                        requests:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                      type: object\n                    securityContext:\n                      description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/'\n                      properties:\n                        allowPrivilegeEscalation:\n                          description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN'\n                          type: boolean\n                        capabilities:\n                          description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.\n                          properties:\n                            add:\n                              description: Added capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                            drop:\n                              description: Removed capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                          type: object\n                        privileged:\n                          description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\n                          type: boolean\n                        procMount:\n                          description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\n                          type: string\n                        readOnlyRootFilesystem:\n                          description: Whether this container has a read-only root filesystem. Default is false.\n                          type: boolean\n                        runAsGroup:\n                          description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        runAsNonRoot:\n                          description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          type: boolean\n                        runAsUser:\n                          description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        seLinuxOptions:\n                          description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            level:\n                              description: Level is SELinux level label that applies to the container.\n                              type: string\n                            role:\n                              description: Role is a SELinux role label that applies to the container.\n                              type: string\n                            type:\n                              description: Type is a SELinux type label that applies to the container.\n                              type: string\n                            user:\n                              description: User is a SELinux user label that applies to the container.\n                              type: string\n                          type: object\n                        windowsOptions:\n                          description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            gmsaCredentialSpec:\n                              description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                              type: string\n                            gmsaCredentialSpecName:\n                              description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                              type: string\n                            runAsUserName:\n                              description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                              type: string\n                          type: object\n                      type: object\n                    startupProbe:\n                      description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    stdin:\n                      description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\n                      type: boolean\n                    stdinOnce:\n                      description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\n                      type: boolean\n                    terminationMessagePath:\n                      description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.'\n                      type: string\n                    terminationMessagePolicy:\n                      description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\n                      type: string\n                    tty:\n                      description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\n                      type: boolean\n                    volumeDevices:\n                      description: volumeDevices is the list of block devices to be used by the container.\n                      items:\n                        description: volumeDevice describes a mapping of a raw block device within a container.\n                        properties:\n                          devicePath:\n                            description: devicePath is the path inside of the container that the device will be mapped to.\n                            type: string\n                          name:\n                            description: name must match the name of a persistentVolumeClaim in the pod\n                            type: string\n                        required:\n                        - devicePath\n                        - name\n                        type: object\n                      type: array\n                    volumeMounts:\n                      description: Pod volumes to mount into the container's filesystem. Cannot be updated.\n                      items:\n                        description: VolumeMount describes a mounting of a Volume within a container.\n                        properties:\n                          mountPath:\n                            description: Path within the container at which the volume should be mounted.  Must not contain ':'.\n                            type: string\n                          mountPropagation:\n                            description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n                            type: string\n                          name:\n                            description: This must match the Name of a Volume.\n                            type: string\n                          readOnly:\n                            description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\n                            type: boolean\n                          subPath:\n                            description: Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).\n                            type: string\n                          subPathExpr:\n                            description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\n                            type: string\n                        required:\n                        - mountPath\n                        - name\n                        type: object\n                      type: array\n                    workingDir:\n                      description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\n                      type: string\n                  required:\n                  - name\n                  type: object\n                type: array\n              externalUrl:\n                description: The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name.\n                type: string\n              forceEnableClusterMode:\n                description: ForceEnableClusterMode ensures Alertmanager does not deactivate the cluster mode when running with a single replica. Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each.\n                type: boolean\n              image:\n                description: Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Alertmanager is being configured.\n                type: string\n              imagePullSecrets:\n                description: An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod\n                items:\n                  description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.\n                  properties:\n                    name:\n                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                      type: string\n                  type: object\n                type: array\n              initContainers:\n                description: 'InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the Alertmanager configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ Using initContainers for any use case other then secret fetching is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.'\n                items:\n                  description: A single application container that you want to run within a pod.\n                  properties:\n                    args:\n                      description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    command:\n                      description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    env:\n                      description: List of environment variables to set in the container. Cannot be updated.\n                      items:\n                        description: EnvVar represents an environment variable present in a Container.\n                        properties:\n                          name:\n                            description: Name of the environment variable. Must be a C_IDENTIFIER.\n                            type: string\n                          value:\n                            description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".'\n                            type: string\n                          valueFrom:\n                            description: Source for the environment variable's value. Cannot be used if value is not empty.\n                            properties:\n                              configMapKeyRef:\n                                description: Selects a key of a ConfigMap.\n                                properties:\n                                  key:\n                                    description: The key to select.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              fieldRef:\n                                description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                              secretKeyRef:\n                                description: Selects a key of a secret in the pod's namespace\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                            type: object\n                        required:\n                        - name\n                        type: object\n                      type: array\n                    envFrom:\n                      description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\n                      items:\n                        description: EnvFromSource represents the source of a set of ConfigMaps\n                        properties:\n                          configMapRef:\n                            description: The ConfigMap to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap must be defined\n                                type: boolean\n                            type: object\n                          prefix:\n                            description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\n                            type: string\n                          secretRef:\n                            description: The Secret to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret must be defined\n                                type: boolean\n                            type: object\n                        type: object\n                      type: array\n                    image:\n                      description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.'\n                      type: string\n                    imagePullPolicy:\n                      description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'\n                      type: string\n                    lifecycle:\n                      description: Actions that the management system should take in response to container lifecycle events. Cannot be updated.\n                      properties:\n                        postStart:\n                          description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                        preStop:\n                          description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                      type: object\n                    livenessProbe:\n                      description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    name:\n                      description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\n                      type: string\n                    ports:\n                      description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.\n                      items:\n                        description: ContainerPort represents a network port in a single container.\n                        properties:\n                          containerPort:\n                            description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.\n                            format: int32\n                            type: integer\n                          hostIP:\n                            description: What host IP to bind the external port to.\n                            type: string\n                          hostPort:\n                            description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\n                            format: int32\n                            type: integer\n                          name:\n                            description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\n                            type: string\n                          protocol:\n                            default: TCP\n                            description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".\n                            type: string\n                        required:\n                        - containerPort\n                        type: object\n                      type: array\n                      x-kubernetes-list-map-keys:\n                      - containerPort\n                      - protocol\n                      x-kubernetes-list-type: map\n                    readinessProbe:\n                      description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    resources:\n                      description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                      properties:\n                        limits:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                        requests:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                      type: object\n                    securityContext:\n                      description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/'\n                      properties:\n                        allowPrivilegeEscalation:\n                          description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN'\n                          type: boolean\n                        capabilities:\n                          description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.\n                          properties:\n                            add:\n                              description: Added capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                            drop:\n                              description: Removed capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                          type: object\n                        privileged:\n                          description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\n                          type: boolean\n                        procMount:\n                          description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\n                          type: string\n                        readOnlyRootFilesystem:\n                          description: Whether this container has a read-only root filesystem. Default is false.\n                          type: boolean\n                        runAsGroup:\n                          description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        runAsNonRoot:\n                          description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          type: boolean\n                        runAsUser:\n                          description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        seLinuxOptions:\n                          description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            level:\n                              description: Level is SELinux level label that applies to the container.\n                              type: string\n                            role:\n                              description: Role is a SELinux role label that applies to the container.\n                              type: string\n                            type:\n                              description: Type is a SELinux type label that applies to the container.\n                              type: string\n                            user:\n                              description: User is a SELinux user label that applies to the container.\n                              type: string\n                          type: object\n                        windowsOptions:\n                          description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            gmsaCredentialSpec:\n                              description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                              type: string\n                            gmsaCredentialSpecName:\n                              description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                              type: string\n                            runAsUserName:\n                              description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                              type: string\n                          type: object\n                      type: object\n                    startupProbe:\n                      description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    stdin:\n                      description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\n                      type: boolean\n                    stdinOnce:\n                      description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\n                      type: boolean\n                    terminationMessagePath:\n                      description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.'\n                      type: string\n                    terminationMessagePolicy:\n                      description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\n                      type: string\n                    tty:\n                      description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\n                      type: boolean\n                    volumeDevices:\n                      description: volumeDevices is the list of block devices to be used by the container.\n                      items:\n                        description: volumeDevice describes a mapping of a raw block device within a container.\n                        properties:\n                          devicePath:\n                            description: devicePath is the path inside of the container that the device will be mapped to.\n                            type: string\n                          name:\n                            description: name must match the name of a persistentVolumeClaim in the pod\n                            type: string\n                        required:\n                        - devicePath\n                        - name\n                        type: object\n                      type: array\n                    volumeMounts:\n                      description: Pod volumes to mount into the container's filesystem. Cannot be updated.\n                      items:\n                        description: VolumeMount describes a mounting of a Volume within a container.\n                        properties:\n                          mountPath:\n                            description: Path within the container at which the volume should be mounted.  Must not contain ':'.\n                            type: string\n                          mountPropagation:\n                            description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n                            type: string\n                          name:\n                            description: This must match the Name of a Volume.\n                            type: string\n                          readOnly:\n                            description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\n                            type: boolean\n                          subPath:\n                            description: Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).\n                            type: string\n                          subPathExpr:\n                            description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\n                            type: string\n                        required:\n                        - mountPath\n                        - name\n                        type: object\n                      type: array\n                    workingDir:\n                      description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\n                      type: string\n                  required:\n                  - name\n                  type: object\n                type: array\n              listenLocal:\n                description: ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. Note this is only for the Alertmanager UI, not the gossip communication.\n                type: boolean\n              logFormat:\n                description: Log format for Alertmanager to be configured with.\n                type: string\n              logLevel:\n                description: Log level for Alertmanager to be configured with.\n                type: string\n              nodeSelector:\n                additionalProperties:\n                  type: string\n                description: Define which Nodes the Pods are scheduled on.\n                type: object\n              paused:\n                description: If set to true all actions on the underlying managed objects are not goint to be performed, except for delete actions.\n                type: boolean\n              podMetadata:\n                description: PodMetadata configures Labels and Annotations which are propagated to the alertmanager pods.\n                properties:\n                  annotations:\n                    additionalProperties:\n                      type: string\n                    description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'\n                    type: object\n                  labels:\n                    additionalProperties:\n                      type: string\n                    description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'\n                    type: object\n                  name:\n                    description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'\n                    type: string\n                type: object\n              portName:\n                description: Port name used for the pods and governing service. This defaults to web\n                type: string\n              priorityClassName:\n                description: Priority class assigned to the Pods\n                type: string\n              replicas:\n                description: Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size.\n                format: int32\n                type: integer\n              resources:\n                description: Define resources requests and limits for single Pods.\n                properties:\n                  limits:\n                    additionalProperties:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                      x-kubernetes-int-or-string: true\n                    description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                    type: object\n                  requests:\n                    additionalProperties:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                      x-kubernetes-int-or-string: true\n                    description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                    type: object\n                type: object\n              retention:\n                description: Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours).\n                type: string\n              routePrefix:\n                description: The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.\n                type: string\n              secrets:\n                description: Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/<secret-name>.\n                items:\n                  type: string\n                type: array\n              securityContext:\n                description: SecurityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext.\n                properties:\n                  fsGroup:\n                    description: \"A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \\n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \\n If unset, the Kubelet will not modify the ownership and permissions of any volume.\"\n                    format: int64\n                    type: integer\n                  fsGroupChangePolicy:\n                    description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are \"OnRootMismatch\" and \"Always\". If not specified defaults to \"Always\".'\n                    type: string\n                  runAsGroup:\n                    description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    format: int64\n                    type: integer\n                  runAsNonRoot:\n                    description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                    type: boolean\n                  runAsUser:\n                    description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    format: int64\n                    type: integer\n                  seLinuxOptions:\n                    description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    properties:\n                      level:\n                        description: Level is SELinux level label that applies to the container.\n                        type: string\n                      role:\n                        description: Role is a SELinux role label that applies to the container.\n                        type: string\n                      type:\n                        description: Type is a SELinux type label that applies to the container.\n                        type: string\n                      user:\n                        description: User is a SELinux user label that applies to the container.\n                        type: string\n                    type: object\n                  supplementalGroups:\n                    description: A list of groups applied to the first process run in each container, in addition to the container's primary GID.  If unspecified, no groups will be added to any container.\n                    items:\n                      format: int64\n                      type: integer\n                    type: array\n                  sysctls:\n                    description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.\n                    items:\n                      description: Sysctl defines a kernel parameter to be set\n                      properties:\n                        name:\n                          description: Name of a property to set\n                          type: string\n                        value:\n                          description: Value of a property to set\n                          type: string\n                      required:\n                      - name\n                      - value\n                      type: object\n                    type: array\n                  windowsOptions:\n                    description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                    properties:\n                      gmsaCredentialSpec:\n                        description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                        type: string\n                      gmsaCredentialSpecName:\n                        description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                        type: string\n                      runAsUserName:\n                        description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                        type: string\n                    type: object\n                type: object\n              serviceAccountName:\n                description: ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.\n                type: string\n              sha:\n                description: 'SHA of Alertmanager container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set. Deprecated: use ''image'' instead.  The image digest can be specified as part of the image URL.'\n                type: string\n              storage:\n                description: Storage is the definition of how storage will be used by the Alertmanager instances.\n                properties:\n                  disableMountSubPath:\n                    description: 'Deprecated: subPath usage will be disabled by default in a future release, this option will become unnecessary. DisableMountSubPath allows to remove any subPath usage in volume mounts.'\n                    type: boolean\n                  emptyDir:\n                    description: 'EmptyDirVolumeSource to be used by the Prometheus StatefulSets. If specified, used in place of any volumeClaimTemplate. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'\n                    properties:\n                      medium:\n                        description: 'What type of storage medium should back this directory. The default is \"\" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                        type: string\n                      sizeLimit:\n                        anyOf:\n                        - type: integer\n                        - type: string\n                        description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'\n                        pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                        x-kubernetes-int-or-string: true\n                    type: object\n                  volumeClaimTemplate:\n                    description: A PVC spec to be used by the Prometheus StatefulSets.\n                    properties:\n                      apiVersion:\n                        description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n                        type: string\n                      kind:\n                        description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n                        type: string\n                      metadata:\n                        description: EmbeddedMetadata contains metadata relevant to an EmbeddedResource.\n                        properties:\n                          annotations:\n                            additionalProperties:\n                              type: string\n                            description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'\n                            type: object\n                          labels:\n                            additionalProperties:\n                              type: string\n                            description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'\n                            type: object\n                          name:\n                            description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'\n                            type: string\n                        type: object\n                      spec:\n                        description: 'Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                        properties:\n                          accessModes:\n                            description: 'AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'\n                            items:\n                              type: string\n                            type: array\n                          dataSource:\n                            description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - Beta) * An existing PVC (PersistentVolumeClaim) * An existing custom resource/object that implements data population (Alpha) In order to use VolumeSnapshot object types, the appropriate feature gate must be enabled (VolumeSnapshotDataSource or AnyVolumeDataSource) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the specified data source is not supported, the volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.'\n                            properties:\n                              apiGroup:\n                                description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\n                                type: string\n                              kind:\n                                description: Kind is the type of resource being referenced\n                                type: string\n                              name:\n                                description: Name is the name of resource being referenced\n                                type: string\n                            required:\n                            - kind\n                            - name\n                            type: object\n                          resources:\n                            description: 'Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'\n                            properties:\n                              limits:\n                                additionalProperties:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                  x-kubernetes-int-or-string: true\n                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                                type: object\n                              requests:\n                                additionalProperties:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                  x-kubernetes-int-or-string: true\n                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                                type: object\n                            type: object\n                          selector:\n                            description: A label query over volumes to consider for binding.\n                            properties:\n                              matchExpressions:\n                                description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                items:\n                                  description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                  properties:\n                                    key:\n                                      description: key is the label key that the selector applies to.\n                                      type: string\n                                    operator:\n                                      description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                      type: string\n                                    values:\n                                      description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                      items:\n                                        type: string\n                                      type: array\n                                  required:\n                                  - key\n                                  - operator\n                                  type: object\n                                type: array\n                              matchLabels:\n                                additionalProperties:\n                                  type: string\n                                description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                type: object\n                            type: object\n                          storageClassName:\n                            description: 'Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1'\n                            type: string\n                          volumeMode:\n                            description: volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec.\n                            type: string\n                          volumeName:\n                            description: VolumeName is the binding reference to the PersistentVolume backing this claim.\n                            type: string\n                        type: object\n                      status:\n                        description: 'Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                        properties:\n                          accessModes:\n                            description: 'AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'\n                            items:\n                              type: string\n                            type: array\n                          capacity:\n                            additionalProperties:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                              x-kubernetes-int-or-string: true\n                            description: Represents the actual resources of the underlying volume.\n                            type: object\n                          conditions:\n                            description: Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\n                            items:\n                              description: PersistentVolumeClaimCondition contails details about state of pvc\n                              properties:\n                                lastProbeTime:\n                                  description: Last time we probed the condition.\n                                  format: date-time\n                                  type: string\n                                lastTransitionTime:\n                                  description: Last time the condition transitioned from one status to another.\n                                  format: date-time\n                                  type: string\n                                message:\n                                  description: Human-readable message indicating details about last transition.\n                                  type: string\n                                reason:\n                                  description: Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.\n                                  type: string\n                                status:\n                                  type: string\n                                type:\n                                  description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type\n                                  type: string\n                              required:\n                              - status\n                              - type\n                              type: object\n                            type: array\n                          phase:\n                            description: Phase represents the current phase of PersistentVolumeClaim.\n                            type: string\n                        type: object\n                    type: object\n                type: object\n              tag:\n                description: 'Tag of Alertmanager container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set. Deprecated: use ''image'' instead.  The image tag can be specified as part of the image URL.'\n                type: string\n              tolerations:\n                description: If specified, the pod's tolerations.\n                items:\n                  description: The pod this Toleration is attached to tolerates any taint that matches the triple <key,value,effect> using the matching operator <operator>.\n                  properties:\n                    effect:\n                      description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\n                      type: string\n                    key:\n                      description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.\n                      type: string\n                    operator:\n                      description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\n                      type: string\n                    tolerationSeconds:\n                      description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.\n                      format: int64\n                      type: integer\n                    value:\n                      description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.\n                      type: string\n                  type: object\n                type: array\n              topologySpreadConstraints:\n                description: If specified, the pod's topology spread constraints.\n                items:\n                  description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.\n                  properties:\n                    labelSelector:\n                      description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.\n                      properties:\n                        matchExpressions:\n                          description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                          items:\n                            description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                            properties:\n                              key:\n                                description: key is the label key that the selector applies to.\n                                type: string\n                              operator:\n                                description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                type: string\n                              values:\n                                description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                items:\n                                  type: string\n                                type: array\n                            required:\n                            - key\n                            - operator\n                            type: object\n                          type: array\n                        matchLabels:\n                          additionalProperties:\n                            type: string\n                          description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                          type: object\n                      type: object\n                    maxSkew:\n                      description: 'MaxSkew describes the degree to which pods may be unevenly distributed. It''s the maximum permitted difference between the number of matching pods in any two topology domains of a given topology type. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | |   P   |   P   |       | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. It''s a required field. Default value is 1 and 0 is not allowed.'\n                      format: int32\n                      type: integer\n                    topologyKey:\n                      description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each <key, value> as a \"bucket\", and try to put balanced number of pods into each bucket. It's a required field.\n                      type: string\n                    whenUnsatisfiable:\n                      description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it - ScheduleAnyway tells the scheduler to still schedule it It''s considered as \"Unsatisfiable\" if and only if placing incoming pod on any topology violates \"MaxSkew\". For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.'\n                      type: string\n                  required:\n                  - maxSkew\n                  - topologyKey\n                  - whenUnsatisfiable\n                  type: object\n                type: array\n              version:\n                description: Version the cluster should be on.\n                type: string\n              volumeMounts:\n                description: VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the alertmanager container, that are generated as a result of StorageSpec objects.\n                items:\n                  description: VolumeMount describes a mounting of a Volume within a container.\n                  properties:\n                    mountPath:\n                      description: Path within the container at which the volume should be mounted.  Must not contain ':'.\n                      type: string\n                    mountPropagation:\n                      description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n                      type: string\n                    name:\n                      description: This must match the Name of a Volume.\n                      type: string\n                    readOnly:\n                      description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\n                      type: boolean\n                    subPath:\n                      description: Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).\n                      type: string\n                    subPathExpr:\n                      description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\n                      type: string\n                  required:\n                  - mountPath\n                  - name\n                  type: object\n                type: array\n              volumes:\n                description: Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects.\n                items:\n                  description: Volume represents a named volume in a pod that may be accessed by any container in the pod.\n                  properties:\n                    awsElasticBlockStore:\n                      description: 'AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        partition:\n                          description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty).'\n                          format: int32\n                          type: integer\n                        readOnly:\n                          description: 'Specify \"true\" to force and set the ReadOnly property in VolumeMounts to \"true\". If omitted, the default is \"false\". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                          type: boolean\n                        volumeID:\n                          description: 'Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    azureDisk:\n                      description: AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.\n                      properties:\n                        cachingMode:\n                          description: 'Host Caching mode: None, Read Only, Read Write.'\n                          type: string\n                        diskName:\n                          description: The Name of the data disk in the blob storage\n                          type: string\n                        diskURI:\n                          description: The URI the data disk in the blob storage\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        kind:\n                          description: 'Expected values Shared: multiple blob disks per storage account  Dedicated: single blob disk per storage account  Managed: azure managed data disk (only in managed availability set). defaults to shared'\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                      required:\n                      - diskName\n                      - diskURI\n                      type: object\n                    azureFile:\n                      description: AzureFile represents an Azure File Service mount on the host and bind mount to the pod.\n                      properties:\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretName:\n                          description: the name of secret that contains Azure Storage Account Name and Key\n                          type: string\n                        shareName:\n                          description: Share Name\n                          type: string\n                      required:\n                      - secretName\n                      - shareName\n                      type: object\n                    cephfs:\n                      description: CephFS represents a Ceph FS mount on the host that shares a pod's lifetime\n                      properties:\n                        monitors:\n                          description: 'Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          items:\n                            type: string\n                          type: array\n                        path:\n                          description: 'Optional: Used as the mounted root, rather than the full Ceph tree, default is /'\n                          type: string\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: boolean\n                        secretFile:\n                          description: 'Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: string\n                        secretRef:\n                          description: 'Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        user:\n                          description: 'Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: string\n                      required:\n                      - monitors\n                      type: object\n                    cinder:\n                      description: 'Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: string\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: boolean\n                        secretRef:\n                          description: 'Optional: points to a secret object containing parameters used to connect to OpenStack.'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        volumeID:\n                          description: 'volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    configMap:\n                      description: ConfigMap represents a configMap that should populate this volume\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                          items:\n                            description: Maps a string key to a path within a volume.\n                            properties:\n                              key:\n                                description: The key to project.\n                                type: string\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                type: string\n                            required:\n                            - key\n                            - path\n                            type: object\n                          type: array\n                        name:\n                          description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                          type: string\n                        optional:\n                          description: Specify whether the ConfigMap or its keys must be defined\n                          type: boolean\n                      type: object\n                    csi:\n                      description: CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature).\n                      properties:\n                        driver:\n                          description: Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster.\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Ex. \"ext4\", \"xfs\", \"ntfs\". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply.\n                          type: string\n                        nodePublishSecretRef:\n                          description: NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and  may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        readOnly:\n                          description: Specifies a read-only configuration for the volume. Defaults to false (read/write).\n                          type: boolean\n                        volumeAttributes:\n                          additionalProperties:\n                            type: string\n                          description: VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values.\n                          type: object\n                      required:\n                      - driver\n                      type: object\n                    downwardAPI:\n                      description: DownwardAPI represents downward API about the pod that should populate this volume\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: Items is a list of downward API volume file\n                          items:\n                            description: DownwardAPIVolumeFile represents information to create the file containing the pod field\n                            properties:\n                              fieldRef:\n                                description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: 'Required: Path is  the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..'''\n                                type: string\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                            required:\n                            - path\n                            type: object\n                          type: array\n                      type: object\n                    emptyDir:\n                      description: 'EmptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                      properties:\n                        medium:\n                          description: 'What type of storage medium should back this directory. The default is \"\" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                          type: string\n                        sizeLimit:\n                          anyOf:\n                          - type: integer\n                          - type: string\n                          description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'\n                          pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                          x-kubernetes-int-or-string: true\n                      type: object\n                    fc:\n                      description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod.\n                      properties:\n                        fsType:\n                          description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        lun:\n                          description: 'Optional: FC target lun number'\n                          format: int32\n                          type: integer\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.'\n                          type: boolean\n                        targetWWNs:\n                          description: 'Optional: FC target worldwide names (WWNs)'\n                          items:\n                            type: string\n                          type: array\n                        wwids:\n                          description: 'Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.'\n                          items:\n                            type: string\n                          type: array\n                      type: object\n                    flexVolume:\n                      description: FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin.\n                      properties:\n                        driver:\n                          description: Driver is the name of the driver to use for this volume.\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default filesystem depends on FlexVolume script.\n                          type: string\n                        options:\n                          additionalProperties:\n                            type: string\n                          description: 'Optional: Extra command options if any.'\n                          type: object\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.'\n                          type: boolean\n                        secretRef:\n                          description: 'Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                      required:\n                      - driver\n                      type: object\n                    flocker:\n                      description: Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running\n                      properties:\n                        datasetName:\n                          description: Name of the dataset stored as metadata -> name on the dataset for Flocker should be considered as deprecated\n                          type: string\n                        datasetUUID:\n                          description: UUID of the dataset. This is unique identifier of a Flocker dataset\n                          type: string\n                      type: object\n                    gcePersistentDisk:\n                      description: 'GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        partition:\n                          description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          format: int32\n                          type: integer\n                        pdName:\n                          description: 'Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          type: boolean\n                      required:\n                      - pdName\n                      type: object\n                    gitRepo:\n                      description: 'GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod''s container.'\n                      properties:\n                        directory:\n                          description: Target directory name. Must not contain or start with '..'.  If '.' is supplied, the volume directory will be the git repository.  Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name.\n                          type: string\n                        repository:\n                          description: Repository URL\n                          type: string\n                        revision:\n                          description: Commit hash for the specified revision.\n                          type: string\n                      required:\n                      - repository\n                      type: object\n                    glusterfs:\n                      description: 'Glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md'\n                      properties:\n                        endpoints:\n                          description: 'EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: string\n                        path:\n                          description: 'Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: boolean\n                      required:\n                      - endpoints\n                      - path\n                      type: object\n                    hostPath:\n                      description: 'HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.'\n                      properties:\n                        path:\n                          description: 'Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath'\n                          type: string\n                        type:\n                          description: 'Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath'\n                          type: string\n                      required:\n                      - path\n                      type: object\n                    iscsi:\n                      description: 'ISCSI represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md'\n                      properties:\n                        chapAuthDiscovery:\n                          description: whether support iSCSI Discovery CHAP authentication\n                          type: boolean\n                        chapAuthSession:\n                          description: whether support iSCSI Session CHAP authentication\n                          type: boolean\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        initiatorName:\n                          description: Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface <target portal>:<volume name> will be created for the connection.\n                          type: string\n                        iqn:\n                          description: Target iSCSI Qualified Name.\n                          type: string\n                        iscsiInterface:\n                          description: iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp).\n                          type: string\n                        lun:\n                          description: iSCSI Target Lun number.\n                          format: int32\n                          type: integer\n                        portals:\n                          description: iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).\n                          items:\n                            type: string\n                          type: array\n                        readOnly:\n                          description: ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false.\n                          type: boolean\n                        secretRef:\n                          description: CHAP Secret for iSCSI target and initiator authentication\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        targetPortal:\n                          description: iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).\n                          type: string\n                      required:\n                      - iqn\n                      - lun\n                      - targetPortal\n                      type: object\n                    name:\n                      description: 'Volume''s name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'\n                      type: string\n                    nfs:\n                      description: 'NFS represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                      properties:\n                        path:\n                          description: 'Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: boolean\n                        server:\n                          description: 'Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: string\n                      required:\n                      - path\n                      - server\n                      type: object\n                    persistentVolumeClaim:\n                      description: 'PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                      properties:\n                        claimName:\n                          description: 'ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                          type: string\n                        readOnly:\n                          description: Will force the ReadOnly setting in VolumeMounts. Default false.\n                          type: boolean\n                      required:\n                      - claimName\n                      type: object\n                    photonPersistentDisk:\n                      description: PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        pdID:\n                          description: ID that identifies Photon Controller persistent disk\n                          type: string\n                      required:\n                      - pdID\n                      type: object\n                    portworxVolume:\n                      description: PortworxVolume represents a portworx volume attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        volumeID:\n                          description: VolumeID uniquely identifies a Portworx volume\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    projected:\n                      description: Items for all in one resources secrets, configmaps, and downward API\n                      properties:\n                        defaultMode:\n                          description: Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\n                          format: int32\n                          type: integer\n                        sources:\n                          description: list of volume projections\n                          items:\n                            description: Projection that may be projected along with other supported volume types\n                            properties:\n                              configMap:\n                                description: information about the configMap data to project\n                                properties:\n                                  items:\n                                    description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                                    items:\n                                      description: Maps a string key to a path within a volume.\n                                      properties:\n                                        key:\n                                          description: The key to project.\n                                          type: string\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                          type: string\n                                      required:\n                                      - key\n                                      - path\n                                      type: object\n                                    type: array\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its keys must be defined\n                                    type: boolean\n                                type: object\n                              downwardAPI:\n                                description: information about the downwardAPI data to project\n                                properties:\n                                  items:\n                                    description: Items is a list of DownwardAPIVolume file\n                                    items:\n                                      description: DownwardAPIVolumeFile represents information to create the file containing the pod field\n                                      properties:\n                                        fieldRef:\n                                          description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.'\n                                          properties:\n                                            apiVersion:\n                                              description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                              type: string\n                                            fieldPath:\n                                              description: Path of the field to select in the specified API version.\n                                              type: string\n                                          required:\n                                          - fieldPath\n                                          type: object\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: 'Required: Path is  the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..'''\n                                          type: string\n                                        resourceFieldRef:\n                                          description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.'\n                                          properties:\n                                            containerName:\n                                              description: 'Container name: required for volumes, optional for env vars'\n                                              type: string\n                                            divisor:\n                                              anyOf:\n                                              - type: integer\n                                              - type: string\n                                              description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                              pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                              x-kubernetes-int-or-string: true\n                                            resource:\n                                              description: 'Required: resource to select'\n                                              type: string\n                                          required:\n                                          - resource\n                                          type: object\n                                      required:\n                                      - path\n                                      type: object\n                                    type: array\n                                type: object\n                              secret:\n                                description: information about the secret data to project\n                                properties:\n                                  items:\n                                    description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                                    items:\n                                      description: Maps a string key to a path within a volume.\n                                      properties:\n                                        key:\n                                          description: The key to project.\n                                          type: string\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                          type: string\n                                      required:\n                                      - key\n                                      - path\n                                      type: object\n                                    type: array\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                type: object\n                              serviceAccountToken:\n                                description: information about the serviceAccountToken data to project\n                                properties:\n                                  audience:\n                                    description: Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver.\n                                    type: string\n                                  expirationSeconds:\n                                    description: ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes.\n                                    format: int64\n                                    type: integer\n                                  path:\n                                    description: Path is the path relative to the mount point of the file to project the token into.\n                                    type: string\n                                required:\n                                - path\n                                type: object\n                            type: object\n                          type: array\n                      required:\n                      - sources\n                      type: object\n                    quobyte:\n                      description: Quobyte represents a Quobyte mount on the host that shares a pod's lifetime\n                      properties:\n                        group:\n                          description: Group to map volume access to Default is no group\n                          type: string\n                        readOnly:\n                          description: ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false.\n                          type: boolean\n                        registry:\n                          description: Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes\n                          type: string\n                        tenant:\n                          description: Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin\n                          type: string\n                        user:\n                          description: User to map volume access to Defaults to serivceaccount user\n                          type: string\n                        volume:\n                          description: Volume is a string that references an already created Quobyte volume by name.\n                          type: string\n                      required:\n                      - registry\n                      - volume\n                      type: object\n                    rbd:\n                      description: 'RBD represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        image:\n                          description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        keyring:\n                          description: 'Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        monitors:\n                          description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          items:\n                            type: string\n                          type: array\n                        pool:\n                          description: 'The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: boolean\n                        secretRef:\n                          description: 'SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        user:\n                          description: 'The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                      required:\n                      - image\n                      - monitors\n                      type: object\n                    scaleIO:\n                      description: ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Default is \"xfs\".\n                          type: string\n                        gateway:\n                          description: The host address of the ScaleIO API Gateway.\n                          type: string\n                        protectionDomain:\n                          description: The name of the ScaleIO Protection Domain for the configured storage.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretRef:\n                          description: SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        sslEnabled:\n                          description: Flag to enable/disable SSL communication with Gateway, default false\n                          type: boolean\n                        storageMode:\n                          description: Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned.\n                          type: string\n                        storagePool:\n                          description: The ScaleIO Storage Pool associated with the protection domain.\n                          type: string\n                        system:\n                          description: The name of the storage system as configured in ScaleIO.\n                          type: string\n                        volumeName:\n                          description: The name of a volume already created in the ScaleIO system that is associated with this volume source.\n                          type: string\n                      required:\n                      - gateway\n                      - secretRef\n                      - system\n                      type: object\n                    secret:\n                      description: 'Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret'\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                          items:\n                            description: Maps a string key to a path within a volume.\n                            properties:\n                              key:\n                                description: The key to project.\n                                type: string\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                type: string\n                            required:\n                            - key\n                            - path\n                            type: object\n                          type: array\n                        optional:\n                          description: Specify whether the Secret or its keys must be defined\n                          type: boolean\n                        secretName:\n                          description: 'Name of the secret in the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret'\n                          type: string\n                      type: object\n                    storageos:\n                      description: StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes.\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretRef:\n                          description: SecretRef specifies the secret to use for obtaining the StorageOS API credentials.  If not specified, default values will be attempted.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        volumeName:\n                          description: VolumeName is the human-readable name of the StorageOS volume.  Volume names are only unique within a namespace.\n                          type: string\n                        volumeNamespace:\n                          description: VolumeNamespace specifies the scope of the volume within StorageOS.  If no namespace is specified then the Pod's namespace will be used.  This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to \"default\" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created.\n                          type: string\n                      type: object\n                    vsphereVolume:\n                      description: VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        storagePolicyID:\n                          description: Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName.\n                          type: string\n                        storagePolicyName:\n                          description: Storage Policy Based Management (SPBM) profile name.\n                          type: string\n                        volumePath:\n                          description: Path that identifies vSphere volume vmdk\n                          type: string\n                      required:\n                      - volumePath\n                      type: object\n                  required:\n                  - name\n                  type: object\n                type: array\n            type: object\n          status:\n            description: 'Most recent observed status of the Alertmanager cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'\n            properties:\n              availableReplicas:\n                description: Total number of available pods (ready for at least minReadySeconds) targeted by this Alertmanager cluster.\n                format: int32\n                type: integer\n              paused:\n                description: Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.\n                type: boolean\n              replicas:\n                description: Total number of non-terminated pods targeted by this Alertmanager cluster (their labels match the selector).\n                format: int32\n                type: integer\n              unavailableReplicas:\n                description: Total number of unavailable pods targeted by this Alertmanager cluster.\n                format: int32\n                type: integer\n              updatedReplicas:\n                description: Total number of non-terminated pods targeted by this Alertmanager cluster that have the desired version spec.\n                format: int32\n                type: integer\n            required:\n            - availableReplicas\n            - paused\n            - replicas\n            - unavailableReplicas\n            - updatedReplicas\n            type: object\n        required:\n        - spec\n        type: object\n    served: true\n    storage: true\n    subresources: {}\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n---\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: podmonitors.monitoring.coreos.com\nspec:\n  group: monitoring.coreos.com\n  names:\n    categories:\n    - prometheus-operator\n    kind: PodMonitor\n    listKind: PodMonitorList\n    plural: podmonitors\n    singular: podmonitor\n  scope: Namespaced\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n        description: PodMonitor defines monitoring for a set of pods.\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: Specification of desired Pod selection for target discovery by Prometheus.\n            properties:\n              jobLabel:\n                description: The label to use to retrieve the job name from.\n                type: string\n              namespaceSelector:\n                description: Selector to select which namespaces the Endpoints objects are discovered from.\n                properties:\n                  any:\n                    description: Boolean describing whether all namespaces are selected in contrast to a list restricting them.\n                    type: boolean\n                  matchNames:\n                    description: List of namespace names.\n                    items:\n                      type: string\n                    type: array\n                type: object\n              podMetricsEndpoints:\n                description: A list of endpoints allowed as part of this PodMonitor.\n                items:\n                  description: PodMetricsEndpoint defines a scrapeable endpoint of a Kubernetes Pod serving Prometheus metrics.\n                  properties:\n                    basicAuth:\n                      description: 'BasicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint'\n                      properties:\n                        password:\n                          description: The secret in the service monitor namespace that contains the password for authentication.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                        username:\n                          description: The secret in the service monitor namespace that contains the username for authentication.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                      type: object\n                    bearerTokenSecret:\n                      description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator.\n                      properties:\n                        key:\n                          description: The key of the secret to select from.  Must be a valid secret key.\n                          type: string\n                        name:\n                          description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                          type: string\n                        optional:\n                          description: Specify whether the Secret or its key must be defined\n                          type: boolean\n                      required:\n                      - key\n                      type: object\n                    honorLabels:\n                      description: HonorLabels chooses the metric's labels on collisions with target labels.\n                      type: boolean\n                    honorTimestamps:\n                      description: HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data.\n                      type: boolean\n                    interval:\n                      description: Interval at which metrics should be scraped\n                      type: string\n                    metricRelabelings:\n                      description: MetricRelabelConfigs to apply to samples before ingestion.\n                      items:\n                        description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'\n                        properties:\n                          action:\n                            description: Action to perform based on regex matching. Default is 'replace'\n                            type: string\n                          modulus:\n                            description: Modulus to take of the hash of the source label values.\n                            format: int64\n                            type: integer\n                          regex:\n                            description: Regular expression against which the extracted value is matched. Default is '(.*)'\n                            type: string\n                          replacement:\n                            description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\n                            type: string\n                          separator:\n                            description: Separator placed between concatenated source label values. default is ';'.\n                            type: string\n                          sourceLabels:\n                            description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\n                            items:\n                              type: string\n                            type: array\n                          targetLabel:\n                            description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\n                            type: string\n                        type: object\n                      type: array\n                    params:\n                      additionalProperties:\n                        items:\n                          type: string\n                        type: array\n                      description: Optional HTTP URL parameters\n                      type: object\n                    path:\n                      description: HTTP path to scrape for metrics.\n                      type: string\n                    port:\n                      description: Name of the pod port this endpoint refers to. Mutually exclusive with targetPort.\n                      type: string\n                    proxyUrl:\n                      description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint.\n                      type: string\n                    relabelings:\n                      description: 'RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'\n                      items:\n                        description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'\n                        properties:\n                          action:\n                            description: Action to perform based on regex matching. Default is 'replace'\n                            type: string\n                          modulus:\n                            description: Modulus to take of the hash of the source label values.\n                            format: int64\n                            type: integer\n                          regex:\n                            description: Regular expression against which the extracted value is matched. Default is '(.*)'\n                            type: string\n                          replacement:\n                            description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\n                            type: string\n                          separator:\n                            description: Separator placed between concatenated source label values. default is ';'.\n                            type: string\n                          sourceLabels:\n                            description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\n                            items:\n                              type: string\n                            type: array\n                          targetLabel:\n                            description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\n                            type: string\n                        type: object\n                      type: array\n                    scheme:\n                      description: HTTP scheme to use for scraping.\n                      type: string\n                    scrapeTimeout:\n                      description: Timeout after which the scrape is ended\n                      type: string\n                    targetPort:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      description: 'Deprecated: Use ''port'' instead.'\n                      x-kubernetes-int-or-string: true\n                    tlsConfig:\n                      description: TLS configuration to use when scraping the endpoint.\n                      properties:\n                        ca:\n                          description: Struct containing the CA cert to use for the targets.\n                          properties:\n                            configMap:\n                              description: ConfigMap containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key to select.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the ConfigMap or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            secret:\n                              description: Secret containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                          type: object\n                        cert:\n                          description: Struct containing the client cert file for the targets.\n                          properties:\n                            configMap:\n                              description: ConfigMap containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key to select.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the ConfigMap or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            secret:\n                              description: Secret containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                          type: object\n                        insecureSkipVerify:\n                          description: Disable target certificate validation.\n                          type: boolean\n                        keySecret:\n                          description: Secret containing the client key file for the targets.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                        serverName:\n                          description: Used to verify the hostname for the targets.\n                          type: string\n                      type: object\n                  type: object\n                type: array\n              podTargetLabels:\n                description: PodTargetLabels transfers labels on the Kubernetes Pod onto the target.\n                items:\n                  type: string\n                type: array\n              sampleLimit:\n                description: SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.\n                format: int64\n                type: integer\n              selector:\n                description: Selector to select Pod objects.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              targetLimit:\n                description: TargetLimit defines a limit on the number of scraped targets that will be accepted.\n                format: int64\n                type: integer\n            required:\n            - podMetricsEndpoints\n            - selector\n            type: object\n        required:\n        - spec\n        type: object\n    served: true\n    storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n---\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: probes.monitoring.coreos.com\nspec:\n  group: monitoring.coreos.com\n  names:\n    categories:\n    - prometheus-operator\n    kind: Probe\n    listKind: ProbeList\n    plural: probes\n    singular: probe\n  scope: Namespaced\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n        description: Probe defines monitoring for a set of static targets or ingresses.\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: Specification of desired Ingress selection for target discovery by Prometheus.\n            properties:\n              basicAuth:\n                description: 'BasicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint'\n                properties:\n                  password:\n                    description: The secret in the service monitor namespace that contains the password for authentication.\n                    properties:\n                      key:\n                        description: The key of the secret to select from.  Must be a valid secret key.\n                        type: string\n                      name:\n                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                        type: string\n                      optional:\n                        description: Specify whether the Secret or its key must be defined\n                        type: boolean\n                    required:\n                    - key\n                    type: object\n                  username:\n                    description: The secret in the service monitor namespace that contains the username for authentication.\n                    properties:\n                      key:\n                        description: The key of the secret to select from.  Must be a valid secret key.\n                        type: string\n                      name:\n                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                        type: string\n                      optional:\n                        description: Specify whether the Secret or its key must be defined\n                        type: boolean\n                    required:\n                    - key\n                    type: object\n                type: object\n              bearerTokenSecret:\n                description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the probe and accessible by the Prometheus Operator.\n                properties:\n                  key:\n                    description: The key of the secret to select from.  Must be a valid secret key.\n                    type: string\n                  name:\n                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                    type: string\n                  optional:\n                    description: Specify whether the Secret or its key must be defined\n                    type: boolean\n                required:\n                - key\n                type: object\n              interval:\n                description: Interval at which targets are probed using the configured prober. If not specified Prometheus' global scrape interval is used.\n                type: string\n              jobName:\n                description: The job name assigned to scraped metrics by default.\n                type: string\n              module:\n                description: 'The module to use for probing specifying how to probe the target. Example module configuring in the blackbox exporter: https://github.com/prometheus/blackbox_exporter/blob/master/example.yml'\n                type: string\n              prober:\n                description: Specification for the prober to use for probing targets. The prober.URL parameter is required. Targets cannot be probed if left empty.\n                properties:\n                  path:\n                    description: Path to collect metrics from. Defaults to `/probe`.\n                    type: string\n                  scheme:\n                    description: HTTP scheme to use for scraping. Defaults to `http`.\n                    type: string\n                  url:\n                    description: Mandatory URL of the prober.\n                    type: string\n                required:\n                - url\n                type: object\n              scrapeTimeout:\n                description: Timeout for scraping metrics from the Prometheus exporter.\n                type: string\n              targets:\n                description: Targets defines a set of static and/or dynamically discovered targets to be probed using the prober.\n                properties:\n                  ingress:\n                    description: Ingress defines the set of dynamically discovered ingress objects which hosts are considered for probing.\n                    properties:\n                      namespaceSelector:\n                        description: Select Ingress objects by namespace.\n                        properties:\n                          any:\n                            description: Boolean describing whether all namespaces are selected in contrast to a list restricting them.\n                            type: boolean\n                          matchNames:\n                            description: List of namespace names.\n                            items:\n                              type: string\n                            type: array\n                        type: object\n                      relabelingConfigs:\n                        description: 'RelabelConfigs to apply to samples before ingestion. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'\n                        items:\n                          description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'\n                          properties:\n                            action:\n                              description: Action to perform based on regex matching. Default is 'replace'\n                              type: string\n                            modulus:\n                              description: Modulus to take of the hash of the source label values.\n                              format: int64\n                              type: integer\n                            regex:\n                              description: Regular expression against which the extracted value is matched. Default is '(.*)'\n                              type: string\n                            replacement:\n                              description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\n                              type: string\n                            separator:\n                              description: Separator placed between concatenated source label values. default is ';'.\n                              type: string\n                            sourceLabels:\n                              description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\n                              items:\n                                type: string\n                              type: array\n                            targetLabel:\n                              description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\n                              type: string\n                          type: object\n                        type: array\n                      selector:\n                        description: Select Ingress objects by labels.\n                        properties:\n                          matchExpressions:\n                            description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                            items:\n                              description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                              properties:\n                                key:\n                                  description: key is the label key that the selector applies to.\n                                  type: string\n                                operator:\n                                  description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                  type: string\n                                values:\n                                  description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                  items:\n                                    type: string\n                                  type: array\n                              required:\n                              - key\n                              - operator\n                              type: object\n                            type: array\n                          matchLabels:\n                            additionalProperties:\n                              type: string\n                            description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                            type: object\n                        type: object\n                    type: object\n                  staticConfig:\n                    description: 'StaticConfig defines static targets which are considers for probing. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config.'\n                    properties:\n                      labels:\n                        additionalProperties:\n                          type: string\n                        description: Labels assigned to all metrics scraped from the targets.\n                        type: object\n                      relabelingConfigs:\n                        description: 'RelabelConfigs to apply to samples before ingestion. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'\n                        items:\n                          description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'\n                          properties:\n                            action:\n                              description: Action to perform based on regex matching. Default is 'replace'\n                              type: string\n                            modulus:\n                              description: Modulus to take of the hash of the source label values.\n                              format: int64\n                              type: integer\n                            regex:\n                              description: Regular expression against which the extracted value is matched. Default is '(.*)'\n                              type: string\n                            replacement:\n                              description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\n                              type: string\n                            separator:\n                              description: Separator placed between concatenated source label values. default is ';'.\n                              type: string\n                            sourceLabels:\n                              description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\n                              items:\n                                type: string\n                              type: array\n                            targetLabel:\n                              description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\n                              type: string\n                          type: object\n                        type: array\n                      static:\n                        description: Targets is a list of URLs to probe using the configured prober.\n                        items:\n                          type: string\n                        type: array\n                    type: object\n                type: object\n              tlsConfig:\n                description: TLS configuration to use when scraping the endpoint.\n                properties:\n                  ca:\n                    description: Struct containing the CA cert to use for the targets.\n                    properties:\n                      configMap:\n                        description: ConfigMap containing data to use for the targets.\n                        properties:\n                          key:\n                            description: The key to select.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the ConfigMap or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                      secret:\n                        description: Secret containing data to use for the targets.\n                        properties:\n                          key:\n                            description: The key of the secret to select from.  Must be a valid secret key.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the Secret or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                    type: object\n                  cert:\n                    description: Struct containing the client cert file for the targets.\n                    properties:\n                      configMap:\n                        description: ConfigMap containing data to use for the targets.\n                        properties:\n                          key:\n                            description: The key to select.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the ConfigMap or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                      secret:\n                        description: Secret containing data to use for the targets.\n                        properties:\n                          key:\n                            description: The key of the secret to select from.  Must be a valid secret key.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the Secret or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                    type: object\n                  insecureSkipVerify:\n                    description: Disable target certificate validation.\n                    type: boolean\n                  keySecret:\n                    description: Secret containing the client key file for the targets.\n                    properties:\n                      key:\n                        description: The key of the secret to select from.  Must be a valid secret key.\n                        type: string\n                      name:\n                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                        type: string\n                      optional:\n                        description: Specify whether the Secret or its key must be defined\n                        type: boolean\n                    required:\n                    - key\n                    type: object\n                  serverName:\n                    description: Used to verify the hostname for the targets.\n                    type: string\n                type: object\n            type: object\n        required:\n        - spec\n        type: object\n    served: true\n    storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n---\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: prometheuses.monitoring.coreos.com\nspec:\n  group: monitoring.coreos.com\n  names:\n    categories:\n    - prometheus-operator\n    kind: Prometheus\n    listKind: PrometheusList\n    plural: prometheuses\n    singular: prometheus\n  scope: Namespaced\n  versions:\n  - additionalPrinterColumns:\n    - description: The version of Prometheus\n      jsonPath: .spec.version\n      name: Version\n      type: string\n    - description: The desired replicas number of Prometheuses\n      jsonPath: .spec.replicas\n      name: Replicas\n      type: integer\n    - jsonPath: .metadata.creationTimestamp\n      name: Age\n      type: date\n    name: v1\n    schema:\n      openAPIV3Schema:\n        description: Prometheus defines a Prometheus deployment.\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: 'Specification of the desired behavior of the Prometheus cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'\n            properties:\n              additionalAlertManagerConfigs:\n                description: 'AdditionalAlertManagerConfigs allows specifying a key of a Secret containing additional Prometheus AlertManager configurations. AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alertmanager_config. As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade.'\n                properties:\n                  key:\n                    description: The key of the secret to select from.  Must be a valid secret key.\n                    type: string\n                  name:\n                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                    type: string\n                  optional:\n                    description: Specify whether the Secret or its key must be defined\n                    type: boolean\n                required:\n                - key\n                type: object\n              additionalAlertRelabelConfigs:\n                description: 'AdditionalAlertRelabelConfigs allows specifying a key of a Secret containing additional Prometheus alert relabel configurations. Alert relabel configurations specified are appended to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel configs are going to break Prometheus after the upgrade.'\n                properties:\n                  key:\n                    description: The key of the secret to select from.  Must be a valid secret key.\n                    type: string\n                  name:\n                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                    type: string\n                  optional:\n                    description: Specify whether the Secret or its key must be defined\n                    type: boolean\n                required:\n                - key\n                type: object\n              additionalScrapeConfigs:\n                description: 'AdditionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape configurations specified are appended to the configurations generated by the Prometheus Operator. Job configurations specified must have the form as specified in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible scrape configs are going to break Prometheus after the upgrade.'\n                properties:\n                  key:\n                    description: The key of the secret to select from.  Must be a valid secret key.\n                    type: string\n                  name:\n                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                    type: string\n                  optional:\n                    description: Specify whether the Secret or its key must be defined\n                    type: boolean\n                required:\n                - key\n                type: object\n              affinity:\n                description: If specified, the pod's scheduling constraints.\n                properties:\n                  nodeAffinity:\n                    description: Describes node affinity scheduling rules for the pod.\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\n                          properties:\n                            preference:\n                              description: A node selector term, associated with the corresponding weight.\n                              properties:\n                                matchExpressions:\n                                  description: A list of node selector requirements by node's labels.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchFields:\n                                  description: A list of node selector requirements by node's fields.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                              type: object\n                            weight:\n                              description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - preference\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.\n                        properties:\n                          nodeSelectorTerms:\n                            description: Required. A list of node selector terms. The terms are ORed.\n                            items:\n                              description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\n                              properties:\n                                matchExpressions:\n                                  description: A list of node selector requirements by node's labels.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchFields:\n                                  description: A list of node selector requirements by node's fields.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                              type: object\n                            type: array\n                        required:\n                        - nodeSelectorTerms\n                        type: object\n                    type: object\n                  podAffinity:\n                    description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n                          properties:\n                            podAffinityTerm:\n                              description: Required. A pod affinity term, associated with the corresponding weight.\n                              properties:\n                                labelSelector:\n                                  description: A label query over a set of resources, in this case pods.\n                                  properties:\n                                    matchExpressions:\n                                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                      items:\n                                        description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                        properties:\n                                          key:\n                                            description: key is the label key that the selector applies to.\n                                            type: string\n                                          operator:\n                                            description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                            type: string\n                                          values:\n                                            description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                            items:\n                                              type: string\n                                            type: array\n                                        required:\n                                        - key\n                                        - operator\n                                        type: object\n                                      type: array\n                                    matchLabels:\n                                      additionalProperties:\n                                        type: string\n                                      description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                      type: object\n                                  type: object\n                                namespaces:\n                                  description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                                  items:\n                                    type: string\n                                  type: array\n                                topologyKey:\n                                  description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                                  type: string\n                              required:\n                              - topologyKey\n                              type: object\n                            weight:\n                              description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - podAffinityTerm\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\n                        items:\n                          description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key <topologyKey> matches that of any node on which a pod of the set of pods is running\n                          properties:\n                            labelSelector:\n                              description: A label query over a set of resources, in this case pods.\n                              properties:\n                                matchExpressions:\n                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                  items:\n                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: key is the label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                        type: string\n                                      values:\n                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchLabels:\n                                  additionalProperties:\n                                    type: string\n                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                  type: object\n                              type: object\n                            namespaces:\n                              description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                              items:\n                                type: string\n                              type: array\n                            topologyKey:\n                              description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                              type: string\n                          required:\n                          - topologyKey\n                          type: object\n                        type: array\n                    type: object\n                  podAntiAffinity:\n                    description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n                          properties:\n                            podAffinityTerm:\n                              description: Required. A pod affinity term, associated with the corresponding weight.\n                              properties:\n                                labelSelector:\n                                  description: A label query over a set of resources, in this case pods.\n                                  properties:\n                                    matchExpressions:\n                                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                      items:\n                                        description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                        properties:\n                                          key:\n                                            description: key is the label key that the selector applies to.\n                                            type: string\n                                          operator:\n                                            description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                            type: string\n                                          values:\n                                            description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                            items:\n                                              type: string\n                                            type: array\n                                        required:\n                                        - key\n                                        - operator\n                                        type: object\n                                      type: array\n                                    matchLabels:\n                                      additionalProperties:\n                                        type: string\n                                      description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                      type: object\n                                  type: object\n                                namespaces:\n                                  description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                                  items:\n                                    type: string\n                                  type: array\n                                topologyKey:\n                                  description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                                  type: string\n                              required:\n                              - topologyKey\n                              type: object\n                            weight:\n                              description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - podAffinityTerm\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\n                        items:\n                          description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key <topologyKey> matches that of any node on which a pod of the set of pods is running\n                          properties:\n                            labelSelector:\n                              description: A label query over a set of resources, in this case pods.\n                              properties:\n                                matchExpressions:\n                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                  items:\n                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: key is the label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                        type: string\n                                      values:\n                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchLabels:\n                                  additionalProperties:\n                                    type: string\n                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                  type: object\n                              type: object\n                            namespaces:\n                              description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                              items:\n                                type: string\n                              type: array\n                            topologyKey:\n                              description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                              type: string\n                          required:\n                          - topologyKey\n                          type: object\n                        type: array\n                    type: object\n                type: object\n              alerting:\n                description: Define details regarding alerting.\n                properties:\n                  alertmanagers:\n                    description: AlertmanagerEndpoints Prometheus should fire alerts against.\n                    items:\n                      description: AlertmanagerEndpoints defines a selection of a single Endpoints object containing alertmanager IPs to fire alerts against.\n                      properties:\n                        apiVersion:\n                          description: Version of the Alertmanager API that Prometheus uses to send alerts. It can be \"v1\" or \"v2\".\n                          type: string\n                        bearerTokenFile:\n                          description: BearerTokenFile to read from filesystem to use when authenticating to Alertmanager.\n                          type: string\n                        name:\n                          description: Name of Endpoints object in Namespace.\n                          type: string\n                        namespace:\n                          description: Namespace of Endpoints object.\n                          type: string\n                        pathPrefix:\n                          description: Prefix for the HTTP path alerts are pushed to.\n                          type: string\n                        port:\n                          anyOf:\n                          - type: integer\n                          - type: string\n                          description: Port the Alertmanager API is exposed on.\n                          x-kubernetes-int-or-string: true\n                        scheme:\n                          description: Scheme to use when firing alerts.\n                          type: string\n                        timeout:\n                          description: Timeout is a per-target Alertmanager timeout when pushing alerts.\n                          type: string\n                        tlsConfig:\n                          description: TLS Config to use for alertmanager connection.\n                          properties:\n                            ca:\n                              description: Struct containing the CA cert to use for the targets.\n                              properties:\n                                configMap:\n                                  description: ConfigMap containing data to use for the targets.\n                                  properties:\n                                    key:\n                                      description: The key to select.\n                                      type: string\n                                    name:\n                                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                      type: string\n                                    optional:\n                                      description: Specify whether the ConfigMap or its key must be defined\n                                      type: boolean\n                                  required:\n                                  - key\n                                  type: object\n                                secret:\n                                  description: Secret containing data to use for the targets.\n                                  properties:\n                                    key:\n                                      description: The key of the secret to select from.  Must be a valid secret key.\n                                      type: string\n                                    name:\n                                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                      type: string\n                                    optional:\n                                      description: Specify whether the Secret or its key must be defined\n                                      type: boolean\n                                  required:\n                                  - key\n                                  type: object\n                              type: object\n                            caFile:\n                              description: Path to the CA cert in the Prometheus container to use for the targets.\n                              type: string\n                            cert:\n                              description: Struct containing the client cert file for the targets.\n                              properties:\n                                configMap:\n                                  description: ConfigMap containing data to use for the targets.\n                                  properties:\n                                    key:\n                                      description: The key to select.\n                                      type: string\n                                    name:\n                                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                      type: string\n                                    optional:\n                                      description: Specify whether the ConfigMap or its key must be defined\n                                      type: boolean\n                                  required:\n                                  - key\n                                  type: object\n                                secret:\n                                  description: Secret containing data to use for the targets.\n                                  properties:\n                                    key:\n                                      description: The key of the secret to select from.  Must be a valid secret key.\n                                      type: string\n                                    name:\n                                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                      type: string\n                                    optional:\n                                      description: Specify whether the Secret or its key must be defined\n                                      type: boolean\n                                  required:\n                                  - key\n                                  type: object\n                              type: object\n                            certFile:\n                              description: Path to the client cert file in the Prometheus container for the targets.\n                              type: string\n                            insecureSkipVerify:\n                              description: Disable target certificate validation.\n                              type: boolean\n                            keyFile:\n                              description: Path to the client key file in the Prometheus container for the targets.\n                              type: string\n                            keySecret:\n                              description: Secret containing the client key file for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            serverName:\n                              description: Used to verify the hostname for the targets.\n                              type: string\n                          type: object\n                      required:\n                      - name\n                      - namespace\n                      - port\n                      type: object\n                    type: array\n                required:\n                - alertmanagers\n                type: object\n              allowOverlappingBlocks:\n                description: AllowOverlappingBlocks enables vertical compaction and vertical query merge in Prometheus. This is still experimental in Prometheus so it may change in any upcoming release.\n                type: boolean\n              apiserverConfig:\n                description: APIServerConfig allows specifying a host and auth methods to access apiserver. If left empty, Prometheus is assumed to run inside of the cluster and will discover API servers automatically and use the pod's CA certificate and bearer token file at /var/run/secrets/kubernetes.io/serviceaccount/.\n                properties:\n                  basicAuth:\n                    description: BasicAuth allow an endpoint to authenticate over basic authentication\n                    properties:\n                      password:\n                        description: The secret in the service monitor namespace that contains the password for authentication.\n                        properties:\n                          key:\n                            description: The key of the secret to select from.  Must be a valid secret key.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the Secret or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                      username:\n                        description: The secret in the service monitor namespace that contains the username for authentication.\n                        properties:\n                          key:\n                            description: The key of the secret to select from.  Must be a valid secret key.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the Secret or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                    type: object\n                  bearerToken:\n                    description: Bearer token for accessing apiserver.\n                    type: string\n                  bearerTokenFile:\n                    description: File to read bearer token for accessing apiserver.\n                    type: string\n                  host:\n                    description: Host of apiserver. A valid string consisting of a hostname or IP followed by an optional port number\n                    type: string\n                  tlsConfig:\n                    description: TLS Config to use for accessing apiserver.\n                    properties:\n                      ca:\n                        description: Struct containing the CA cert to use for the targets.\n                        properties:\n                          configMap:\n                            description: ConfigMap containing data to use for the targets.\n                            properties:\n                              key:\n                                description: The key to select.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          secret:\n                            description: Secret containing data to use for the targets.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                        type: object\n                      caFile:\n                        description: Path to the CA cert in the Prometheus container to use for the targets.\n                        type: string\n                      cert:\n                        description: Struct containing the client cert file for the targets.\n                        properties:\n                          configMap:\n                            description: ConfigMap containing data to use for the targets.\n                            properties:\n                              key:\n                                description: The key to select.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          secret:\n                            description: Secret containing data to use for the targets.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                        type: object\n                      certFile:\n                        description: Path to the client cert file in the Prometheus container for the targets.\n                        type: string\n                      insecureSkipVerify:\n                        description: Disable target certificate validation.\n                        type: boolean\n                      keyFile:\n                        description: Path to the client key file in the Prometheus container for the targets.\n                        type: string\n                      keySecret:\n                        description: Secret containing the client key file for the targets.\n                        properties:\n                          key:\n                            description: The key of the secret to select from.  Must be a valid secret key.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the Secret or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                      serverName:\n                        description: Used to verify the hostname for the targets.\n                        type: string\n                    type: object\n                required:\n                - host\n                type: object\n              arbitraryFSAccessThroughSMs:\n                description: ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files on the file system of the Prometheus container e.g. bearer token files.\n                properties:\n                  deny:\n                    type: boolean\n                type: object\n              baseImage:\n                description: 'Base image to use for a Prometheus deployment. Deprecated: use ''image'' instead'\n                type: string\n              configMaps:\n                description: ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/<configmap-name>.\n                items:\n                  type: string\n                type: array\n              containers:\n                description: 'Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a Prometheus pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `prometheus`, `config-reloader`, and `thanos-sidecar`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.'\n                items:\n                  description: A single application container that you want to run within a pod.\n                  properties:\n                    args:\n                      description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    command:\n                      description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    env:\n                      description: List of environment variables to set in the container. Cannot be updated.\n                      items:\n                        description: EnvVar represents an environment variable present in a Container.\n                        properties:\n                          name:\n                            description: Name of the environment variable. Must be a C_IDENTIFIER.\n                            type: string\n                          value:\n                            description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".'\n                            type: string\n                          valueFrom:\n                            description: Source for the environment variable's value. Cannot be used if value is not empty.\n                            properties:\n                              configMapKeyRef:\n                                description: Selects a key of a ConfigMap.\n                                properties:\n                                  key:\n                                    description: The key to select.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              fieldRef:\n                                description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                              secretKeyRef:\n                                description: Selects a key of a secret in the pod's namespace\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                            type: object\n                        required:\n                        - name\n                        type: object\n                      type: array\n                    envFrom:\n                      description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\n                      items:\n                        description: EnvFromSource represents the source of a set of ConfigMaps\n                        properties:\n                          configMapRef:\n                            description: The ConfigMap to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap must be defined\n                                type: boolean\n                            type: object\n                          prefix:\n                            description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\n                            type: string\n                          secretRef:\n                            description: The Secret to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret must be defined\n                                type: boolean\n                            type: object\n                        type: object\n                      type: array\n                    image:\n                      description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.'\n                      type: string\n                    imagePullPolicy:\n                      description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'\n                      type: string\n                    lifecycle:\n                      description: Actions that the management system should take in response to container lifecycle events. Cannot be updated.\n                      properties:\n                        postStart:\n                          description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                        preStop:\n                          description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                      type: object\n                    livenessProbe:\n                      description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    name:\n                      description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\n                      type: string\n                    ports:\n                      description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.\n                      items:\n                        description: ContainerPort represents a network port in a single container.\n                        properties:\n                          containerPort:\n                            description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.\n                            format: int32\n                            type: integer\n                          hostIP:\n                            description: What host IP to bind the external port to.\n                            type: string\n                          hostPort:\n                            description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\n                            format: int32\n                            type: integer\n                          name:\n                            description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\n                            type: string\n                          protocol:\n                            default: TCP\n                            description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".\n                            type: string\n                        required:\n                        - containerPort\n                        type: object\n                      type: array\n                      x-kubernetes-list-map-keys:\n                      - containerPort\n                      - protocol\n                      x-kubernetes-list-type: map\n                    readinessProbe:\n                      description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    resources:\n                      description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                      properties:\n                        limits:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                        requests:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                      type: object\n                    securityContext:\n                      description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/'\n                      properties:\n                        allowPrivilegeEscalation:\n                          description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN'\n                          type: boolean\n                        capabilities:\n                          description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.\n                          properties:\n                            add:\n                              description: Added capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                            drop:\n                              description: Removed capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                          type: object\n                        privileged:\n                          description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\n                          type: boolean\n                        procMount:\n                          description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\n                          type: string\n                        readOnlyRootFilesystem:\n                          description: Whether this container has a read-only root filesystem. Default is false.\n                          type: boolean\n                        runAsGroup:\n                          description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        runAsNonRoot:\n                          description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          type: boolean\n                        runAsUser:\n                          description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        seLinuxOptions:\n                          description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            level:\n                              description: Level is SELinux level label that applies to the container.\n                              type: string\n                            role:\n                              description: Role is a SELinux role label that applies to the container.\n                              type: string\n                            type:\n                              description: Type is a SELinux type label that applies to the container.\n                              type: string\n                            user:\n                              description: User is a SELinux user label that applies to the container.\n                              type: string\n                          type: object\n                        windowsOptions:\n                          description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            gmsaCredentialSpec:\n                              description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                              type: string\n                            gmsaCredentialSpecName:\n                              description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                              type: string\n                            runAsUserName:\n                              description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                              type: string\n                          type: object\n                      type: object\n                    startupProbe:\n                      description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    stdin:\n                      description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\n                      type: boolean\n                    stdinOnce:\n                      description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\n                      type: boolean\n                    terminationMessagePath:\n                      description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.'\n                      type: string\n                    terminationMessagePolicy:\n                      description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\n                      type: string\n                    tty:\n                      description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\n                      type: boolean\n                    volumeDevices:\n                      description: volumeDevices is the list of block devices to be used by the container.\n                      items:\n                        description: volumeDevice describes a mapping of a raw block device within a container.\n                        properties:\n                          devicePath:\n                            description: devicePath is the path inside of the container that the device will be mapped to.\n                            type: string\n                          name:\n                            description: name must match the name of a persistentVolumeClaim in the pod\n                            type: string\n                        required:\n                        - devicePath\n                        - name\n                        type: object\n                      type: array\n                    volumeMounts:\n                      description: Pod volumes to mount into the container's filesystem. Cannot be updated.\n                      items:\n                        description: VolumeMount describes a mounting of a Volume within a container.\n                        properties:\n                          mountPath:\n                            description: Path within the container at which the volume should be mounted.  Must not contain ':'.\n                            type: string\n                          mountPropagation:\n                            description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n                            type: string\n                          name:\n                            description: This must match the Name of a Volume.\n                            type: string\n                          readOnly:\n                            description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\n                            type: boolean\n                          subPath:\n                            description: Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).\n                            type: string\n                          subPathExpr:\n                            description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\n                            type: string\n                        required:\n                        - mountPath\n                        - name\n                        type: object\n                      type: array\n                    workingDir:\n                      description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\n                      type: string\n                  required:\n                  - name\n                  type: object\n                type: array\n              disableCompaction:\n                description: Disable prometheus compaction.\n                type: boolean\n              enableAdminAPI:\n                description: 'Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis'\n                type: boolean\n              enableFeatures:\n                description: Enable access to Prometheus disabled features. By default, no features are enabled. Enabling disabled features is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice. For more information see https://prometheus.io/docs/prometheus/latest/disabled_features/\n                items:\n                  type: string\n                type: array\n              enforcedNamespaceLabel:\n                description: EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created.\n                type: string\n              enforcedSampleLimit:\n                description: EnforcedSampleLimit defines global limit on number of scraped samples that will be accepted. This overrides any SampleLimit set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the SampleLimit to keep overall number of samples/series under the desired limit. Note that if SampleLimit is lower that value will be taken instead.\n                format: int64\n                type: integer\n              enforcedTargetLimit:\n                description: EnforcedTargetLimit defines a global limit on the number of scraped targets. This overrides any TargetLimit set per ServiceMonitor or/and PodMonitor. It is meant to be used by admins to enforce the TargetLimit to keep overall number of targets under the desired limit. Note that if TargetLimit is higher that value will be taken instead.\n                format: int64\n                type: integer\n              evaluationInterval:\n                description: Interval between consecutive evaluations.\n                type: string\n              externalLabels:\n                additionalProperties:\n                  type: string\n                description: The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).\n                type: object\n              externalUrl:\n                description: The external URL the Prometheus instances will be available under. This is necessary to generate correct URLs. This is necessary if Prometheus is not served from root of a DNS name.\n                type: string\n              ignoreNamespaceSelectors:\n                description: IgnoreNamespaceSelectors if set to true will ignore NamespaceSelector settings from the podmonitor and servicemonitor configs, and they will only discover endpoints within their current namespace.  Defaults to false.\n                type: boolean\n              image:\n                description: Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Prometheus is being configured.\n                type: string\n              imagePullSecrets:\n                description: An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod\n                items:\n                  description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.\n                  properties:\n                    name:\n                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                      type: string\n                  type: object\n                type: array\n              initContainers:\n                description: 'InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the Prometheus configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ Using initContainers for any use case other then secret fetching is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.'\n                items:\n                  description: A single application container that you want to run within a pod.\n                  properties:\n                    args:\n                      description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    command:\n                      description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    env:\n                      description: List of environment variables to set in the container. Cannot be updated.\n                      items:\n                        description: EnvVar represents an environment variable present in a Container.\n                        properties:\n                          name:\n                            description: Name of the environment variable. Must be a C_IDENTIFIER.\n                            type: string\n                          value:\n                            description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".'\n                            type: string\n                          valueFrom:\n                            description: Source for the environment variable's value. Cannot be used if value is not empty.\n                            properties:\n                              configMapKeyRef:\n                                description: Selects a key of a ConfigMap.\n                                properties:\n                                  key:\n                                    description: The key to select.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              fieldRef:\n                                description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                              secretKeyRef:\n                                description: Selects a key of a secret in the pod's namespace\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                            type: object\n                        required:\n                        - name\n                        type: object\n                      type: array\n                    envFrom:\n                      description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\n                      items:\n                        description: EnvFromSource represents the source of a set of ConfigMaps\n                        properties:\n                          configMapRef:\n                            description: The ConfigMap to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap must be defined\n                                type: boolean\n                            type: object\n                          prefix:\n                            description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\n                            type: string\n                          secretRef:\n                            description: The Secret to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret must be defined\n                                type: boolean\n                            type: object\n                        type: object\n                      type: array\n                    image:\n                      description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.'\n                      type: string\n                    imagePullPolicy:\n                      description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'\n                      type: string\n                    lifecycle:\n                      description: Actions that the management system should take in response to container lifecycle events. Cannot be updated.\n                      properties:\n                        postStart:\n                          description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                        preStop:\n                          description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                      type: object\n                    livenessProbe:\n                      description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    name:\n                      description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\n                      type: string\n                    ports:\n                      description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.\n                      items:\n                        description: ContainerPort represents a network port in a single container.\n                        properties:\n                          containerPort:\n                            description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.\n                            format: int32\n                            type: integer\n                          hostIP:\n                            description: What host IP to bind the external port to.\n                            type: string\n                          hostPort:\n                            description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\n                            format: int32\n                            type: integer\n                          name:\n                            description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\n                            type: string\n                          protocol:\n                            default: TCP\n                            description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".\n                            type: string\n                        required:\n                        - containerPort\n                        type: object\n                      type: array\n                      x-kubernetes-list-map-keys:\n                      - containerPort\n                      - protocol\n                      x-kubernetes-list-type: map\n                    readinessProbe:\n                      description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    resources:\n                      description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                      properties:\n                        limits:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                        requests:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                      type: object\n                    securityContext:\n                      description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/'\n                      properties:\n                        allowPrivilegeEscalation:\n                          description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN'\n                          type: boolean\n                        capabilities:\n                          description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.\n                          properties:\n                            add:\n                              description: Added capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                            drop:\n                              description: Removed capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                          type: object\n                        privileged:\n                          description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\n                          type: boolean\n                        procMount:\n                          description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\n                          type: string\n                        readOnlyRootFilesystem:\n                          description: Whether this container has a read-only root filesystem. Default is false.\n                          type: boolean\n                        runAsGroup:\n                          description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        runAsNonRoot:\n                          description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          type: boolean\n                        runAsUser:\n                          description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        seLinuxOptions:\n                          description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            level:\n                              description: Level is SELinux level label that applies to the container.\n                              type: string\n                            role:\n                              description: Role is a SELinux role label that applies to the container.\n                              type: string\n                            type:\n                              description: Type is a SELinux type label that applies to the container.\n                              type: string\n                            user:\n                              description: User is a SELinux user label that applies to the container.\n                              type: string\n                          type: object\n                        windowsOptions:\n                          description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            gmsaCredentialSpec:\n                              description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                              type: string\n                            gmsaCredentialSpecName:\n                              description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                              type: string\n                            runAsUserName:\n                              description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                              type: string\n                          type: object\n                      type: object\n                    startupProbe:\n                      description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    stdin:\n                      description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\n                      type: boolean\n                    stdinOnce:\n                      description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\n                      type: boolean\n                    terminationMessagePath:\n                      description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.'\n                      type: string\n                    terminationMessagePolicy:\n                      description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\n                      type: string\n                    tty:\n                      description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\n                      type: boolean\n                    volumeDevices:\n                      description: volumeDevices is the list of block devices to be used by the container.\n                      items:\n                        description: volumeDevice describes a mapping of a raw block device within a container.\n                        properties:\n                          devicePath:\n                            description: devicePath is the path inside of the container that the device will be mapped to.\n                            type: string\n                          name:\n                            description: name must match the name of a persistentVolumeClaim in the pod\n                            type: string\n                        required:\n                        - devicePath\n                        - name\n                        type: object\n                      type: array\n                    volumeMounts:\n                      description: Pod volumes to mount into the container's filesystem. Cannot be updated.\n                      items:\n                        description: VolumeMount describes a mounting of a Volume within a container.\n                        properties:\n                          mountPath:\n                            description: Path within the container at which the volume should be mounted.  Must not contain ':'.\n                            type: string\n                          mountPropagation:\n                            description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n                            type: string\n                          name:\n                            description: This must match the Name of a Volume.\n                            type: string\n                          readOnly:\n                            description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\n                            type: boolean\n                          subPath:\n                            description: Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).\n                            type: string\n                          subPathExpr:\n                            description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\n                            type: string\n                        required:\n                        - mountPath\n                        - name\n                        type: object\n                      type: array\n                    workingDir:\n                      description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\n                      type: string\n                  required:\n                  - name\n                  type: object\n                type: array\n              listenLocal:\n                description: ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP.\n                type: boolean\n              logFormat:\n                description: Log format for Prometheus to be configured with.\n                type: string\n              logLevel:\n                description: Log level for Prometheus to be configured with.\n                type: string\n              nodeSelector:\n                additionalProperties:\n                  type: string\n                description: Define which Nodes the Pods are scheduled on.\n                type: object\n              overrideHonorLabels:\n                description: OverrideHonorLabels if set to true overrides all user configured honor_labels. If HonorLabels is set in ServiceMonitor or PodMonitor to true, this overrides honor_labels to false.\n                type: boolean\n              overrideHonorTimestamps:\n                description: OverrideHonorTimestamps allows to globally enforce honoring timestamps in all scrape configs.\n                type: boolean\n              paused:\n                description: When a Prometheus deployment is paused, no actions except for deletion will be performed on the underlying objects.\n                type: boolean\n              podMetadata:\n                description: PodMetadata configures Labels and Annotations which are propagated to the prometheus pods.\n                properties:\n                  annotations:\n                    additionalProperties:\n                      type: string\n                    description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'\n                    type: object\n                  labels:\n                    additionalProperties:\n                      type: string\n                    description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'\n                    type: object\n                  name:\n                    description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'\n                    type: string\n                type: object\n              podMonitorNamespaceSelector:\n                description: Namespace's labels to match for PodMonitor discovery. If nil, only check own namespace.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              podMonitorSelector:\n                description: '*Experimental* PodMonitors to be selected for target discovery. *Deprecated:* if neither this nor serviceMonitorSelector are specified, configuration is unmanaged.'\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              portName:\n                description: Port name used for the pods and governing service. This defaults to web\n                type: string\n              priorityClassName:\n                description: Priority class assigned to the Pods\n                type: string\n              probeNamespaceSelector:\n                description: '*Experimental* Namespaces to be selected for Probe discovery. If nil, only check own namespace.'\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              probeSelector:\n                description: '*Experimental* Probes to be selected for target discovery.'\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              prometheusExternalLabelName:\n                description: Name of Prometheus external label used to denote Prometheus instance name. Defaults to the value of `prometheus`. External label will _not_ be added when value is set to empty string (`\"\"`).\n                type: string\n              prometheusRulesExcludedFromEnforce:\n                description: PrometheusRulesExcludedFromEnforce - list of prometheus rules to be excluded from enforcing of adding namespace labels. Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair\n                items:\n                  description: PrometheusRuleExcludeConfig enables users to configure excluded PrometheusRule names and their namespaces to be ignored while enforcing namespace label for alerts and metrics.\n                  properties:\n                    ruleName:\n                      description: RuleNamespace - name of excluded rule\n                      type: string\n                    ruleNamespace:\n                      description: RuleNamespace - namespace of excluded rule\n                      type: string\n                  required:\n                  - ruleName\n                  - ruleNamespace\n                  type: object\n                type: array\n              query:\n                description: QuerySpec defines the query command line flags when starting Prometheus.\n                properties:\n                  lookbackDelta:\n                    description: The delta difference allowed for retrieving metrics during expression evaluations.\n                    type: string\n                  maxConcurrency:\n                    description: Number of concurrent queries that can be run at once.\n                    format: int32\n                    type: integer\n                  maxSamples:\n                    description: Maximum number of samples a single query can load into memory. Note that queries will fail if they would load more samples than this into memory, so this also limits the number of samples a query can return.\n                    format: int32\n                    type: integer\n                  timeout:\n                    description: Maximum time a query may take before being aborted.\n                    type: string\n                type: object\n              queryLogFile:\n                description: QueryLogFile specifies the file to which PromQL queries are logged. Note that this location must be writable, and can be persisted using an attached volume. Alternatively, the location can be set to a stdout location such as `/dev/stdout` to log querie information to the default Prometheus log stream. This is only available in versions of Prometheus >= 2.16.0. For more details, see the Prometheus docs (https://prometheus.io/docs/guides/query-log/)\n                type: string\n              remoteRead:\n                description: If specified, the remote_read spec. This is an experimental feature, it may change in any upcoming release in a breaking way.\n                items:\n                  description: RemoteReadSpec defines the remote_read configuration for prometheus.\n                  properties:\n                    basicAuth:\n                      description: BasicAuth for the URL.\n                      properties:\n                        password:\n                          description: The secret in the service monitor namespace that contains the password for authentication.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                        username:\n                          description: The secret in the service monitor namespace that contains the username for authentication.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                      type: object\n                    bearerToken:\n                      description: Bearer token for remote read.\n                      type: string\n                    bearerTokenFile:\n                      description: File to read bearer token for remote read.\n                      type: string\n                    name:\n                      description: The name of the remote read queue, must be unique if specified. The name is used in metrics and logging in order to differentiate read configurations.  Only valid in Prometheus versions 2.15.0 and newer.\n                      type: string\n                    proxyUrl:\n                      description: Optional ProxyURL\n                      type: string\n                    readRecent:\n                      description: Whether reads should be made for queries for time ranges that the local storage should have complete data for.\n                      type: boolean\n                    remoteTimeout:\n                      description: Timeout for requests to the remote read endpoint.\n                      type: string\n                    requiredMatchers:\n                      additionalProperties:\n                        type: string\n                      description: An optional list of equality matchers which have to be present in a selector to query the remote read endpoint.\n                      type: object\n                    tlsConfig:\n                      description: TLS Config to use for remote read.\n                      properties:\n                        ca:\n                          description: Struct containing the CA cert to use for the targets.\n                          properties:\n                            configMap:\n                              description: ConfigMap containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key to select.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the ConfigMap or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            secret:\n                              description: Secret containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                          type: object\n                        caFile:\n                          description: Path to the CA cert in the Prometheus container to use for the targets.\n                          type: string\n                        cert:\n                          description: Struct containing the client cert file for the targets.\n                          properties:\n                            configMap:\n                              description: ConfigMap containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key to select.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the ConfigMap or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            secret:\n                              description: Secret containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                          type: object\n                        certFile:\n                          description: Path to the client cert file in the Prometheus container for the targets.\n                          type: string\n                        insecureSkipVerify:\n                          description: Disable target certificate validation.\n                          type: boolean\n                        keyFile:\n                          description: Path to the client key file in the Prometheus container for the targets.\n                          type: string\n                        keySecret:\n                          description: Secret containing the client key file for the targets.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                        serverName:\n                          description: Used to verify the hostname for the targets.\n                          type: string\n                      type: object\n                    url:\n                      description: The URL of the endpoint to send samples to.\n                      type: string\n                  required:\n                  - url\n                  type: object\n                type: array\n              remoteWrite:\n                description: If specified, the remote_write spec. This is an experimental feature, it may change in any upcoming release in a breaking way.\n                items:\n                  description: RemoteWriteSpec defines the remote_write configuration for prometheus.\n                  properties:\n                    basicAuth:\n                      description: BasicAuth for the URL.\n                      properties:\n                        password:\n                          description: The secret in the service monitor namespace that contains the password for authentication.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                        username:\n                          description: The secret in the service monitor namespace that contains the username for authentication.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                      type: object\n                    bearerToken:\n                      description: Bearer token for remote write.\n                      type: string\n                    bearerTokenFile:\n                      description: File to read bearer token for remote write.\n                      type: string\n                    headers:\n                      additionalProperties:\n                        type: string\n                      description: Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. Only valid in Prometheus versions 2.25.0 and newer.\n                      type: object\n                    metadataConfig:\n                      description: MetadataConfig configures the sending of series metadata to remote storage.\n                      properties:\n                        send:\n                          description: Whether metric metadata is sent to remote storage or not.\n                          type: boolean\n                        sendInterval:\n                          description: How frequently metric metadata is sent to remote storage.\n                          type: string\n                      type: object\n                    name:\n                      description: The name of the remote write queue, must be unique if specified. The name is used in metrics and logging in order to differentiate queues. Only valid in Prometheus versions 2.15.0 and newer.\n                      type: string\n                    proxyUrl:\n                      description: Optional ProxyURL\n                      type: string\n                    queueConfig:\n                      description: QueueConfig allows tuning of the remote write queue parameters.\n                      properties:\n                        batchSendDeadline:\n                          description: BatchSendDeadline is the maximum time a sample will wait in buffer.\n                          type: string\n                        capacity:\n                          description: Capacity is the number of samples to buffer per shard before we start dropping them.\n                          type: integer\n                        maxBackoff:\n                          description: MaxBackoff is the maximum retry delay.\n                          type: string\n                        maxRetries:\n                          description: MaxRetries is the maximum number of times to retry a batch on recoverable errors.\n                          type: integer\n                        maxSamplesPerSend:\n                          description: MaxSamplesPerSend is the maximum number of samples per send.\n                          type: integer\n                        maxShards:\n                          description: MaxShards is the maximum number of shards, i.e. amount of concurrency.\n                          type: integer\n                        minBackoff:\n                          description: MinBackoff is the initial retry delay. Gets doubled for every retry.\n                          type: string\n                        minShards:\n                          description: MinShards is the minimum number of shards, i.e. amount of concurrency.\n                          type: integer\n                      type: object\n                    remoteTimeout:\n                      description: Timeout for requests to the remote write endpoint.\n                      type: string\n                    tlsConfig:\n                      description: TLS Config to use for remote write.\n                      properties:\n                        ca:\n                          description: Struct containing the CA cert to use for the targets.\n                          properties:\n                            configMap:\n                              description: ConfigMap containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key to select.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the ConfigMap or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            secret:\n                              description: Secret containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                          type: object\n                        caFile:\n                          description: Path to the CA cert in the Prometheus container to use for the targets.\n                          type: string\n                        cert:\n                          description: Struct containing the client cert file for the targets.\n                          properties:\n                            configMap:\n                              description: ConfigMap containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key to select.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the ConfigMap or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            secret:\n                              description: Secret containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                          type: object\n                        certFile:\n                          description: Path to the client cert file in the Prometheus container for the targets.\n                          type: string\n                        insecureSkipVerify:\n                          description: Disable target certificate validation.\n                          type: boolean\n                        keyFile:\n                          description: Path to the client key file in the Prometheus container for the targets.\n                          type: string\n                        keySecret:\n                          description: Secret containing the client key file for the targets.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                        serverName:\n                          description: Used to verify the hostname for the targets.\n                          type: string\n                      type: object\n                    url:\n                      description: The URL of the endpoint to send samples to.\n                      type: string\n                    writeRelabelConfigs:\n                      description: The list of remote write relabel configurations.\n                      items:\n                        description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'\n                        properties:\n                          action:\n                            description: Action to perform based on regex matching. Default is 'replace'\n                            type: string\n                          modulus:\n                            description: Modulus to take of the hash of the source label values.\n                            format: int64\n                            type: integer\n                          regex:\n                            description: Regular expression against which the extracted value is matched. Default is '(.*)'\n                            type: string\n                          replacement:\n                            description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\n                            type: string\n                          separator:\n                            description: Separator placed between concatenated source label values. default is ';'.\n                            type: string\n                          sourceLabels:\n                            description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\n                            items:\n                              type: string\n                            type: array\n                          targetLabel:\n                            description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\n                            type: string\n                        type: object\n                      type: array\n                  required:\n                  - url\n                  type: object\n                type: array\n              replicaExternalLabelName:\n                description: Name of Prometheus external label used to denote replica name. Defaults to the value of `prometheus_replica`. External label will _not_ be added when value is set to empty string (`\"\"`).\n                type: string\n              replicas:\n                description: Number of replicas of each shard to deploy for a Prometheus deployment. Number of replicas multiplied by shards is the total number of Pods created.\n                format: int32\n                type: integer\n              resources:\n                description: Define resources requests and limits for single Pods.\n                properties:\n                  limits:\n                    additionalProperties:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                      x-kubernetes-int-or-string: true\n                    description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                    type: object\n                  requests:\n                    additionalProperties:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                      x-kubernetes-int-or-string: true\n                    description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                    type: object\n                type: object\n              retention:\n                description: Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years).\n                type: string\n              retentionSize:\n                description: 'Maximum amount of disk space used by blocks. Supported units: B, KB, MB, GB, TB, PB, EB. Ex: `512MB`.'\n                type: string\n              routePrefix:\n                description: The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`.\n                type: string\n              ruleNamespaceSelector:\n                description: Namespaces to be selected for PrometheusRules discovery. If unspecified, only the same namespace as the Prometheus object is in is used.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              ruleSelector:\n                description: A selector to select which PrometheusRules to mount for loading alerting/recording rules from. Until (excluding) Prometheus Operator v0.24.0 Prometheus Operator will migrate any legacy rule ConfigMaps to PrometheusRule custom resources selected by RuleSelector. Make sure it does not match any config maps that you do not want to be migrated.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              rules:\n                description: /--rules.*/ command-line arguments.\n                properties:\n                  alert:\n                    description: /--rules.alert.*/ command-line arguments\n                    properties:\n                      forGracePeriod:\n                        description: Minimum duration between alert and restored 'for' state. This is maintained only for alerts with configured 'for' time greater than grace period.\n                        type: string\n                      forOutageTolerance:\n                        description: Max time to tolerate prometheus outage for restoring 'for' state of alert.\n                        type: string\n                      resendDelay:\n                        description: Minimum amount of time to wait before resending an alert to Alertmanager.\n                        type: string\n                    type: object\n                type: object\n              scrapeInterval:\n                description: Interval between consecutive scrapes.\n                type: string\n              scrapeTimeout:\n                description: Number of seconds to wait for target to respond before erroring.\n                type: string\n              secrets:\n                description: Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. The Secrets are mounted into /etc/prometheus/secrets/<secret-name>.\n                items:\n                  type: string\n                type: array\n              securityContext:\n                description: SecurityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext.\n                properties:\n                  fsGroup:\n                    description: \"A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \\n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \\n If unset, the Kubelet will not modify the ownership and permissions of any volume.\"\n                    format: int64\n                    type: integer\n                  fsGroupChangePolicy:\n                    description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are \"OnRootMismatch\" and \"Always\". If not specified defaults to \"Always\".'\n                    type: string\n                  runAsGroup:\n                    description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    format: int64\n                    type: integer\n                  runAsNonRoot:\n                    description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                    type: boolean\n                  runAsUser:\n                    description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    format: int64\n                    type: integer\n                  seLinuxOptions:\n                    description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    properties:\n                      level:\n                        description: Level is SELinux level label that applies to the container.\n                        type: string\n                      role:\n                        description: Role is a SELinux role label that applies to the container.\n                        type: string\n                      type:\n                        description: Type is a SELinux type label that applies to the container.\n                        type: string\n                      user:\n                        description: User is a SELinux user label that applies to the container.\n                        type: string\n                    type: object\n                  supplementalGroups:\n                    description: A list of groups applied to the first process run in each container, in addition to the container's primary GID.  If unspecified, no groups will be added to any container.\n                    items:\n                      format: int64\n                      type: integer\n                    type: array\n                  sysctls:\n                    description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.\n                    items:\n                      description: Sysctl defines a kernel parameter to be set\n                      properties:\n                        name:\n                          description: Name of a property to set\n                          type: string\n                        value:\n                          description: Value of a property to set\n                          type: string\n                      required:\n                      - name\n                      - value\n                      type: object\n                    type: array\n                  windowsOptions:\n                    description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                    properties:\n                      gmsaCredentialSpec:\n                        description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                        type: string\n                      gmsaCredentialSpecName:\n                        description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                        type: string\n                      runAsUserName:\n                        description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                        type: string\n                    type: object\n                type: object\n              serviceAccountName:\n                description: ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods.\n                type: string\n              serviceMonitorNamespaceSelector:\n                description: Namespace's labels to match for ServiceMonitor discovery. If nil, only check own namespace.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              serviceMonitorSelector:\n                description: ServiceMonitors to be selected for target discovery. *Deprecated:* if neither this nor podMonitorSelector are specified, configuration is unmanaged.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              sha:\n                description: 'SHA of Prometheus container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set. Deprecated: use ''image'' instead.  The image digest can be specified as part of the image URL.'\n                type: string\n              shards:\n                description: 'EXPERIMENTAL: Number of shards to distribute targets onto. Number of replicas multiplied by shards is the total number of Pods created. Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. Increasing shards will not reshard data either but it will continue to be available from the same instances. To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. Sharding is done on the content of the `__address__` target meta-label.'\n                format: int32\n                type: integer\n              storage:\n                description: Storage spec to specify how storage shall be used.\n                properties:\n                  disableMountSubPath:\n                    description: 'Deprecated: subPath usage will be disabled by default in a future release, this option will become unnecessary. DisableMountSubPath allows to remove any subPath usage in volume mounts.'\n                    type: boolean\n                  emptyDir:\n                    description: 'EmptyDirVolumeSource to be used by the Prometheus StatefulSets. If specified, used in place of any volumeClaimTemplate. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'\n                    properties:\n                      medium:\n                        description: 'What type of storage medium should back this directory. The default is \"\" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                        type: string\n                      sizeLimit:\n                        anyOf:\n                        - type: integer\n                        - type: string\n                        description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'\n                        pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                        x-kubernetes-int-or-string: true\n                    type: object\n                  volumeClaimTemplate:\n                    description: A PVC spec to be used by the Prometheus StatefulSets.\n                    properties:\n                      apiVersion:\n                        description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n                        type: string\n                      kind:\n                        description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n                        type: string\n                      metadata:\n                        description: EmbeddedMetadata contains metadata relevant to an EmbeddedResource.\n                        properties:\n                          annotations:\n                            additionalProperties:\n                              type: string\n                            description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'\n                            type: object\n                          labels:\n                            additionalProperties:\n                              type: string\n                            description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'\n                            type: object\n                          name:\n                            description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'\n                            type: string\n                        type: object\n                      spec:\n                        description: 'Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                        properties:\n                          accessModes:\n                            description: 'AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'\n                            items:\n                              type: string\n                            type: array\n                          dataSource:\n                            description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - Beta) * An existing PVC (PersistentVolumeClaim) * An existing custom resource/object that implements data population (Alpha) In order to use VolumeSnapshot object types, the appropriate feature gate must be enabled (VolumeSnapshotDataSource or AnyVolumeDataSource) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the specified data source is not supported, the volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.'\n                            properties:\n                              apiGroup:\n                                description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\n                                type: string\n                              kind:\n                                description: Kind is the type of resource being referenced\n                                type: string\n                              name:\n                                description: Name is the name of resource being referenced\n                                type: string\n                            required:\n                            - kind\n                            - name\n                            type: object\n                          resources:\n                            description: 'Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'\n                            properties:\n                              limits:\n                                additionalProperties:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                  x-kubernetes-int-or-string: true\n                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                                type: object\n                              requests:\n                                additionalProperties:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                  x-kubernetes-int-or-string: true\n                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                                type: object\n                            type: object\n                          selector:\n                            description: A label query over volumes to consider for binding.\n                            properties:\n                              matchExpressions:\n                                description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                items:\n                                  description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                  properties:\n                                    key:\n                                      description: key is the label key that the selector applies to.\n                                      type: string\n                                    operator:\n                                      description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                      type: string\n                                    values:\n                                      description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                      items:\n                                        type: string\n                                      type: array\n                                  required:\n                                  - key\n                                  - operator\n                                  type: object\n                                type: array\n                              matchLabels:\n                                additionalProperties:\n                                  type: string\n                                description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                type: object\n                            type: object\n                          storageClassName:\n                            description: 'Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1'\n                            type: string\n                          volumeMode:\n                            description: volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec.\n                            type: string\n                          volumeName:\n                            description: VolumeName is the binding reference to the PersistentVolume backing this claim.\n                            type: string\n                        type: object\n                      status:\n                        description: 'Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                        properties:\n                          accessModes:\n                            description: 'AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'\n                            items:\n                              type: string\n                            type: array\n                          capacity:\n                            additionalProperties:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                              x-kubernetes-int-or-string: true\n                            description: Represents the actual resources of the underlying volume.\n                            type: object\n                          conditions:\n                            description: Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\n                            items:\n                              description: PersistentVolumeClaimCondition contails details about state of pvc\n                              properties:\n                                lastProbeTime:\n                                  description: Last time we probed the condition.\n                                  format: date-time\n                                  type: string\n                                lastTransitionTime:\n                                  description: Last time the condition transitioned from one status to another.\n                                  format: date-time\n                                  type: string\n                                message:\n                                  description: Human-readable message indicating details about last transition.\n                                  type: string\n                                reason:\n                                  description: Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.\n                                  type: string\n                                status:\n                                  type: string\n                                type:\n                                  description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type\n                                  type: string\n                              required:\n                              - status\n                              - type\n                              type: object\n                            type: array\n                          phase:\n                            description: Phase represents the current phase of PersistentVolumeClaim.\n                            type: string\n                        type: object\n                    type: object\n                type: object\n              tag:\n                description: 'Tag of Prometheus container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set. Deprecated: use ''image'' instead.  The image tag can be specified as part of the image URL.'\n                type: string\n              thanos:\n                description: \"Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. \\n This section is experimental, it may change significantly without deprecation notice in any release. \\n This is experimental and may change significantly without backward compatibility in any release.\"\n                properties:\n                  baseImage:\n                    description: 'Thanos base image if other than default. Deprecated: use ''image'' instead'\n                    type: string\n                  grpcServerTlsConfig:\n                    description: 'GRPCServerTLSConfig configures the gRPC server from which Thanos Querier reads recorded rule data. Note: Currently only the CAFile, CertFile, and KeyFile fields are supported. Maps to the ''--grpc-server-tls-*'' CLI args.'\n                    properties:\n                      ca:\n                        description: Struct containing the CA cert to use for the targets.\n                        properties:\n                          configMap:\n                            description: ConfigMap containing data to use for the targets.\n                            properties:\n                              key:\n                                description: The key to select.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          secret:\n                            description: Secret containing data to use for the targets.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                        type: object\n                      caFile:\n                        description: Path to the CA cert in the Prometheus container to use for the targets.\n                        type: string\n                      cert:\n                        description: Struct containing the client cert file for the targets.\n                        properties:\n                          configMap:\n                            description: ConfigMap containing data to use for the targets.\n                            properties:\n                              key:\n                                description: The key to select.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                          secret:\n                            description: Secret containing data to use for the targets.\n                            properties:\n                              key:\n                                description: The key of the secret to select from.  Must be a valid secret key.\n                                type: string\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret or its key must be defined\n                                type: boolean\n                            required:\n                            - key\n                            type: object\n                        type: object\n                      certFile:\n                        description: Path to the client cert file in the Prometheus container for the targets.\n                        type: string\n                      insecureSkipVerify:\n                        description: Disable target certificate validation.\n                        type: boolean\n                      keyFile:\n                        description: Path to the client key file in the Prometheus container for the targets.\n                        type: string\n                      keySecret:\n                        description: Secret containing the client key file for the targets.\n                        properties:\n                          key:\n                            description: The key of the secret to select from.  Must be a valid secret key.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the Secret or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                      serverName:\n                        description: Used to verify the hostname for the targets.\n                        type: string\n                    type: object\n                  image:\n                    description: Image if specified has precedence over baseImage, tag and sha combinations. Specifying the version is still necessary to ensure the Prometheus Operator knows what version of Thanos is being configured.\n                    type: string\n                  listenLocal:\n                    description: ListenLocal makes the Thanos sidecar listen on loopback, so that it does not bind against the Pod IP.\n                    type: boolean\n                  logFormat:\n                    description: LogFormat for Thanos sidecar to be configured with.\n                    type: string\n                  logLevel:\n                    description: LogLevel for Thanos sidecar to be configured with.\n                    type: string\n                  minTime:\n                    description: MinTime for Thanos sidecar to be configured with. Option can be a constant time in RFC3339 format or time duration relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y.\n                    type: string\n                  objectStorageConfig:\n                    description: ObjectStorageConfig configures object storage in Thanos. Alternative to ObjectStorageConfigFile, and lower order priority.\n                    properties:\n                      key:\n                        description: The key of the secret to select from.  Must be a valid secret key.\n                        type: string\n                      name:\n                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                        type: string\n                      optional:\n                        description: Specify whether the Secret or its key must be defined\n                        type: boolean\n                    required:\n                    - key\n                    type: object\n                  objectStorageConfigFile:\n                    description: ObjectStorageConfigFile specifies the path of the object storage configuration file. When used alongside with ObjectStorageConfig, ObjectStorageConfigFile takes precedence.\n                    type: string\n                  resources:\n                    description: Resources defines the resource requirements for the Thanos sidecar. If not provided, no requests/limits will be set\n                    properties:\n                      limits:\n                        additionalProperties:\n                          anyOf:\n                          - type: integer\n                          - type: string\n                          pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                          x-kubernetes-int-or-string: true\n                        description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                        type: object\n                      requests:\n                        additionalProperties:\n                          anyOf:\n                          - type: integer\n                          - type: string\n                          pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                          x-kubernetes-int-or-string: true\n                        description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                        type: object\n                    type: object\n                  sha:\n                    description: 'SHA of Thanos container image to be deployed. Defaults to the value of `version`. Similar to a tag, but the SHA explicitly deploys an immutable container image. Version and Tag are ignored if SHA is set. Deprecated: use ''image'' instead.  The image digest can be specified as part of the image URL.'\n                    type: string\n                  tag:\n                    description: 'Tag of Thanos sidecar container image to be deployed. Defaults to the value of `version`. Version is ignored if Tag is set. Deprecated: use ''image'' instead.  The image tag can be specified as part of the image URL.'\n                    type: string\n                  tracingConfig:\n                    description: TracingConfig configures tracing in Thanos. This is an experimental feature, it may change in any upcoming release in a breaking way.\n                    properties:\n                      key:\n                        description: The key of the secret to select from.  Must be a valid secret key.\n                        type: string\n                      name:\n                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                        type: string\n                      optional:\n                        description: Specify whether the Secret or its key must be defined\n                        type: boolean\n                    required:\n                    - key\n                    type: object\n                  tracingConfigFile:\n                    description: TracingConfig specifies the path of the tracing configuration file. When used alongside with TracingConfig, TracingConfigFile takes precedence.\n                    type: string\n                  version:\n                    description: Version describes the version of Thanos to use.\n                    type: string\n                type: object\n              tolerations:\n                description: If specified, the pod's tolerations.\n                items:\n                  description: The pod this Toleration is attached to tolerates any taint that matches the triple <key,value,effect> using the matching operator <operator>.\n                  properties:\n                    effect:\n                      description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\n                      type: string\n                    key:\n                      description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.\n                      type: string\n                    operator:\n                      description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\n                      type: string\n                    tolerationSeconds:\n                      description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.\n                      format: int64\n                      type: integer\n                    value:\n                      description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.\n                      type: string\n                  type: object\n                type: array\n              topologySpreadConstraints:\n                description: If specified, the pod's topology spread constraints.\n                items:\n                  description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.\n                  properties:\n                    labelSelector:\n                      description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.\n                      properties:\n                        matchExpressions:\n                          description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                          items:\n                            description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                            properties:\n                              key:\n                                description: key is the label key that the selector applies to.\n                                type: string\n                              operator:\n                                description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                type: string\n                              values:\n                                description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                items:\n                                  type: string\n                                type: array\n                            required:\n                            - key\n                            - operator\n                            type: object\n                          type: array\n                        matchLabels:\n                          additionalProperties:\n                            type: string\n                          description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                          type: object\n                      type: object\n                    maxSkew:\n                      description: 'MaxSkew describes the degree to which pods may be unevenly distributed. It''s the maximum permitted difference between the number of matching pods in any two topology domains of a given topology type. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | |   P   |   P   |       | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. It''s a required field. Default value is 1 and 0 is not allowed.'\n                      format: int32\n                      type: integer\n                    topologyKey:\n                      description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each <key, value> as a \"bucket\", and try to put balanced number of pods into each bucket. It's a required field.\n                      type: string\n                    whenUnsatisfiable:\n                      description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it - ScheduleAnyway tells the scheduler to still schedule it It''s considered as \"Unsatisfiable\" if and only if placing incoming pod on any topology violates \"MaxSkew\". For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.'\n                      type: string\n                  required:\n                  - maxSkew\n                  - topologyKey\n                  - whenUnsatisfiable\n                  type: object\n                type: array\n              version:\n                description: Version of Prometheus to be deployed.\n                type: string\n              volumeMounts:\n                description: VolumeMounts allows configuration of additional VolumeMounts on the output StatefulSet definition. VolumeMounts specified will be appended to other VolumeMounts in the prometheus container, that are generated as a result of StorageSpec objects.\n                items:\n                  description: VolumeMount describes a mounting of a Volume within a container.\n                  properties:\n                    mountPath:\n                      description: Path within the container at which the volume should be mounted.  Must not contain ':'.\n                      type: string\n                    mountPropagation:\n                      description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n                      type: string\n                    name:\n                      description: This must match the Name of a Volume.\n                      type: string\n                    readOnly:\n                      description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\n                      type: boolean\n                    subPath:\n                      description: Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).\n                      type: string\n                    subPathExpr:\n                      description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\n                      type: string\n                  required:\n                  - mountPath\n                  - name\n                  type: object\n                type: array\n              volumes:\n                description: Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects.\n                items:\n                  description: Volume represents a named volume in a pod that may be accessed by any container in the pod.\n                  properties:\n                    awsElasticBlockStore:\n                      description: 'AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        partition:\n                          description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty).'\n                          format: int32\n                          type: integer\n                        readOnly:\n                          description: 'Specify \"true\" to force and set the ReadOnly property in VolumeMounts to \"true\". If omitted, the default is \"false\". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                          type: boolean\n                        volumeID:\n                          description: 'Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    azureDisk:\n                      description: AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.\n                      properties:\n                        cachingMode:\n                          description: 'Host Caching mode: None, Read Only, Read Write.'\n                          type: string\n                        diskName:\n                          description: The Name of the data disk in the blob storage\n                          type: string\n                        diskURI:\n                          description: The URI the data disk in the blob storage\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        kind:\n                          description: 'Expected values Shared: multiple blob disks per storage account  Dedicated: single blob disk per storage account  Managed: azure managed data disk (only in managed availability set). defaults to shared'\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                      required:\n                      - diskName\n                      - diskURI\n                      type: object\n                    azureFile:\n                      description: AzureFile represents an Azure File Service mount on the host and bind mount to the pod.\n                      properties:\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretName:\n                          description: the name of secret that contains Azure Storage Account Name and Key\n                          type: string\n                        shareName:\n                          description: Share Name\n                          type: string\n                      required:\n                      - secretName\n                      - shareName\n                      type: object\n                    cephfs:\n                      description: CephFS represents a Ceph FS mount on the host that shares a pod's lifetime\n                      properties:\n                        monitors:\n                          description: 'Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          items:\n                            type: string\n                          type: array\n                        path:\n                          description: 'Optional: Used as the mounted root, rather than the full Ceph tree, default is /'\n                          type: string\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: boolean\n                        secretFile:\n                          description: 'Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: string\n                        secretRef:\n                          description: 'Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        user:\n                          description: 'Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: string\n                      required:\n                      - monitors\n                      type: object\n                    cinder:\n                      description: 'Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: string\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: boolean\n                        secretRef:\n                          description: 'Optional: points to a secret object containing parameters used to connect to OpenStack.'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        volumeID:\n                          description: 'volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    configMap:\n                      description: ConfigMap represents a configMap that should populate this volume\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                          items:\n                            description: Maps a string key to a path within a volume.\n                            properties:\n                              key:\n                                description: The key to project.\n                                type: string\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                type: string\n                            required:\n                            - key\n                            - path\n                            type: object\n                          type: array\n                        name:\n                          description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                          type: string\n                        optional:\n                          description: Specify whether the ConfigMap or its keys must be defined\n                          type: boolean\n                      type: object\n                    csi:\n                      description: CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature).\n                      properties:\n                        driver:\n                          description: Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster.\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Ex. \"ext4\", \"xfs\", \"ntfs\". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply.\n                          type: string\n                        nodePublishSecretRef:\n                          description: NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and  may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        readOnly:\n                          description: Specifies a read-only configuration for the volume. Defaults to false (read/write).\n                          type: boolean\n                        volumeAttributes:\n                          additionalProperties:\n                            type: string\n                          description: VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values.\n                          type: object\n                      required:\n                      - driver\n                      type: object\n                    downwardAPI:\n                      description: DownwardAPI represents downward API about the pod that should populate this volume\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: Items is a list of downward API volume file\n                          items:\n                            description: DownwardAPIVolumeFile represents information to create the file containing the pod field\n                            properties:\n                              fieldRef:\n                                description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: 'Required: Path is  the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..'''\n                                type: string\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                            required:\n                            - path\n                            type: object\n                          type: array\n                      type: object\n                    emptyDir:\n                      description: 'EmptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                      properties:\n                        medium:\n                          description: 'What type of storage medium should back this directory. The default is \"\" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                          type: string\n                        sizeLimit:\n                          anyOf:\n                          - type: integer\n                          - type: string\n                          description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'\n                          pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                          x-kubernetes-int-or-string: true\n                      type: object\n                    fc:\n                      description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod.\n                      properties:\n                        fsType:\n                          description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        lun:\n                          description: 'Optional: FC target lun number'\n                          format: int32\n                          type: integer\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.'\n                          type: boolean\n                        targetWWNs:\n                          description: 'Optional: FC target worldwide names (WWNs)'\n                          items:\n                            type: string\n                          type: array\n                        wwids:\n                          description: 'Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.'\n                          items:\n                            type: string\n                          type: array\n                      type: object\n                    flexVolume:\n                      description: FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin.\n                      properties:\n                        driver:\n                          description: Driver is the name of the driver to use for this volume.\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default filesystem depends on FlexVolume script.\n                          type: string\n                        options:\n                          additionalProperties:\n                            type: string\n                          description: 'Optional: Extra command options if any.'\n                          type: object\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.'\n                          type: boolean\n                        secretRef:\n                          description: 'Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                      required:\n                      - driver\n                      type: object\n                    flocker:\n                      description: Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running\n                      properties:\n                        datasetName:\n                          description: Name of the dataset stored as metadata -> name on the dataset for Flocker should be considered as deprecated\n                          type: string\n                        datasetUUID:\n                          description: UUID of the dataset. This is unique identifier of a Flocker dataset\n                          type: string\n                      type: object\n                    gcePersistentDisk:\n                      description: 'GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        partition:\n                          description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          format: int32\n                          type: integer\n                        pdName:\n                          description: 'Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          type: boolean\n                      required:\n                      - pdName\n                      type: object\n                    gitRepo:\n                      description: 'GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod''s container.'\n                      properties:\n                        directory:\n                          description: Target directory name. Must not contain or start with '..'.  If '.' is supplied, the volume directory will be the git repository.  Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name.\n                          type: string\n                        repository:\n                          description: Repository URL\n                          type: string\n                        revision:\n                          description: Commit hash for the specified revision.\n                          type: string\n                      required:\n                      - repository\n                      type: object\n                    glusterfs:\n                      description: 'Glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md'\n                      properties:\n                        endpoints:\n                          description: 'EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: string\n                        path:\n                          description: 'Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: boolean\n                      required:\n                      - endpoints\n                      - path\n                      type: object\n                    hostPath:\n                      description: 'HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.'\n                      properties:\n                        path:\n                          description: 'Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath'\n                          type: string\n                        type:\n                          description: 'Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath'\n                          type: string\n                      required:\n                      - path\n                      type: object\n                    iscsi:\n                      description: 'ISCSI represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md'\n                      properties:\n                        chapAuthDiscovery:\n                          description: whether support iSCSI Discovery CHAP authentication\n                          type: boolean\n                        chapAuthSession:\n                          description: whether support iSCSI Session CHAP authentication\n                          type: boolean\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        initiatorName:\n                          description: Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface <target portal>:<volume name> will be created for the connection.\n                          type: string\n                        iqn:\n                          description: Target iSCSI Qualified Name.\n                          type: string\n                        iscsiInterface:\n                          description: iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp).\n                          type: string\n                        lun:\n                          description: iSCSI Target Lun number.\n                          format: int32\n                          type: integer\n                        portals:\n                          description: iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).\n                          items:\n                            type: string\n                          type: array\n                        readOnly:\n                          description: ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false.\n                          type: boolean\n                        secretRef:\n                          description: CHAP Secret for iSCSI target and initiator authentication\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        targetPortal:\n                          description: iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).\n                          type: string\n                      required:\n                      - iqn\n                      - lun\n                      - targetPortal\n                      type: object\n                    name:\n                      description: 'Volume''s name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'\n                      type: string\n                    nfs:\n                      description: 'NFS represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                      properties:\n                        path:\n                          description: 'Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: boolean\n                        server:\n                          description: 'Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: string\n                      required:\n                      - path\n                      - server\n                      type: object\n                    persistentVolumeClaim:\n                      description: 'PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                      properties:\n                        claimName:\n                          description: 'ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                          type: string\n                        readOnly:\n                          description: Will force the ReadOnly setting in VolumeMounts. Default false.\n                          type: boolean\n                      required:\n                      - claimName\n                      type: object\n                    photonPersistentDisk:\n                      description: PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        pdID:\n                          description: ID that identifies Photon Controller persistent disk\n                          type: string\n                      required:\n                      - pdID\n                      type: object\n                    portworxVolume:\n                      description: PortworxVolume represents a portworx volume attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        volumeID:\n                          description: VolumeID uniquely identifies a Portworx volume\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    projected:\n                      description: Items for all in one resources secrets, configmaps, and downward API\n                      properties:\n                        defaultMode:\n                          description: Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\n                          format: int32\n                          type: integer\n                        sources:\n                          description: list of volume projections\n                          items:\n                            description: Projection that may be projected along with other supported volume types\n                            properties:\n                              configMap:\n                                description: information about the configMap data to project\n                                properties:\n                                  items:\n                                    description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                                    items:\n                                      description: Maps a string key to a path within a volume.\n                                      properties:\n                                        key:\n                                          description: The key to project.\n                                          type: string\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                          type: string\n                                      required:\n                                      - key\n                                      - path\n                                      type: object\n                                    type: array\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its keys must be defined\n                                    type: boolean\n                                type: object\n                              downwardAPI:\n                                description: information about the downwardAPI data to project\n                                properties:\n                                  items:\n                                    description: Items is a list of DownwardAPIVolume file\n                                    items:\n                                      description: DownwardAPIVolumeFile represents information to create the file containing the pod field\n                                      properties:\n                                        fieldRef:\n                                          description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.'\n                                          properties:\n                                            apiVersion:\n                                              description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                              type: string\n                                            fieldPath:\n                                              description: Path of the field to select in the specified API version.\n                                              type: string\n                                          required:\n                                          - fieldPath\n                                          type: object\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: 'Required: Path is  the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..'''\n                                          type: string\n                                        resourceFieldRef:\n                                          description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.'\n                                          properties:\n                                            containerName:\n                                              description: 'Container name: required for volumes, optional for env vars'\n                                              type: string\n                                            divisor:\n                                              anyOf:\n                                              - type: integer\n                                              - type: string\n                                              description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                              pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                              x-kubernetes-int-or-string: true\n                                            resource:\n                                              description: 'Required: resource to select'\n                                              type: string\n                                          required:\n                                          - resource\n                                          type: object\n                                      required:\n                                      - path\n                                      type: object\n                                    type: array\n                                type: object\n                              secret:\n                                description: information about the secret data to project\n                                properties:\n                                  items:\n                                    description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                                    items:\n                                      description: Maps a string key to a path within a volume.\n                                      properties:\n                                        key:\n                                          description: The key to project.\n                                          type: string\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                          type: string\n                                      required:\n                                      - key\n                                      - path\n                                      type: object\n                                    type: array\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                type: object\n                              serviceAccountToken:\n                                description: information about the serviceAccountToken data to project\n                                properties:\n                                  audience:\n                                    description: Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver.\n                                    type: string\n                                  expirationSeconds:\n                                    description: ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes.\n                                    format: int64\n                                    type: integer\n                                  path:\n                                    description: Path is the path relative to the mount point of the file to project the token into.\n                                    type: string\n                                required:\n                                - path\n                                type: object\n                            type: object\n                          type: array\n                      required:\n                      - sources\n                      type: object\n                    quobyte:\n                      description: Quobyte represents a Quobyte mount on the host that shares a pod's lifetime\n                      properties:\n                        group:\n                          description: Group to map volume access to Default is no group\n                          type: string\n                        readOnly:\n                          description: ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false.\n                          type: boolean\n                        registry:\n                          description: Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes\n                          type: string\n                        tenant:\n                          description: Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin\n                          type: string\n                        user:\n                          description: User to map volume access to Defaults to serivceaccount user\n                          type: string\n                        volume:\n                          description: Volume is a string that references an already created Quobyte volume by name.\n                          type: string\n                      required:\n                      - registry\n                      - volume\n                      type: object\n                    rbd:\n                      description: 'RBD represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        image:\n                          description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        keyring:\n                          description: 'Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        monitors:\n                          description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          items:\n                            type: string\n                          type: array\n                        pool:\n                          description: 'The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: boolean\n                        secretRef:\n                          description: 'SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        user:\n                          description: 'The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                      required:\n                      - image\n                      - monitors\n                      type: object\n                    scaleIO:\n                      description: ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Default is \"xfs\".\n                          type: string\n                        gateway:\n                          description: The host address of the ScaleIO API Gateway.\n                          type: string\n                        protectionDomain:\n                          description: The name of the ScaleIO Protection Domain for the configured storage.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretRef:\n                          description: SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        sslEnabled:\n                          description: Flag to enable/disable SSL communication with Gateway, default false\n                          type: boolean\n                        storageMode:\n                          description: Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned.\n                          type: string\n                        storagePool:\n                          description: The ScaleIO Storage Pool associated with the protection domain.\n                          type: string\n                        system:\n                          description: The name of the storage system as configured in ScaleIO.\n                          type: string\n                        volumeName:\n                          description: The name of a volume already created in the ScaleIO system that is associated with this volume source.\n                          type: string\n                      required:\n                      - gateway\n                      - secretRef\n                      - system\n                      type: object\n                    secret:\n                      description: 'Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret'\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                          items:\n                            description: Maps a string key to a path within a volume.\n                            properties:\n                              key:\n                                description: The key to project.\n                                type: string\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                type: string\n                            required:\n                            - key\n                            - path\n                            type: object\n                          type: array\n                        optional:\n                          description: Specify whether the Secret or its keys must be defined\n                          type: boolean\n                        secretName:\n                          description: 'Name of the secret in the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret'\n                          type: string\n                      type: object\n                    storageos:\n                      description: StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes.\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretRef:\n                          description: SecretRef specifies the secret to use for obtaining the StorageOS API credentials.  If not specified, default values will be attempted.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        volumeName:\n                          description: VolumeName is the human-readable name of the StorageOS volume.  Volume names are only unique within a namespace.\n                          type: string\n                        volumeNamespace:\n                          description: VolumeNamespace specifies the scope of the volume within StorageOS.  If no namespace is specified then the Pod's namespace will be used.  This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to \"default\" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created.\n                          type: string\n                      type: object\n                    vsphereVolume:\n                      description: VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        storagePolicyID:\n                          description: Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName.\n                          type: string\n                        storagePolicyName:\n                          description: Storage Policy Based Management (SPBM) profile name.\n                          type: string\n                        volumePath:\n                          description: Path that identifies vSphere volume vmdk\n                          type: string\n                      required:\n                      - volumePath\n                      type: object\n                  required:\n                  - name\n                  type: object\n                type: array\n              walCompression:\n                description: Enable compression of the write-ahead log using Snappy. This flag is only available in versions of Prometheus >= 2.11.0.\n                type: boolean\n              web:\n                description: WebSpec defines the web command line flags when starting Prometheus.\n                properties:\n                  pageTitle:\n                    description: The prometheus web page title\n                    type: string\n                type: object\n            type: object\n          status:\n            description: 'Most recent observed status of the Prometheus cluster. Read-only. Not included when requesting from the apiserver, only from the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'\n            properties:\n              availableReplicas:\n                description: Total number of available pods (ready for at least minReadySeconds) targeted by this Prometheus deployment.\n                format: int32\n                type: integer\n              paused:\n                description: Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.\n                type: boolean\n              replicas:\n                description: Total number of non-terminated pods targeted by this Prometheus deployment (their labels match the selector).\n                format: int32\n                type: integer\n              unavailableReplicas:\n                description: Total number of unavailable pods targeted by this Prometheus deployment.\n                format: int32\n                type: integer\n              updatedReplicas:\n                description: Total number of non-terminated pods targeted by this Prometheus deployment that have the desired version spec.\n                format: int32\n                type: integer\n            required:\n            - availableReplicas\n            - paused\n            - replicas\n            - unavailableReplicas\n            - updatedReplicas\n            type: object\n        required:\n        - spec\n        type: object\n    served: true\n    storage: true\n    subresources: {}\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n---\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: prometheusrules.monitoring.coreos.com\nspec:\n  group: monitoring.coreos.com\n  names:\n    kind: PrometheusRule\n    listKind: PrometheusRuleList\n    plural: prometheusrules\n    singular: prometheusrule\n  scope: Namespaced\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n        description: PrometheusRule defines recording and alerting rules for a Prometheus instance\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: Specification of desired alerting rule definitions for Prometheus.\n            properties:\n              groups:\n                description: Content of Prometheus rule file\n                items:\n                  description: 'RuleGroup is a list of sequentially evaluated recording and alerting rules. Note: PartialResponseStrategy is only used by ThanosRuler and will be ignored by Prometheus instances.  Valid values for this field are ''warn'' or ''abort''.  More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response'\n                  properties:\n                    interval:\n                      type: string\n                    name:\n                      type: string\n                    partial_response_strategy:\n                      type: string\n                    rules:\n                      items:\n                        description: Rule describes an alerting or recording rule.\n                        properties:\n                          alert:\n                            type: string\n                          annotations:\n                            additionalProperties:\n                              type: string\n                            type: object\n                          expr:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            x-kubernetes-int-or-string: true\n                          for:\n                            type: string\n                          labels:\n                            additionalProperties:\n                              type: string\n                            type: object\n                          record:\n                            type: string\n                        required:\n                        - expr\n                        type: object\n                      type: array\n                  required:\n                  - name\n                  - rules\n                  type: object\n                type: array\n            type: object\n        required:\n        - spec\n        type: object\n    served: true\n    storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n---\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: servicemonitors.monitoring.coreos.com\nspec:\n  group: monitoring.coreos.com\n  names:\n    categories:\n    - prometheus-operator\n    kind: ServiceMonitor\n    listKind: ServiceMonitorList\n    plural: servicemonitors\n    singular: servicemonitor\n  scope: Namespaced\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n        description: ServiceMonitor defines monitoring for a set of services.\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: Specification of desired Service selection for target discovery by Prometheus.\n            properties:\n              endpoints:\n                description: A list of endpoints allowed as part of this ServiceMonitor.\n                items:\n                  description: Endpoint defines a scrapeable endpoint serving Prometheus metrics.\n                  properties:\n                    basicAuth:\n                      description: 'BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints'\n                      properties:\n                        password:\n                          description: The secret in the service monitor namespace that contains the password for authentication.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                        username:\n                          description: The secret in the service monitor namespace that contains the username for authentication.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                      type: object\n                    bearerTokenFile:\n                      description: File to read bearer token for scraping targets.\n                      type: string\n                    bearerTokenSecret:\n                      description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the service monitor and accessible by the Prometheus Operator.\n                      properties:\n                        key:\n                          description: The key of the secret to select from.  Must be a valid secret key.\n                          type: string\n                        name:\n                          description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                          type: string\n                        optional:\n                          description: Specify whether the Secret or its key must be defined\n                          type: boolean\n                      required:\n                      - key\n                      type: object\n                    honorLabels:\n                      description: HonorLabels chooses the metric's labels on collisions with target labels.\n                      type: boolean\n                    honorTimestamps:\n                      description: HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data.\n                      type: boolean\n                    interval:\n                      description: Interval at which metrics should be scraped\n                      type: string\n                    metricRelabelings:\n                      description: MetricRelabelConfigs to apply to samples before ingestion.\n                      items:\n                        description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'\n                        properties:\n                          action:\n                            description: Action to perform based on regex matching. Default is 'replace'\n                            type: string\n                          modulus:\n                            description: Modulus to take of the hash of the source label values.\n                            format: int64\n                            type: integer\n                          regex:\n                            description: Regular expression against which the extracted value is matched. Default is '(.*)'\n                            type: string\n                          replacement:\n                            description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\n                            type: string\n                          separator:\n                            description: Separator placed between concatenated source label values. default is ';'.\n                            type: string\n                          sourceLabels:\n                            description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\n                            items:\n                              type: string\n                            type: array\n                          targetLabel:\n                            description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\n                            type: string\n                        type: object\n                      type: array\n                    params:\n                      additionalProperties:\n                        items:\n                          type: string\n                        type: array\n                      description: Optional HTTP URL parameters\n                      type: object\n                    path:\n                      description: HTTP path to scrape for metrics.\n                      type: string\n                    port:\n                      description: Name of the service port this endpoint refers to. Mutually exclusive with targetPort.\n                      type: string\n                    proxyUrl:\n                      description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint.\n                      type: string\n                    relabelings:\n                      description: 'RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'\n                      items:\n                        description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'\n                        properties:\n                          action:\n                            description: Action to perform based on regex matching. Default is 'replace'\n                            type: string\n                          modulus:\n                            description: Modulus to take of the hash of the source label values.\n                            format: int64\n                            type: integer\n                          regex:\n                            description: Regular expression against which the extracted value is matched. Default is '(.*)'\n                            type: string\n                          replacement:\n                            description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'\n                            type: string\n                          separator:\n                            description: Separator placed between concatenated source label values. default is ';'.\n                            type: string\n                          sourceLabels:\n                            description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.\n                            items:\n                              type: string\n                            type: array\n                          targetLabel:\n                            description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.\n                            type: string\n                        type: object\n                      type: array\n                    scheme:\n                      description: HTTP scheme to use for scraping.\n                      type: string\n                    scrapeTimeout:\n                      description: Timeout after which the scrape is ended\n                      type: string\n                    targetPort:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      description: Name or number of the target port of the Pod behind the Service, the port must be specified with container port property. Mutually exclusive with port.\n                      x-kubernetes-int-or-string: true\n                    tlsConfig:\n                      description: TLS configuration to use when scraping the endpoint\n                      properties:\n                        ca:\n                          description: Struct containing the CA cert to use for the targets.\n                          properties:\n                            configMap:\n                              description: ConfigMap containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key to select.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the ConfigMap or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            secret:\n                              description: Secret containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                          type: object\n                        caFile:\n                          description: Path to the CA cert in the Prometheus container to use for the targets.\n                          type: string\n                        cert:\n                          description: Struct containing the client cert file for the targets.\n                          properties:\n                            configMap:\n                              description: ConfigMap containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key to select.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the ConfigMap or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                            secret:\n                              description: Secret containing data to use for the targets.\n                              properties:\n                                key:\n                                  description: The key of the secret to select from.  Must be a valid secret key.\n                                  type: string\n                                name:\n                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                  type: string\n                                optional:\n                                  description: Specify whether the Secret or its key must be defined\n                                  type: boolean\n                              required:\n                              - key\n                              type: object\n                          type: object\n                        certFile:\n                          description: Path to the client cert file in the Prometheus container for the targets.\n                          type: string\n                        insecureSkipVerify:\n                          description: Disable target certificate validation.\n                          type: boolean\n                        keyFile:\n                          description: Path to the client key file in the Prometheus container for the targets.\n                          type: string\n                        keySecret:\n                          description: Secret containing the client key file for the targets.\n                          properties:\n                            key:\n                              description: The key of the secret to select from.  Must be a valid secret key.\n                              type: string\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                            optional:\n                              description: Specify whether the Secret or its key must be defined\n                              type: boolean\n                          required:\n                          - key\n                          type: object\n                        serverName:\n                          description: Used to verify the hostname for the targets.\n                          type: string\n                      type: object\n                  type: object\n                type: array\n              jobLabel:\n                description: The label to use to retrieve the job name from.\n                type: string\n              namespaceSelector:\n                description: Selector to select which namespaces the Endpoints objects are discovered from.\n                properties:\n                  any:\n                    description: Boolean describing whether all namespaces are selected in contrast to a list restricting them.\n                    type: boolean\n                  matchNames:\n                    description: List of namespace names.\n                    items:\n                      type: string\n                    type: array\n                type: object\n              podTargetLabels:\n                description: PodTargetLabels transfers labels on the Kubernetes Pod onto the target.\n                items:\n                  type: string\n                type: array\n              sampleLimit:\n                description: SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.\n                format: int64\n                type: integer\n              selector:\n                description: Selector to select Endpoints objects.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              targetLabels:\n                description: TargetLabels transfers labels on the Kubernetes Service onto the target.\n                items:\n                  type: string\n                type: array\n              targetLimit:\n                description: TargetLimit defines a limit on the number of scraped targets that will be accepted.\n                format: int64\n                type: integer\n            required:\n            - endpoints\n            - selector\n            type: object\n        required:\n        - spec\n        type: object\n    served: true\n    storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n---\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: thanosrulers.monitoring.coreos.com\nspec:\n  group: monitoring.coreos.com\n  names:\n    categories:\n    - prometheus-operator\n    kind: ThanosRuler\n    listKind: ThanosRulerList\n    plural: thanosrulers\n    singular: thanosruler\n  scope: Namespaced\n  versions:\n  - name: v1\n    schema:\n      openAPIV3Schema:\n        description: ThanosRuler defines a ThanosRuler deployment.\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: 'Specification of the desired behavior of the ThanosRuler cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'\n            properties:\n              affinity:\n                description: If specified, the pod's scheduling constraints.\n                properties:\n                  nodeAffinity:\n                    description: Describes node affinity scheduling rules for the pod.\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\n                          properties:\n                            preference:\n                              description: A node selector term, associated with the corresponding weight.\n                              properties:\n                                matchExpressions:\n                                  description: A list of node selector requirements by node's labels.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchFields:\n                                  description: A list of node selector requirements by node's fields.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                              type: object\n                            weight:\n                              description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - preference\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.\n                        properties:\n                          nodeSelectorTerms:\n                            description: Required. A list of node selector terms. The terms are ORed.\n                            items:\n                              description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\n                              properties:\n                                matchExpressions:\n                                  description: A list of node selector requirements by node's labels.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchFields:\n                                  description: A list of node selector requirements by node's fields.\n                                  items:\n                                    description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: The label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n                                        type: string\n                                      values:\n                                        description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                              type: object\n                            type: array\n                        required:\n                        - nodeSelectorTerms\n                        type: object\n                    type: object\n                  podAffinity:\n                    description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n                          properties:\n                            podAffinityTerm:\n                              description: Required. A pod affinity term, associated with the corresponding weight.\n                              properties:\n                                labelSelector:\n                                  description: A label query over a set of resources, in this case pods.\n                                  properties:\n                                    matchExpressions:\n                                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                      items:\n                                        description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                        properties:\n                                          key:\n                                            description: key is the label key that the selector applies to.\n                                            type: string\n                                          operator:\n                                            description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                            type: string\n                                          values:\n                                            description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                            items:\n                                              type: string\n                                            type: array\n                                        required:\n                                        - key\n                                        - operator\n                                        type: object\n                                      type: array\n                                    matchLabels:\n                                      additionalProperties:\n                                        type: string\n                                      description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                      type: object\n                                  type: object\n                                namespaces:\n                                  description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                                  items:\n                                    type: string\n                                  type: array\n                                topologyKey:\n                                  description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                                  type: string\n                              required:\n                              - topologyKey\n                              type: object\n                            weight:\n                              description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - podAffinityTerm\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\n                        items:\n                          description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key <topologyKey> matches that of any node on which a pod of the set of pods is running\n                          properties:\n                            labelSelector:\n                              description: A label query over a set of resources, in this case pods.\n                              properties:\n                                matchExpressions:\n                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                  items:\n                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: key is the label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                        type: string\n                                      values:\n                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchLabels:\n                                  additionalProperties:\n                                    type: string\n                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                  type: object\n                              type: object\n                            namespaces:\n                              description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                              items:\n                                type: string\n                              type: array\n                            topologyKey:\n                              description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                              type: string\n                          required:\n                          - topologyKey\n                          type: object\n                        type: array\n                    type: object\n                  podAntiAffinity:\n                    description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).\n                    properties:\n                      preferredDuringSchedulingIgnoredDuringExecution:\n                        description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.\n                        items:\n                          description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)\n                          properties:\n                            podAffinityTerm:\n                              description: Required. A pod affinity term, associated with the corresponding weight.\n                              properties:\n                                labelSelector:\n                                  description: A label query over a set of resources, in this case pods.\n                                  properties:\n                                    matchExpressions:\n                                      description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                      items:\n                                        description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                        properties:\n                                          key:\n                                            description: key is the label key that the selector applies to.\n                                            type: string\n                                          operator:\n                                            description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                            type: string\n                                          values:\n                                            description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                            items:\n                                              type: string\n                                            type: array\n                                        required:\n                                        - key\n                                        - operator\n                                        type: object\n                                      type: array\n                                    matchLabels:\n                                      additionalProperties:\n                                        type: string\n                                      description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                      type: object\n                                  type: object\n                                namespaces:\n                                  description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                                  items:\n                                    type: string\n                                  type: array\n                                topologyKey:\n                                  description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                                  type: string\n                              required:\n                              - topologyKey\n                              type: object\n                            weight:\n                              description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.\n                              format: int32\n                              type: integer\n                          required:\n                          - podAffinityTerm\n                          - weight\n                          type: object\n                        type: array\n                      requiredDuringSchedulingIgnoredDuringExecution:\n                        description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.\n                        items:\n                          description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key <topologyKey> matches that of any node on which a pod of the set of pods is running\n                          properties:\n                            labelSelector:\n                              description: A label query over a set of resources, in this case pods.\n                              properties:\n                                matchExpressions:\n                                  description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                  items:\n                                    description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                    properties:\n                                      key:\n                                        description: key is the label key that the selector applies to.\n                                        type: string\n                                      operator:\n                                        description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                        type: string\n                                      values:\n                                        description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                        items:\n                                          type: string\n                                        type: array\n                                    required:\n                                    - key\n                                    - operator\n                                    type: object\n                                  type: array\n                                matchLabels:\n                                  additionalProperties:\n                                    type: string\n                                  description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                  type: object\n                              type: object\n                            namespaces:\n                              description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means \"this pod's namespace\"\n                              items:\n                                type: string\n                              type: array\n                            topologyKey:\n                              description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.\n                              type: string\n                          required:\n                          - topologyKey\n                          type: object\n                        type: array\n                    type: object\n                type: object\n              alertDropLabels:\n                description: AlertDropLabels configure the label names which should be dropped in ThanosRuler alerts. If `labels` field is not provided, `thanos_ruler_replica` will be dropped in alerts by default.\n                items:\n                  type: string\n                type: array\n              alertQueryUrl:\n                description: The external Query URL the Thanos Ruler will set in the 'Source' field of all alerts. Maps to the '--alert.query-url' CLI arg.\n                type: string\n              alertmanagersConfig:\n                description: Define configuration for connecting to alertmanager.  Only available with thanos v0.10.0 and higher.  Maps to the `alertmanagers.config` arg.\n                properties:\n                  key:\n                    description: The key of the secret to select from.  Must be a valid secret key.\n                    type: string\n                  name:\n                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                    type: string\n                  optional:\n                    description: Specify whether the Secret or its key must be defined\n                    type: boolean\n                required:\n                - key\n                type: object\n              alertmanagersUrl:\n                description: 'Define URLs to send alerts to Alertmanager.  For Thanos v0.10.0 and higher, AlertManagersConfig should be used instead.  Note: this field will be ignored if AlertManagersConfig is specified. Maps to the `alertmanagers.url` arg.'\n                items:\n                  type: string\n                type: array\n              containers:\n                description: 'Containers allows injecting additional containers or modifying operator generated containers. This can be used to allow adding an authentication proxy to a ThanosRuler pod or to change the behavior of an operator generated container. Containers described here modify an operator generated container if they share the same name and modifications are done via a strategic merge patch. The current container names are: `thanos-ruler` and `config-reloader`. Overriding containers is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.'\n                items:\n                  description: A single application container that you want to run within a pod.\n                  properties:\n                    args:\n                      description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    command:\n                      description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    env:\n                      description: List of environment variables to set in the container. Cannot be updated.\n                      items:\n                        description: EnvVar represents an environment variable present in a Container.\n                        properties:\n                          name:\n                            description: Name of the environment variable. Must be a C_IDENTIFIER.\n                            type: string\n                          value:\n                            description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".'\n                            type: string\n                          valueFrom:\n                            description: Source for the environment variable's value. Cannot be used if value is not empty.\n                            properties:\n                              configMapKeyRef:\n                                description: Selects a key of a ConfigMap.\n                                properties:\n                                  key:\n                                    description: The key to select.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              fieldRef:\n                                description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                              secretKeyRef:\n                                description: Selects a key of a secret in the pod's namespace\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                            type: object\n                        required:\n                        - name\n                        type: object\n                      type: array\n                    envFrom:\n                      description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\n                      items:\n                        description: EnvFromSource represents the source of a set of ConfigMaps\n                        properties:\n                          configMapRef:\n                            description: The ConfigMap to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap must be defined\n                                type: boolean\n                            type: object\n                          prefix:\n                            description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\n                            type: string\n                          secretRef:\n                            description: The Secret to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret must be defined\n                                type: boolean\n                            type: object\n                        type: object\n                      type: array\n                    image:\n                      description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.'\n                      type: string\n                    imagePullPolicy:\n                      description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'\n                      type: string\n                    lifecycle:\n                      description: Actions that the management system should take in response to container lifecycle events. Cannot be updated.\n                      properties:\n                        postStart:\n                          description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                        preStop:\n                          description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                      type: object\n                    livenessProbe:\n                      description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    name:\n                      description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\n                      type: string\n                    ports:\n                      description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.\n                      items:\n                        description: ContainerPort represents a network port in a single container.\n                        properties:\n                          containerPort:\n                            description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.\n                            format: int32\n                            type: integer\n                          hostIP:\n                            description: What host IP to bind the external port to.\n                            type: string\n                          hostPort:\n                            description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\n                            format: int32\n                            type: integer\n                          name:\n                            description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\n                            type: string\n                          protocol:\n                            default: TCP\n                            description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".\n                            type: string\n                        required:\n                        - containerPort\n                        type: object\n                      type: array\n                      x-kubernetes-list-map-keys:\n                      - containerPort\n                      - protocol\n                      x-kubernetes-list-type: map\n                    readinessProbe:\n                      description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    resources:\n                      description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                      properties:\n                        limits:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                        requests:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                      type: object\n                    securityContext:\n                      description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/'\n                      properties:\n                        allowPrivilegeEscalation:\n                          description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN'\n                          type: boolean\n                        capabilities:\n                          description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.\n                          properties:\n                            add:\n                              description: Added capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                            drop:\n                              description: Removed capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                          type: object\n                        privileged:\n                          description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\n                          type: boolean\n                        procMount:\n                          description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\n                          type: string\n                        readOnlyRootFilesystem:\n                          description: Whether this container has a read-only root filesystem. Default is false.\n                          type: boolean\n                        runAsGroup:\n                          description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        runAsNonRoot:\n                          description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          type: boolean\n                        runAsUser:\n                          description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        seLinuxOptions:\n                          description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            level:\n                              description: Level is SELinux level label that applies to the container.\n                              type: string\n                            role:\n                              description: Role is a SELinux role label that applies to the container.\n                              type: string\n                            type:\n                              description: Type is a SELinux type label that applies to the container.\n                              type: string\n                            user:\n                              description: User is a SELinux user label that applies to the container.\n                              type: string\n                          type: object\n                        windowsOptions:\n                          description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            gmsaCredentialSpec:\n                              description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                              type: string\n                            gmsaCredentialSpecName:\n                              description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                              type: string\n                            runAsUserName:\n                              description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                              type: string\n                          type: object\n                      type: object\n                    startupProbe:\n                      description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    stdin:\n                      description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\n                      type: boolean\n                    stdinOnce:\n                      description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\n                      type: boolean\n                    terminationMessagePath:\n                      description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.'\n                      type: string\n                    terminationMessagePolicy:\n                      description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\n                      type: string\n                    tty:\n                      description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\n                      type: boolean\n                    volumeDevices:\n                      description: volumeDevices is the list of block devices to be used by the container.\n                      items:\n                        description: volumeDevice describes a mapping of a raw block device within a container.\n                        properties:\n                          devicePath:\n                            description: devicePath is the path inside of the container that the device will be mapped to.\n                            type: string\n                          name:\n                            description: name must match the name of a persistentVolumeClaim in the pod\n                            type: string\n                        required:\n                        - devicePath\n                        - name\n                        type: object\n                      type: array\n                    volumeMounts:\n                      description: Pod volumes to mount into the container's filesystem. Cannot be updated.\n                      items:\n                        description: VolumeMount describes a mounting of a Volume within a container.\n                        properties:\n                          mountPath:\n                            description: Path within the container at which the volume should be mounted.  Must not contain ':'.\n                            type: string\n                          mountPropagation:\n                            description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n                            type: string\n                          name:\n                            description: This must match the Name of a Volume.\n                            type: string\n                          readOnly:\n                            description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\n                            type: boolean\n                          subPath:\n                            description: Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).\n                            type: string\n                          subPathExpr:\n                            description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\n                            type: string\n                        required:\n                        - mountPath\n                        - name\n                        type: object\n                      type: array\n                    workingDir:\n                      description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\n                      type: string\n                  required:\n                  - name\n                  type: object\n                type: array\n              enforcedNamespaceLabel:\n                description: EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created.\n                type: string\n              evaluationInterval:\n                description: Interval between consecutive evaluations.\n                type: string\n              externalPrefix:\n                description: The external URL the Thanos Ruler instances will be available under. This is necessary to generate correct URLs. This is necessary if Thanos Ruler is not served from root of a DNS name.\n                type: string\n              grpcServerTlsConfig:\n                description: 'GRPCServerTLSConfig configures the gRPC server from which Thanos Querier reads recorded rule data. Note: Currently only the CAFile, CertFile, and KeyFile fields are supported. Maps to the ''--grpc-server-tls-*'' CLI args.'\n                properties:\n                  ca:\n                    description: Struct containing the CA cert to use for the targets.\n                    properties:\n                      configMap:\n                        description: ConfigMap containing data to use for the targets.\n                        properties:\n                          key:\n                            description: The key to select.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the ConfigMap or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                      secret:\n                        description: Secret containing data to use for the targets.\n                        properties:\n                          key:\n                            description: The key of the secret to select from.  Must be a valid secret key.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the Secret or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                    type: object\n                  caFile:\n                    description: Path to the CA cert in the Prometheus container to use for the targets.\n                    type: string\n                  cert:\n                    description: Struct containing the client cert file for the targets.\n                    properties:\n                      configMap:\n                        description: ConfigMap containing data to use for the targets.\n                        properties:\n                          key:\n                            description: The key to select.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the ConfigMap or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                      secret:\n                        description: Secret containing data to use for the targets.\n                        properties:\n                          key:\n                            description: The key of the secret to select from.  Must be a valid secret key.\n                            type: string\n                          name:\n                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                            type: string\n                          optional:\n                            description: Specify whether the Secret or its key must be defined\n                            type: boolean\n                        required:\n                        - key\n                        type: object\n                    type: object\n                  certFile:\n                    description: Path to the client cert file in the Prometheus container for the targets.\n                    type: string\n                  insecureSkipVerify:\n                    description: Disable target certificate validation.\n                    type: boolean\n                  keyFile:\n                    description: Path to the client key file in the Prometheus container for the targets.\n                    type: string\n                  keySecret:\n                    description: Secret containing the client key file for the targets.\n                    properties:\n                      key:\n                        description: The key of the secret to select from.  Must be a valid secret key.\n                        type: string\n                      name:\n                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                        type: string\n                      optional:\n                        description: Specify whether the Secret or its key must be defined\n                        type: boolean\n                    required:\n                    - key\n                    type: object\n                  serverName:\n                    description: Used to verify the hostname for the targets.\n                    type: string\n                type: object\n              image:\n                description: Thanos container image URL.\n                type: string\n              imagePullSecrets:\n                description: An optional list of references to secrets in the same namespace to use for pulling thanos images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod\n                items:\n                  description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.\n                  properties:\n                    name:\n                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                      type: string\n                  type: object\n                type: array\n              initContainers:\n                description: 'InitContainers allows adding initContainers to the pod definition. Those can be used to e.g. fetch secrets for injection into the ThanosRuler configuration from external sources. Any errors during the execution of an initContainer will lead to a restart of the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ Using initContainers for any use case other then secret fetching is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice.'\n                items:\n                  description: A single application container that you want to run within a pod.\n                  properties:\n                    args:\n                      description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    command:\n                      description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell'\n                      items:\n                        type: string\n                      type: array\n                    env:\n                      description: List of environment variables to set in the container. Cannot be updated.\n                      items:\n                        description: EnvVar represents an environment variable present in a Container.\n                        properties:\n                          name:\n                            description: Name of the environment variable. Must be a C_IDENTIFIER.\n                            type: string\n                          value:\n                            description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".'\n                            type: string\n                          valueFrom:\n                            description: Source for the environment variable's value. Cannot be used if value is not empty.\n                            properties:\n                              configMapKeyRef:\n                                description: Selects a key of a ConfigMap.\n                                properties:\n                                  key:\n                                    description: The key to select.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                              fieldRef:\n                                description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                              secretKeyRef:\n                                description: Selects a key of a secret in the pod's namespace\n                                properties:\n                                  key:\n                                    description: The key of the secret to select from.  Must be a valid secret key.\n                                    type: string\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                required:\n                                - key\n                                type: object\n                            type: object\n                        required:\n                        - name\n                        type: object\n                      type: array\n                    envFrom:\n                      description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.\n                      items:\n                        description: EnvFromSource represents the source of a set of ConfigMaps\n                        properties:\n                          configMapRef:\n                            description: The ConfigMap to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the ConfigMap must be defined\n                                type: boolean\n                            type: object\n                          prefix:\n                            description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.\n                            type: string\n                          secretRef:\n                            description: The Secret to select from\n                            properties:\n                              name:\n                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                type: string\n                              optional:\n                                description: Specify whether the Secret must be defined\n                                type: boolean\n                            type: object\n                        type: object\n                      type: array\n                    image:\n                      description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.'\n                      type: string\n                    imagePullPolicy:\n                      description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images'\n                      type: string\n                    lifecycle:\n                      description: Actions that the management system should take in response to container lifecycle events. Cannot be updated.\n                      properties:\n                        postStart:\n                          description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                        preStop:\n                          description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks'\n                          properties:\n                            exec:\n                              description: One and only one of the following should be specified. Exec specifies the action to take.\n                              properties:\n                                command:\n                                  description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                                  items:\n                                    type: string\n                                  type: array\n                              type: object\n                            httpGet:\n                              description: HTTPGet specifies the http request to perform.\n                              properties:\n                                host:\n                                  description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                                  type: string\n                                httpHeaders:\n                                  description: Custom headers to set in the request. HTTP allows repeated headers.\n                                  items:\n                                    description: HTTPHeader describes a custom header to be used in HTTP probes\n                                    properties:\n                                      name:\n                                        description: The header field name\n                                        type: string\n                                      value:\n                                        description: The header field value\n                                        type: string\n                                    required:\n                                    - name\n                                    - value\n                                    type: object\n                                  type: array\n                                path:\n                                  description: Path to access on the HTTP server.\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                                scheme:\n                                  description: Scheme to use for connecting to the host. Defaults to HTTP.\n                                  type: string\n                              required:\n                              - port\n                              type: object\n                            tcpSocket:\n                              description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                              properties:\n                                host:\n                                  description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                                  type: string\n                                port:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                                  x-kubernetes-int-or-string: true\n                              required:\n                              - port\n                              type: object\n                          type: object\n                      type: object\n                    livenessProbe:\n                      description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    name:\n                      description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.\n                      type: string\n                    ports:\n                      description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Cannot be updated.\n                      items:\n                        description: ContainerPort represents a network port in a single container.\n                        properties:\n                          containerPort:\n                            description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536.\n                            format: int32\n                            type: integer\n                          hostIP:\n                            description: What host IP to bind the external port to.\n                            type: string\n                          hostPort:\n                            description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.\n                            format: int32\n                            type: integer\n                          name:\n                            description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.\n                            type: string\n                          protocol:\n                            default: TCP\n                            description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".\n                            type: string\n                        required:\n                        - containerPort\n                        type: object\n                      type: array\n                      x-kubernetes-list-map-keys:\n                      - containerPort\n                      - protocol\n                      x-kubernetes-list-type: map\n                    readinessProbe:\n                      description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    resources:\n                      description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                      properties:\n                        limits:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                        requests:\n                          additionalProperties:\n                            anyOf:\n                            - type: integer\n                            - type: string\n                            pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                            x-kubernetes-int-or-string: true\n                          description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                          type: object\n                      type: object\n                    securityContext:\n                      description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/'\n                      properties:\n                        allowPrivilegeEscalation:\n                          description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN'\n                          type: boolean\n                        capabilities:\n                          description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime.\n                          properties:\n                            add:\n                              description: Added capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                            drop:\n                              description: Removed capabilities\n                              items:\n                                description: Capability represent POSIX capabilities type\n                                type: string\n                              type: array\n                          type: object\n                        privileged:\n                          description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false.\n                          type: boolean\n                        procMount:\n                          description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled.\n                          type: string\n                        readOnlyRootFilesystem:\n                          description: Whether this container has a read-only root filesystem. Default is false.\n                          type: boolean\n                        runAsGroup:\n                          description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        runAsNonRoot:\n                          description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          type: boolean\n                        runAsUser:\n                          description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          format: int64\n                          type: integer\n                        seLinuxOptions:\n                          description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in PodSecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            level:\n                              description: Level is SELinux level label that applies to the container.\n                              type: string\n                            role:\n                              description: Role is a SELinux role label that applies to the container.\n                              type: string\n                            type:\n                              description: Type is a SELinux type label that applies to the container.\n                              type: string\n                            user:\n                              description: User is a SELinux user label that applies to the container.\n                              type: string\n                          type: object\n                        windowsOptions:\n                          description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                          properties:\n                            gmsaCredentialSpec:\n                              description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                              type: string\n                            gmsaCredentialSpecName:\n                              description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                              type: string\n                            runAsUserName:\n                              description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                              type: string\n                          type: object\n                      type: object\n                    startupProbe:\n                      description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      properties:\n                        exec:\n                          description: One and only one of the following should be specified. Exec specifies the action to take.\n                          properties:\n                            command:\n                              description: Command is the command line to execute inside the container, the working directory for the command  is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n                              items:\n                                type: string\n                              type: array\n                          type: object\n                        failureThreshold:\n                          description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        httpGet:\n                          description: HTTPGet specifies the http request to perform.\n                          properties:\n                            host:\n                              description: Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.\n                              type: string\n                            httpHeaders:\n                              description: Custom headers to set in the request. HTTP allows repeated headers.\n                              items:\n                                description: HTTPHeader describes a custom header to be used in HTTP probes\n                                properties:\n                                  name:\n                                    description: The header field name\n                                    type: string\n                                  value:\n                                    description: The header field value\n                                    type: string\n                                required:\n                                - name\n                                - value\n                                type: object\n                              type: array\n                            path:\n                              description: Path to access on the HTTP server.\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                            scheme:\n                              description: Scheme to use for connecting to the host. Defaults to HTTP.\n                              type: string\n                          required:\n                          - port\n                          type: object\n                        initialDelaySeconds:\n                          description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                        periodSeconds:\n                          description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        successThreshold:\n                          description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n                          format: int32\n                          type: integer\n                        tcpSocket:\n                          description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook'\n                          properties:\n                            host:\n                              description: 'Optional: Host name to connect to, defaults to the pod IP.'\n                              type: string\n                            port:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.\n                              x-kubernetes-int-or-string: true\n                          required:\n                          - port\n                          type: object\n                        timeoutSeconds:\n                          description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                          format: int32\n                          type: integer\n                      type: object\n                    stdin:\n                      description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.\n                      type: boolean\n                    stdinOnce:\n                      description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false\n                      type: boolean\n                    terminationMessagePath:\n                      description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.'\n                      type: string\n                    terminationMessagePolicy:\n                      description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.\n                      type: string\n                    tty:\n                      description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.\n                      type: boolean\n                    volumeDevices:\n                      description: volumeDevices is the list of block devices to be used by the container.\n                      items:\n                        description: volumeDevice describes a mapping of a raw block device within a container.\n                        properties:\n                          devicePath:\n                            description: devicePath is the path inside of the container that the device will be mapped to.\n                            type: string\n                          name:\n                            description: name must match the name of a persistentVolumeClaim in the pod\n                            type: string\n                        required:\n                        - devicePath\n                        - name\n                        type: object\n                      type: array\n                    volumeMounts:\n                      description: Pod volumes to mount into the container's filesystem. Cannot be updated.\n                      items:\n                        description: VolumeMount describes a mounting of a Volume within a container.\n                        properties:\n                          mountPath:\n                            description: Path within the container at which the volume should be mounted.  Must not contain ':'.\n                            type: string\n                          mountPropagation:\n                            description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.\n                            type: string\n                          name:\n                            description: This must match the Name of a Volume.\n                            type: string\n                          readOnly:\n                            description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.\n                            type: boolean\n                          subPath:\n                            description: Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).\n                            type: string\n                          subPathExpr:\n                            description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.\n                            type: string\n                        required:\n                        - mountPath\n                        - name\n                        type: object\n                      type: array\n                    workingDir:\n                      description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.\n                      type: string\n                  required:\n                  - name\n                  type: object\n                type: array\n              labels:\n                additionalProperties:\n                  type: string\n                description: Labels configure the external label pairs to ThanosRuler. If not provided, default replica label `thanos_ruler_replica` will be added as a label and be dropped in alerts.\n                type: object\n              listenLocal:\n                description: ListenLocal makes the Thanos ruler listen on loopback, so that it does not bind against the Pod IP.\n                type: boolean\n              logFormat:\n                description: Log format for ThanosRuler to be configured with.\n                type: string\n              logLevel:\n                description: Log level for ThanosRuler to be configured with.\n                type: string\n              nodeSelector:\n                additionalProperties:\n                  type: string\n                description: Define which Nodes the Pods are scheduled on.\n                type: object\n              objectStorageConfig:\n                description: ObjectStorageConfig configures object storage in Thanos. Alternative to ObjectStorageConfigFile, and lower order priority.\n                properties:\n                  key:\n                    description: The key of the secret to select from.  Must be a valid secret key.\n                    type: string\n                  name:\n                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                    type: string\n                  optional:\n                    description: Specify whether the Secret or its key must be defined\n                    type: boolean\n                required:\n                - key\n                type: object\n              objectStorageConfigFile:\n                description: ObjectStorageConfigFile specifies the path of the object storage configuration file. When used alongside with ObjectStorageConfig, ObjectStorageConfigFile takes precedence.\n                type: string\n              paused:\n                description: When a ThanosRuler deployment is paused, no actions except for deletion will be performed on the underlying objects.\n                type: boolean\n              podMetadata:\n                description: PodMetadata contains Labels and Annotations gets propagated to the thanos ruler pods.\n                properties:\n                  annotations:\n                    additionalProperties:\n                      type: string\n                    description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'\n                    type: object\n                  labels:\n                    additionalProperties:\n                      type: string\n                    description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'\n                    type: object\n                  name:\n                    description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'\n                    type: string\n                type: object\n              portName:\n                description: Port name used for the pods and governing service. This defaults to web\n                type: string\n              priorityClassName:\n                description: Priority class assigned to the Pods\n                type: string\n              prometheusRulesExcludedFromEnforce:\n                description: PrometheusRulesExcludedFromEnforce - list of Prometheus rules to be excluded from enforcing of adding namespace labels. Works only if enforcedNamespaceLabel set to true. Make sure both ruleNamespace and ruleName are set for each pair\n                items:\n                  description: PrometheusRuleExcludeConfig enables users to configure excluded PrometheusRule names and their namespaces to be ignored while enforcing namespace label for alerts and metrics.\n                  properties:\n                    ruleName:\n                      description: RuleNamespace - name of excluded rule\n                      type: string\n                    ruleNamespace:\n                      description: RuleNamespace - namespace of excluded rule\n                      type: string\n                  required:\n                  - ruleName\n                  - ruleNamespace\n                  type: object\n                type: array\n              queryConfig:\n                description: Define configuration for connecting to thanos query instances. If this is defined, the QueryEndpoints field will be ignored. Maps to the `query.config` CLI argument. Only available with thanos v0.11.0 and higher.\n                properties:\n                  key:\n                    description: The key of the secret to select from.  Must be a valid secret key.\n                    type: string\n                  name:\n                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                    type: string\n                  optional:\n                    description: Specify whether the Secret or its key must be defined\n                    type: boolean\n                required:\n                - key\n                type: object\n              queryEndpoints:\n                description: QueryEndpoints defines Thanos querier endpoints from which to query metrics. Maps to the --query flag of thanos ruler.\n                items:\n                  type: string\n                type: array\n              replicas:\n                description: Number of thanos ruler instances to deploy.\n                format: int32\n                type: integer\n              resources:\n                description: Resources defines the resource requirements for single Pods. If not provided, no requests/limits will be set\n                properties:\n                  limits:\n                    additionalProperties:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                      x-kubernetes-int-or-string: true\n                    description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                    type: object\n                  requests:\n                    additionalProperties:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                      x-kubernetes-int-or-string: true\n                    description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                    type: object\n                type: object\n              retention:\n                description: Time duration ThanosRuler shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years).\n                type: string\n              routePrefix:\n                description: The route prefix ThanosRuler registers HTTP handlers for. This allows thanos UI to be served on a sub-path.\n                type: string\n              ruleNamespaceSelector:\n                description: Namespaces to be selected for Rules discovery. If unspecified, only the same namespace as the ThanosRuler object is in is used.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              ruleSelector:\n                description: A label selector to select which PrometheusRules to mount for alerting and recording.\n                properties:\n                  matchExpressions:\n                    description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                    items:\n                      description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                      properties:\n                        key:\n                          description: key is the label key that the selector applies to.\n                          type: string\n                        operator:\n                          description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                          type: string\n                        values:\n                          description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                          items:\n                            type: string\n                          type: array\n                      required:\n                      - key\n                      - operator\n                      type: object\n                    type: array\n                  matchLabels:\n                    additionalProperties:\n                      type: string\n                    description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                    type: object\n                type: object\n              securityContext:\n                description: SecurityContext holds pod-level security attributes and common container settings. This defaults to the default PodSecurityContext.\n                properties:\n                  fsGroup:\n                    description: \"A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \\n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \\n If unset, the Kubelet will not modify the ownership and permissions of any volume.\"\n                    format: int64\n                    type: integer\n                  fsGroupChangePolicy:\n                    description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are \"OnRootMismatch\" and \"Always\". If not specified defaults to \"Always\".'\n                    type: string\n                  runAsGroup:\n                    description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    format: int64\n                    type: integer\n                  runAsNonRoot:\n                    description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                    type: boolean\n                  runAsUser:\n                    description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    format: int64\n                    type: integer\n                  seLinuxOptions:\n                    description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container.  May also be set in SecurityContext.  If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container.\n                    properties:\n                      level:\n                        description: Level is SELinux level label that applies to the container.\n                        type: string\n                      role:\n                        description: Role is a SELinux role label that applies to the container.\n                        type: string\n                      type:\n                        description: Type is a SELinux type label that applies to the container.\n                        type: string\n                      user:\n                        description: User is a SELinux user label that applies to the container.\n                        type: string\n                    type: object\n                  supplementalGroups:\n                    description: A list of groups applied to the first process run in each container, in addition to the container's primary GID.  If unspecified, no groups will be added to any container.\n                    items:\n                      format: int64\n                      type: integer\n                    type: array\n                  sysctls:\n                    description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch.\n                    items:\n                      description: Sysctl defines a kernel parameter to be set\n                      properties:\n                        name:\n                          description: Name of a property to set\n                          type: string\n                        value:\n                          description: Value of a property to set\n                          type: string\n                      required:\n                      - name\n                      - value\n                      type: object\n                    type: array\n                  windowsOptions:\n                    description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                    properties:\n                      gmsaCredentialSpec:\n                        description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.\n                        type: string\n                      gmsaCredentialSpecName:\n                        description: GMSACredentialSpecName is the name of the GMSA credential spec to use.\n                        type: string\n                      runAsUserName:\n                        description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n                        type: string\n                    type: object\n                type: object\n              serviceAccountName:\n                description: ServiceAccountName is the name of the ServiceAccount to use to run the Thanos Ruler Pods.\n                type: string\n              storage:\n                description: Storage spec to specify how storage shall be used.\n                properties:\n                  disableMountSubPath:\n                    description: 'Deprecated: subPath usage will be disabled by default in a future release, this option will become unnecessary. DisableMountSubPath allows to remove any subPath usage in volume mounts.'\n                    type: boolean\n                  emptyDir:\n                    description: 'EmptyDirVolumeSource to be used by the Prometheus StatefulSets. If specified, used in place of any volumeClaimTemplate. More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'\n                    properties:\n                      medium:\n                        description: 'What type of storage medium should back this directory. The default is \"\" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                        type: string\n                      sizeLimit:\n                        anyOf:\n                        - type: integer\n                        - type: string\n                        description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'\n                        pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                        x-kubernetes-int-or-string: true\n                    type: object\n                  volumeClaimTemplate:\n                    description: A PVC spec to be used by the Prometheus StatefulSets.\n                    properties:\n                      apiVersion:\n                        description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n                        type: string\n                      kind:\n                        description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n                        type: string\n                      metadata:\n                        description: EmbeddedMetadata contains metadata relevant to an EmbeddedResource.\n                        properties:\n                          annotations:\n                            additionalProperties:\n                              type: string\n                            description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'\n                            type: object\n                          labels:\n                            additionalProperties:\n                              type: string\n                            description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'\n                            type: object\n                          name:\n                            description: 'Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'\n                            type: string\n                        type: object\n                      spec:\n                        description: 'Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                        properties:\n                          accessModes:\n                            description: 'AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'\n                            items:\n                              type: string\n                            type: array\n                          dataSource:\n                            description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - Beta) * An existing PVC (PersistentVolumeClaim) * An existing custom resource/object that implements data population (Alpha) In order to use VolumeSnapshot object types, the appropriate feature gate must be enabled (VolumeSnapshotDataSource or AnyVolumeDataSource) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the specified data source is not supported, the volume will not be created and the failure will be reported as an event. In the future, we plan to support more data source types and the behavior of the provisioner may change.'\n                            properties:\n                              apiGroup:\n                                description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.\n                                type: string\n                              kind:\n                                description: Kind is the type of resource being referenced\n                                type: string\n                              name:\n                                description: Name is the name of resource being referenced\n                                type: string\n                            required:\n                            - kind\n                            - name\n                            type: object\n                          resources:\n                            description: 'Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'\n                            properties:\n                              limits:\n                                additionalProperties:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                  x-kubernetes-int-or-string: true\n                                description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                                type: object\n                              requests:\n                                additionalProperties:\n                                  anyOf:\n                                  - type: integer\n                                  - type: string\n                                  pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                  x-kubernetes-int-or-string: true\n                                description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                                type: object\n                            type: object\n                          selector:\n                            description: A label query over volumes to consider for binding.\n                            properties:\n                              matchExpressions:\n                                description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                                items:\n                                  description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                                  properties:\n                                    key:\n                                      description: key is the label key that the selector applies to.\n                                      type: string\n                                    operator:\n                                      description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                      type: string\n                                    values:\n                                      description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                      items:\n                                        type: string\n                                      type: array\n                                  required:\n                                  - key\n                                  - operator\n                                  type: object\n                                type: array\n                              matchLabels:\n                                additionalProperties:\n                                  type: string\n                                description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                                type: object\n                            type: object\n                          storageClassName:\n                            description: 'Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1'\n                            type: string\n                          volumeMode:\n                            description: volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec.\n                            type: string\n                          volumeName:\n                            description: VolumeName is the binding reference to the PersistentVolume backing this claim.\n                            type: string\n                        type: object\n                      status:\n                        description: 'Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                        properties:\n                          accessModes:\n                            description: 'AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'\n                            items:\n                              type: string\n                            type: array\n                          capacity:\n                            additionalProperties:\n                              anyOf:\n                              - type: integer\n                              - type: string\n                              pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                              x-kubernetes-int-or-string: true\n                            description: Represents the actual resources of the underlying volume.\n                            type: object\n                          conditions:\n                            description: Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'.\n                            items:\n                              description: PersistentVolumeClaimCondition contails details about state of pvc\n                              properties:\n                                lastProbeTime:\n                                  description: Last time we probed the condition.\n                                  format: date-time\n                                  type: string\n                                lastTransitionTime:\n                                  description: Last time the condition transitioned from one status to another.\n                                  format: date-time\n                                  type: string\n                                message:\n                                  description: Human-readable message indicating details about last transition.\n                                  type: string\n                                reason:\n                                  description: Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports \"ResizeStarted\" that means the underlying persistent volume is being resized.\n                                  type: string\n                                status:\n                                  type: string\n                                type:\n                                  description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type\n                                  type: string\n                              required:\n                              - status\n                              - type\n                              type: object\n                            type: array\n                          phase:\n                            description: Phase represents the current phase of PersistentVolumeClaim.\n                            type: string\n                        type: object\n                    type: object\n                type: object\n              tolerations:\n                description: If specified, the pod's tolerations.\n                items:\n                  description: The pod this Toleration is attached to tolerates any taint that matches the triple <key,value,effect> using the matching operator <operator>.\n                  properties:\n                    effect:\n                      description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\n                      type: string\n                    key:\n                      description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.\n                      type: string\n                    operator:\n                      description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.\n                      type: string\n                    tolerationSeconds:\n                      description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.\n                      format: int64\n                      type: integer\n                    value:\n                      description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.\n                      type: string\n                  type: object\n                type: array\n              topologySpreadConstraints:\n                description: If specified, the pod's topology spread constraints.\n                items:\n                  description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.\n                  properties:\n                    labelSelector:\n                      description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.\n                      properties:\n                        matchExpressions:\n                          description: matchExpressions is a list of label selector requirements. The requirements are ANDed.\n                          items:\n                            description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.\n                            properties:\n                              key:\n                                description: key is the label key that the selector applies to.\n                                type: string\n                              operator:\n                                description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.\n                                type: string\n                              values:\n                                description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.\n                                items:\n                                  type: string\n                                type: array\n                            required:\n                            - key\n                            - operator\n                            type: object\n                          type: array\n                        matchLabels:\n                          additionalProperties:\n                            type: string\n                          description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n                          type: object\n                      type: object\n                    maxSkew:\n                      description: 'MaxSkew describes the degree to which pods may be unevenly distributed. It''s the maximum permitted difference between the number of matching pods in any two topology domains of a given topology type. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | |   P   |   P   |       | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. It''s a required field. Default value is 1 and 0 is not allowed.'\n                      format: int32\n                      type: integer\n                    topologyKey:\n                      description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each <key, value> as a \"bucket\", and try to put balanced number of pods into each bucket. It's a required field.\n                      type: string\n                    whenUnsatisfiable:\n                      description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it - ScheduleAnyway tells the scheduler to still schedule it It''s considered as \"Unsatisfiable\" if and only if placing incoming pod on any topology violates \"MaxSkew\". For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P |   P   |   P   | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.'\n                      type: string\n                  required:\n                  - maxSkew\n                  - topologyKey\n                  - whenUnsatisfiable\n                  type: object\n                type: array\n              tracingConfig:\n                description: TracingConfig configures tracing in Thanos. This is an experimental feature, it may change in any upcoming release in a breaking way.\n                properties:\n                  key:\n                    description: The key of the secret to select from.  Must be a valid secret key.\n                    type: string\n                  name:\n                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                    type: string\n                  optional:\n                    description: Specify whether the Secret or its key must be defined\n                    type: boolean\n                required:\n                - key\n                type: object\n              volumes:\n                description: Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended to other volumes that are generated as a result of StorageSpec objects.\n                items:\n                  description: Volume represents a named volume in a pod that may be accessed by any container in the pod.\n                  properties:\n                    awsElasticBlockStore:\n                      description: 'AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        partition:\n                          description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty).'\n                          format: int32\n                          type: integer\n                        readOnly:\n                          description: 'Specify \"true\" to force and set the ReadOnly property in VolumeMounts to \"true\". If omitted, the default is \"false\". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                          type: boolean\n                        volumeID:\n                          description: 'Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore'\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    azureDisk:\n                      description: AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.\n                      properties:\n                        cachingMode:\n                          description: 'Host Caching mode: None, Read Only, Read Write.'\n                          type: string\n                        diskName:\n                          description: The Name of the data disk in the blob storage\n                          type: string\n                        diskURI:\n                          description: The URI the data disk in the blob storage\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        kind:\n                          description: 'Expected values Shared: multiple blob disks per storage account  Dedicated: single blob disk per storage account  Managed: azure managed data disk (only in managed availability set). defaults to shared'\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                      required:\n                      - diskName\n                      - diskURI\n                      type: object\n                    azureFile:\n                      description: AzureFile represents an Azure File Service mount on the host and bind mount to the pod.\n                      properties:\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretName:\n                          description: the name of secret that contains Azure Storage Account Name and Key\n                          type: string\n                        shareName:\n                          description: Share Name\n                          type: string\n                      required:\n                      - secretName\n                      - shareName\n                      type: object\n                    cephfs:\n                      description: CephFS represents a Ceph FS mount on the host that shares a pod's lifetime\n                      properties:\n                        monitors:\n                          description: 'Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          items:\n                            type: string\n                          type: array\n                        path:\n                          description: 'Optional: Used as the mounted root, rather than the full Ceph tree, default is /'\n                          type: string\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: boolean\n                        secretFile:\n                          description: 'Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: string\n                        secretRef:\n                          description: 'Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        user:\n                          description: 'Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it'\n                          type: string\n                      required:\n                      - monitors\n                      type: object\n                    cinder:\n                      description: 'Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: string\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: boolean\n                        secretRef:\n                          description: 'Optional: points to a secret object containing parameters used to connect to OpenStack.'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        volumeID:\n                          description: 'volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md'\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    configMap:\n                      description: ConfigMap represents a configMap that should populate this volume\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                          items:\n                            description: Maps a string key to a path within a volume.\n                            properties:\n                              key:\n                                description: The key to project.\n                                type: string\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                type: string\n                            required:\n                            - key\n                            - path\n                            type: object\n                          type: array\n                        name:\n                          description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                          type: string\n                        optional:\n                          description: Specify whether the ConfigMap or its keys must be defined\n                          type: boolean\n                      type: object\n                    csi:\n                      description: CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature).\n                      properties:\n                        driver:\n                          description: Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster.\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Ex. \"ext4\", \"xfs\", \"ntfs\". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply.\n                          type: string\n                        nodePublishSecretRef:\n                          description: NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and  may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        readOnly:\n                          description: Specifies a read-only configuration for the volume. Defaults to false (read/write).\n                          type: boolean\n                        volumeAttributes:\n                          additionalProperties:\n                            type: string\n                          description: VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values.\n                          type: object\n                      required:\n                      - driver\n                      type: object\n                    downwardAPI:\n                      description: DownwardAPI represents downward API about the pod that should populate this volume\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: Items is a list of downward API volume file\n                          items:\n                            description: DownwardAPIVolumeFile represents information to create the file containing the pod field\n                            properties:\n                              fieldRef:\n                                description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.'\n                                properties:\n                                  apiVersion:\n                                    description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                    type: string\n                                  fieldPath:\n                                    description: Path of the field to select in the specified API version.\n                                    type: string\n                                required:\n                                - fieldPath\n                                type: object\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: 'Required: Path is  the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..'''\n                                type: string\n                              resourceFieldRef:\n                                description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.'\n                                properties:\n                                  containerName:\n                                    description: 'Container name: required for volumes, optional for env vars'\n                                    type: string\n                                  divisor:\n                                    anyOf:\n                                    - type: integer\n                                    - type: string\n                                    description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                    pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                    x-kubernetes-int-or-string: true\n                                  resource:\n                                    description: 'Required: resource to select'\n                                    type: string\n                                required:\n                                - resource\n                                type: object\n                            required:\n                            - path\n                            type: object\n                          type: array\n                      type: object\n                    emptyDir:\n                      description: 'EmptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                      properties:\n                        medium:\n                          description: 'What type of storage medium should back this directory. The default is \"\" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir'\n                          type: string\n                        sizeLimit:\n                          anyOf:\n                          - type: integer\n                          - type: string\n                          description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir'\n                          pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                          x-kubernetes-int-or-string: true\n                      type: object\n                    fc:\n                      description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod.\n                      properties:\n                        fsType:\n                          description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        lun:\n                          description: 'Optional: FC target lun number'\n                          format: int32\n                          type: integer\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.'\n                          type: boolean\n                        targetWWNs:\n                          description: 'Optional: FC target worldwide names (WWNs)'\n                          items:\n                            type: string\n                          type: array\n                        wwids:\n                          description: 'Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.'\n                          items:\n                            type: string\n                          type: array\n                      type: object\n                    flexVolume:\n                      description: FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin.\n                      properties:\n                        driver:\n                          description: Driver is the name of the driver to use for this volume.\n                          type: string\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default filesystem depends on FlexVolume script.\n                          type: string\n                        options:\n                          additionalProperties:\n                            type: string\n                          description: 'Optional: Extra command options if any.'\n                          type: object\n                        readOnly:\n                          description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.'\n                          type: boolean\n                        secretRef:\n                          description: 'Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                      required:\n                      - driver\n                      type: object\n                    flocker:\n                      description: Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running\n                      properties:\n                        datasetName:\n                          description: Name of the dataset stored as metadata -> name on the dataset for Flocker should be considered as deprecated\n                          type: string\n                        datasetUUID:\n                          description: UUID of the dataset. This is unique identifier of a Flocker dataset\n                          type: string\n                      type: object\n                    gcePersistentDisk:\n                      description: 'GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        partition:\n                          description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          format: int32\n                          type: integer\n                        pdName:\n                          description: 'Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk'\n                          type: boolean\n                      required:\n                      - pdName\n                      type: object\n                    gitRepo:\n                      description: 'GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod''s container.'\n                      properties:\n                        directory:\n                          description: Target directory name. Must not contain or start with '..'.  If '.' is supplied, the volume directory will be the git repository.  Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name.\n                          type: string\n                        repository:\n                          description: Repository URL\n                          type: string\n                        revision:\n                          description: Commit hash for the specified revision.\n                          type: string\n                      required:\n                      - repository\n                      type: object\n                    glusterfs:\n                      description: 'Glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md'\n                      properties:\n                        endpoints:\n                          description: 'EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: string\n                        path:\n                          description: 'Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod'\n                          type: boolean\n                      required:\n                      - endpoints\n                      - path\n                      type: object\n                    hostPath:\n                      description: 'HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.'\n                      properties:\n                        path:\n                          description: 'Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath'\n                          type: string\n                        type:\n                          description: 'Type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath'\n                          type: string\n                      required:\n                      - path\n                      type: object\n                    iscsi:\n                      description: 'ISCSI represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md'\n                      properties:\n                        chapAuthDiscovery:\n                          description: whether support iSCSI Discovery CHAP authentication\n                          type: boolean\n                        chapAuthSession:\n                          description: whether support iSCSI Session CHAP authentication\n                          type: boolean\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        initiatorName:\n                          description: Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface <target portal>:<volume name> will be created for the connection.\n                          type: string\n                        iqn:\n                          description: Target iSCSI Qualified Name.\n                          type: string\n                        iscsiInterface:\n                          description: iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp).\n                          type: string\n                        lun:\n                          description: iSCSI Target Lun number.\n                          format: int32\n                          type: integer\n                        portals:\n                          description: iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).\n                          items:\n                            type: string\n                          type: array\n                        readOnly:\n                          description: ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false.\n                          type: boolean\n                        secretRef:\n                          description: CHAP Secret for iSCSI target and initiator authentication\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        targetPortal:\n                          description: iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).\n                          type: string\n                      required:\n                      - iqn\n                      - lun\n                      - targetPortal\n                      type: object\n                    name:\n                      description: 'Volume''s name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'\n                      type: string\n                    nfs:\n                      description: 'NFS represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                      properties:\n                        path:\n                          description: 'Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: boolean\n                        server:\n                          description: 'Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs'\n                          type: string\n                      required:\n                      - path\n                      - server\n                      type: object\n                    persistentVolumeClaim:\n                      description: 'PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                      properties:\n                        claimName:\n                          description: 'ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims'\n                          type: string\n                        readOnly:\n                          description: Will force the ReadOnly setting in VolumeMounts. Default false.\n                          type: boolean\n                      required:\n                      - claimName\n                      type: object\n                    photonPersistentDisk:\n                      description: PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        pdID:\n                          description: ID that identifies Photon Controller persistent disk\n                          type: string\n                      required:\n                      - pdID\n                      type: object\n                    portworxVolume:\n                      description: PortworxVolume represents a portworx volume attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        volumeID:\n                          description: VolumeID uniquely identifies a Portworx volume\n                          type: string\n                      required:\n                      - volumeID\n                      type: object\n                    projected:\n                      description: Items for all in one resources secrets, configmaps, and downward API\n                      properties:\n                        defaultMode:\n                          description: Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.\n                          format: int32\n                          type: integer\n                        sources:\n                          description: list of volume projections\n                          items:\n                            description: Projection that may be projected along with other supported volume types\n                            properties:\n                              configMap:\n                                description: information about the configMap data to project\n                                properties:\n                                  items:\n                                    description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                                    items:\n                                      description: Maps a string key to a path within a volume.\n                                      properties:\n                                        key:\n                                          description: The key to project.\n                                          type: string\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                          type: string\n                                      required:\n                                      - key\n                                      - path\n                                      type: object\n                                    type: array\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the ConfigMap or its keys must be defined\n                                    type: boolean\n                                type: object\n                              downwardAPI:\n                                description: information about the downwardAPI data to project\n                                properties:\n                                  items:\n                                    description: Items is a list of DownwardAPIVolume file\n                                    items:\n                                      description: DownwardAPIVolumeFile represents information to create the file containing the pod field\n                                      properties:\n                                        fieldRef:\n                                          description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.'\n                                          properties:\n                                            apiVersion:\n                                              description: Version of the schema the FieldPath is written in terms of, defaults to \"v1\".\n                                              type: string\n                                            fieldPath:\n                                              description: Path of the field to select in the specified API version.\n                                              type: string\n                                          required:\n                                          - fieldPath\n                                          type: object\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: 'Required: Path is  the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..'''\n                                          type: string\n                                        resourceFieldRef:\n                                          description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.'\n                                          properties:\n                                            containerName:\n                                              description: 'Container name: required for volumes, optional for env vars'\n                                              type: string\n                                            divisor:\n                                              anyOf:\n                                              - type: integer\n                                              - type: string\n                                              description: Specifies the output format of the exposed resources, defaults to \"1\"\n                                              pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                                              x-kubernetes-int-or-string: true\n                                            resource:\n                                              description: 'Required: resource to select'\n                                              type: string\n                                          required:\n                                          - resource\n                                          type: object\n                                      required:\n                                      - path\n                                      type: object\n                                    type: array\n                                type: object\n                              secret:\n                                description: information about the secret data to project\n                                properties:\n                                  items:\n                                    description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                                    items:\n                                      description: Maps a string key to a path within a volume.\n                                      properties:\n                                        key:\n                                          description: The key to project.\n                                          type: string\n                                        mode:\n                                          description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                          format: int32\n                                          type: integer\n                                        path:\n                                          description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                          type: string\n                                      required:\n                                      - key\n                                      - path\n                                      type: object\n                                    type: array\n                                  name:\n                                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                                    type: string\n                                  optional:\n                                    description: Specify whether the Secret or its key must be defined\n                                    type: boolean\n                                type: object\n                              serviceAccountToken:\n                                description: information about the serviceAccountToken data to project\n                                properties:\n                                  audience:\n                                    description: Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver.\n                                    type: string\n                                  expirationSeconds:\n                                    description: ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes.\n                                    format: int64\n                                    type: integer\n                                  path:\n                                    description: Path is the path relative to the mount point of the file to project the token into.\n                                    type: string\n                                required:\n                                - path\n                                type: object\n                            type: object\n                          type: array\n                      required:\n                      - sources\n                      type: object\n                    quobyte:\n                      description: Quobyte represents a Quobyte mount on the host that shares a pod's lifetime\n                      properties:\n                        group:\n                          description: Group to map volume access to Default is no group\n                          type: string\n                        readOnly:\n                          description: ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false.\n                          type: boolean\n                        registry:\n                          description: Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes\n                          type: string\n                        tenant:\n                          description: Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin\n                          type: string\n                        user:\n                          description: User to map volume access to Defaults to serivceaccount user\n                          type: string\n                        volume:\n                          description: Volume is a string that references an already created Quobyte volume by name.\n                          type: string\n                      required:\n                      - registry\n                      - volume\n                      type: object\n                    rbd:\n                      description: 'RBD represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md'\n                      properties:\n                        fsType:\n                          description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine'\n                          type: string\n                        image:\n                          description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        keyring:\n                          description: 'Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        monitors:\n                          description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          items:\n                            type: string\n                          type: array\n                        pool:\n                          description: 'The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                        readOnly:\n                          description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: boolean\n                        secretRef:\n                          description: 'SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        user:\n                          description: 'The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it'\n                          type: string\n                      required:\n                      - image\n                      - monitors\n                      type: object\n                    scaleIO:\n                      description: ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Default is \"xfs\".\n                          type: string\n                        gateway:\n                          description: The host address of the ScaleIO API Gateway.\n                          type: string\n                        protectionDomain:\n                          description: The name of the ScaleIO Protection Domain for the configured storage.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretRef:\n                          description: SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        sslEnabled:\n                          description: Flag to enable/disable SSL communication with Gateway, default false\n                          type: boolean\n                        storageMode:\n                          description: Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned.\n                          type: string\n                        storagePool:\n                          description: The ScaleIO Storage Pool associated with the protection domain.\n                          type: string\n                        system:\n                          description: The name of the storage system as configured in ScaleIO.\n                          type: string\n                        volumeName:\n                          description: The name of a volume already created in the ScaleIO system that is associated with this volume source.\n                          type: string\n                      required:\n                      - gateway\n                      - secretRef\n                      - system\n                      type: object\n                    secret:\n                      description: 'Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret'\n                      properties:\n                        defaultMode:\n                          description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                          format: int32\n                          type: integer\n                        items:\n                          description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.\n                          items:\n                            description: Maps a string key to a path within a volume.\n                            properties:\n                              key:\n                                description: The key to project.\n                                type: string\n                              mode:\n                                description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.'\n                                format: int32\n                                type: integer\n                              path:\n                                description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.\n                                type: string\n                            required:\n                            - key\n                            - path\n                            type: object\n                          type: array\n                        optional:\n                          description: Specify whether the Secret or its keys must be defined\n                          type: boolean\n                        secretName:\n                          description: 'Name of the secret in the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret'\n                          type: string\n                      type: object\n                    storageos:\n                      description: StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes.\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        readOnly:\n                          description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.\n                          type: boolean\n                        secretRef:\n                          description: SecretRef specifies the secret to use for obtaining the StorageOS API credentials.  If not specified, default values will be attempted.\n                          properties:\n                            name:\n                              description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'\n                              type: string\n                          type: object\n                        volumeName:\n                          description: VolumeName is the human-readable name of the StorageOS volume.  Volume names are only unique within a namespace.\n                          type: string\n                        volumeNamespace:\n                          description: VolumeNamespace specifies the scope of the volume within StorageOS.  If no namespace is specified then the Pod's namespace will be used.  This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to \"default\" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created.\n                          type: string\n                      type: object\n                    vsphereVolume:\n                      description: VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine\n                      properties:\n                        fsType:\n                          description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n                          type: string\n                        storagePolicyID:\n                          description: Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName.\n                          type: string\n                        storagePolicyName:\n                          description: Storage Policy Based Management (SPBM) profile name.\n                          type: string\n                        volumePath:\n                          description: Path that identifies vSphere volume vmdk\n                          type: string\n                      required:\n                      - volumePath\n                      type: object\n                  required:\n                  - name\n                  type: object\n                type: array\n            type: object\n          status:\n            description: 'Most recent observed status of the ThanosRuler cluster. Read-only. Not included when requesting from the apiserver, only from the ThanosRuler Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'\n            properties:\n              availableReplicas:\n                description: Total number of available pods (ready for at least minReadySeconds) targeted by this ThanosRuler deployment.\n                format: int32\n                type: integer\n              paused:\n                description: Represents whether any actions on the underlying managed objects are being performed. Only delete actions will be performed.\n                type: boolean\n              replicas:\n                description: Total number of non-terminated pods targeted by this ThanosRuler deployment (their labels match the selector).\n                format: int32\n                type: integer\n              unavailableReplicas:\n                description: Total number of unavailable pods targeted by this ThanosRuler deployment.\n                format: int32\n                type: integer\n              updatedReplicas:\n                description: Total number of non-terminated pods targeted by this ThanosRuler deployment that have the desired version spec.\n                format: int32\n                type: integer\n            required:\n            - availableReplicas\n            - paused\n            - replicas\n            - unavailableReplicas\n            - updatedReplicas\n            type: object\n        required:\n        - spec\n        type: object\n    served: true\n    storage: true\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  labels:\n    app.kubernetes.io/component: controller\n    app.kubernetes.io/name: prometheus-operator\n    app.kubernetes.io/version: 0.48.1\n  name: prometheus-operator\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: prometheus-operator\nsubjects:\n- kind: ServiceAccount\n  name: prometheus-operator\n  namespace: prometheus\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  labels:\n    app.kubernetes.io/component: controller\n    app.kubernetes.io/name: prometheus-operator\n    app.kubernetes.io/version: 0.48.1\n  name: prometheus-operator\nrules:\n- apiGroups:\n  - monitoring.coreos.com\n  resources:\n  - alertmanagers\n  - alertmanagers/finalizers\n  - alertmanagerconfigs\n  - prometheuses\n  - prometheuses/finalizers\n  - thanosrulers\n  - thanosrulers/finalizers\n  - servicemonitors\n  - podmonitors\n  - probes\n  - prometheusrules\n  verbs:\n  - '*'\n- apiGroups:\n  - apps\n  resources:\n  - statefulsets\n  verbs:\n  - '*'\n- apiGroups:\n  - \"\"\n  resources:\n  - configmaps\n  - secrets\n  verbs:\n  - '*'\n- apiGroups:\n  - \"\"\n  resources:\n  - pods\n  verbs:\n  - list\n  - delete\n- apiGroups:\n  - \"\"\n  resources:\n  - services\n  - services/finalizers\n  - endpoints\n  verbs:\n  - get\n  - create\n  - update\n  - delete\n- apiGroups:\n  - \"\"\n  resources:\n  - nodes\n  verbs:\n  - list\n  - watch\n- apiGroups:\n  - \"\"\n  resources:\n  - namespaces\n  verbs:\n  - get\n  - list\n  - watch\n- apiGroups:\n  - networking.k8s.io\n  resources:\n  - ingresses\n  verbs:\n  - get\n  - list\n  - watch\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  labels:\n    app.kubernetes.io/component: controller\n    app.kubernetes.io/name: prometheus-operator\n    app.kubernetes.io/version: 0.48.1\n  name: prometheus-operator\n  namespace: prometheus\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      app.kubernetes.io/component: controller\n      app.kubernetes.io/name: prometheus-operator\n  template:\n    metadata:\n      annotations:\n        kubectl.kubernetes.io/default-container: prometheus-operator\n      labels:\n        app.kubernetes.io/component: controller\n        app.kubernetes.io/name: prometheus-operator\n        app.kubernetes.io/version: 0.48.1\n    spec:\n      containers:\n      - args:\n          - --kubelet-service=kube-system/kubelet\n          - --prometheus-config-reloader=$CORTEX_IMAGE_PROMETHEUS_CONFIG_RELOADER\n        image: $CORTEX_IMAGE_PROMETHEUS_OPERATOR\n        name: prometheus-operator\n        ports:\n        - containerPort: 8080\n          name: http\n        resources:\n          limits:\n            cpu: 200m\n            memory: 200Mi\n          requests:\n            cpu: 100m\n            memory: 100Mi\n        securityContext:\n          allowPrivilegeEscalation: false\n      nodeSelector:\n        kubernetes.io/os: linux\n        prometheus: \"true\"\n      tolerations:\n        - key: prometheus\n          operator: Exists\n          effect: NoSchedule\n      securityContext:\n        runAsNonRoot: true\n        runAsUser: 65534\n      serviceAccountName: prometheus-operator\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  labels:\n    app.kubernetes.io/component: controller\n    app.kubernetes.io/name: prometheus-operator\n    app.kubernetes.io/version: 0.48.1\n  name: prometheus-operator\n  namespace: prometheus\n---\napiVersion: v1\nkind: Service\nmetadata:\n  labels:\n    app.kubernetes.io/component: controller\n    app.kubernetes.io/name: prometheus-operator\n    app.kubernetes.io/version: 0.48.1\n  name: prometheus-operator\n  namespace: prometheus\nspec:\n  clusterIP: None\n  ports:\n  - name: http\n    port: 8080\n    targetPort: http\n  selector:\n    app.kubernetes.io/component: controller\n    app.kubernetes.io/name: prometheus-operator\n"
  },
  {
    "path": "manager/manifests/prometheus-statsd-exporter.yaml",
    "content": "# Copyright 2013 The Prometheus Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n# Modifications Copyright 2022 Cortex Labs, Inc.\n\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: prometheus-statsd-exporter-config\n  namespace: prometheus\ndata:\n  statsd-mapping.yaml: |\n    defaults:\n      observer_type: histogram\n\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: prometheus-statsd-exporter\n  namespace: prometheus\nspec:\n  replicas: 1\n  selector:\n    matchLabels:\n      name: prometheus-statsd-exporter\n  template:\n    metadata:\n      labels:\n        name: prometheus-statsd-exporter\n    spec:\n      containers:\n        - name: prometheus-statsd-exporter\n          image: $CORTEX_IMAGE_PROMETHEUS_STATSD_EXPORTER\n          imagePullPolicy: Always\n          args:\n            - --web.listen-address=:9102\n            - --web.telemetry-path=/metrics\n            - --statsd.listen-udp=:9125\n            - --statsd.listen-tcp=:9125\n            - --statsd.cache-size=1000\n            - --statsd.event-queue-size=10000\n            - --statsd.event-flush-threshold=1000\n            - --statsd.event-flush-interval=200ms\n            - --statsd.mapping-config=/etc/prometheus-statsd-exporter/statsd-mapping.yaml\n          ports:\n            - name: metrics\n              containerPort: 9102\n              protocol: TCP\n            - name: statsd-udp\n              containerPort: 9125\n              protocol: UDP\n          livenessProbe:\n            httpGet:\n              path: /metrics\n              port: metrics\n            initialDelaySeconds: 30\n            periodSeconds: 30\n          resources:\n            limits:\n              memory: 100Mi\n            requests:\n              cpu: 100m\n              memory: 100Mi\n          volumeMounts:\n            - name: statsd-mapping-config\n              mountPath: /etc/prometheus-statsd-exporter\n      nodeSelector:\n        prometheus: \"true\"\n      tolerations:\n        - key: prometheus\n          operator: Exists\n          effect: NoSchedule\n      volumes:\n        - name: statsd-mapping-config\n          configMap:\n            name: prometheus-statsd-exporter-config\n            items:\n              - key: statsd-mapping.yaml\n                path: statsd-mapping.yaml\n      terminationGracePeriodSeconds: 60\n\n---\napiVersion: v1\nkind: Service\nmetadata:\n  namespace: prometheus\n  name: prometheus-statsd-exporter\n  labels:\n    cortex.dev/name: prometheus-statsd-exporter\nspec:\n  selector:\n    name: prometheus-statsd-exporter\n  ports:\n    - port: 9125\n      name: statsd-udp\n      protocol: UDP\n    - port: 9102\n      name: metrics\n      protocol: TCP\n"
  },
  {
    "path": "manager/refresh.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -e\n\nCORTEX_VERSION_MINOR=master\n\ncluster_config_out_path=\"$1\"\nmkdir -p \"$(dirname \"$cluster_config_out_path\")\"\n\nif ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION >/dev/null 2>&1; then\n  echo \"error: there is no cluster named \\\"$CORTEX_CLUSTER_NAME\\\" in $CORTEX_REGION; please update your configuration to point to an existing cortex cluster or create a cortex cluster with \\`cortex cluster up\\`\"\n  exit 1\nfi\n\neksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --verbose=0 | (grep -v \"saved kubeconfig as\" || true)\nout=$(kubectl get pods 2>&1 || true); if [[ \"$out\" == *\"must be logged in to the server\"* ]]; then echo \"error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortexlabs.com/v/${CORTEX_VERSION_MINOR}/\"; exit 1; fi\n\nkubectl get -n=default configmap cluster-config -o json | jq -r '.data.\"cluster.yaml\"' >> $cluster_config_out_path\n"
  },
  {
    "path": "manager/render_template.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport yaml\nimport os\nimport pathlib\nfrom jinja2 import Environment, FileSystemLoader\n\n# python render_template.py [CLUSTER_CONFIG_PATH] TEMPLATE_PATH\nif __name__ == \"__main__\":\n    if len(sys.argv) == 3:\n        yaml_file_path = sys.argv[1]\n        template_path = pathlib.Path(sys.argv[2])\n    elif len(sys.argv) == 2:\n        yaml_file_path = None\n        template_path = pathlib.Path(sys.argv[1])\n    else:\n        raise RuntimeError(f\"incorrect number of parameters ({len(sys.argv)})\")\n\n    file_loader = FileSystemLoader(str(template_path.parent))\n    env = Environment(loader=file_loader)\n    env.trim_blocks = True\n    env.lstrip_blocks = True\n    env.rstrip_blocks = True\n\n    template = env.get_template(str(template_path.name))\n\n    if yaml_file_path:\n        with open(yaml_file_path, \"r\") as f:\n            yaml_data = yaml.safe_load(f)\n            print(template.render(config=yaml_data, env=os.environ))\n    else:\n        print(template.render(env=os.environ))\n"
  },
  {
    "path": "manager/requirements.txt",
    "content": "boto3\njinja2\npyyaml\nyq\nclick==8.0.4\n"
  },
  {
    "path": "manager/uninstall.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -e\n\nEKSCTL_TIMEOUT=45m\n\nfunction main() {\n  echo\n  aws eks --region $CORTEX_REGION update-kubeconfig --name $CORTEX_CLUSTER_NAME >/dev/null\n  eksctl delete cluster --wait --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --disable-nodegroup-eviction --timeout=$EKSCTL_TIMEOUT\n  echo -e \"\\n✓ done spinning down the cluster\"\n}\n\nmain\n"
  },
  {
    "path": "manager/upgrade_kube_proxy_mode.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Usage: python create_user.py $KUBE_PROXY_CONFIG.yaml\n\nimport yaml\nimport sys\n\n\ndef main():\n    kube_proxy_config_file = sys.argv[1]\n    with open(kube_proxy_config_file, \"r\") as f:\n        kube_proxy_config = yaml.safe_load(f)\n\n    kube_proxy_config[\"mode\"] = \"ipvs\"  # IP Virtual Server\n    kube_proxy_config[\"ipvs\"][\"scheduler\"] = \"rr\"  # round robin\n\n    print(yaml.dump(kube_proxy_config, indent=2))\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "pkg/activator/activator.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage activator\n\nimport (\n\t\"context\"\n\t\"sync\"\n\n\t\"github.com/cortexlabs/cortex/pkg/autoscaler\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\t\"go.uber.org/zap\"\n\tistionetworkingclient \"istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/tools/cache\"\n)\n\ntype ctxValue string\n\nconst APINameCtxKey ctxValue = \"apiName\"\n\ntype StatsReporter interface {\n\tAddAPI(apiName string)\n\tRemoveAPI(apiName string)\n}\n\ntype Activator interface {\n\tTry(ctx context.Context, fn func() error) error\n}\n\ntype activator struct {\n\tactivatorsMux     sync.Mutex\n\ttrackersMux       sync.Mutex\n\tautoscalerClient  autoscaler.Client\n\tapiActivators     map[string]*apiActivator\n\treadinessTrackers map[string]*readinessTracker\n\tistioClient       istionetworkingclient.VirtualServiceInterface\n\treporter          StatsReporter\n\tlogger            *zap.SugaredLogger\n}\n\nfunc New(\n\tistioClient istionetworkingclient.VirtualServiceInterface,\n\tdeploymentInformer cache.SharedIndexInformer,\n\tvirtualServiceInformer cache.SharedIndexInformer,\n\tautoscalerClient autoscaler.Client,\n\treporter StatsReporter,\n\tlogger *zap.SugaredLogger,\n) Activator {\n\tlog := logger.With(zap.String(\"apiKind\", userconfig.RealtimeAPIKind.String()))\n\n\tact := &activator{\n\t\tapiActivators:     make(map[string]*apiActivator),\n\t\treadinessTrackers: make(map[string]*readinessTracker),\n\t\tistioClient:       istioClient,\n\t\tlogger:            log,\n\t\tautoscalerClient:  autoscalerClient,\n\t\treporter:          reporter,\n\t}\n\n\tvirtualServiceInformer.AddEventHandler(\n\t\tcache.ResourceEventHandlerFuncs{\n\t\t\tAddFunc:    act.addAPI,\n\t\t\tUpdateFunc: act.updateAPI,\n\t\t\tDeleteFunc: act.removeAPI,\n\t\t},\n\t)\n\n\tdeploymentInformer.AddEventHandler(\n\t\tcache.ResourceEventHandlerFuncs{\n\t\t\tAddFunc: act.updateReadinessTracker,\n\t\t\tUpdateFunc: func(_, newObj interface{}) {\n\t\t\t\tact.updateReadinessTracker(newObj)\n\t\t\t},\n\t\t\tDeleteFunc: act.removeReadinessTracker,\n\t\t},\n\t)\n\n\treturn act\n}\n\nfunc (a *activator) Try(ctx context.Context, fn func() error) error {\n\tapiNameValue := ctx.Value(APINameCtxKey)\n\tapiName, ok := apiNameValue.(string)\n\tif !ok || apiName == \"\" {\n\t\treturn errors.ErrorUnexpected(\"failed to get the api name from context\")\n\t}\n\n\tact, err := a.getOrCreateAPIActivator(ctx, apiName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttracker := a.getOrCreateReadinessTracker(apiName)\n\n\tif act.inFlight() == 0 {\n\t\tgo a.awakenAPI(apiName)\n\t}\n\n\treturn act.try(ctx, fn, tracker)\n}\n\nfunc (a *activator) getOrCreateAPIActivator(ctx context.Context, apiName string) (*apiActivator, error) {\n\ta.activatorsMux.Lock()\n\tdefer a.activatorsMux.Unlock()\n\n\tact, ok := a.apiActivators[apiName]\n\tif ok {\n\t\treturn act, nil\n\t}\n\n\tvs, err := a.istioClient.Get(ctx, workloads.K8sName(apiName), kmeta.GetOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tmaxQueueLength, maxConcurrency, err := userconfig.ConcurrencyFromAnnotations(vs)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tapiAct := newAPIActivator(maxQueueLength, maxConcurrency)\n\n\ta.apiActivators[apiName] = apiAct\n\n\treturn apiAct, nil\n}\n\nfunc (a *activator) getOrCreateReadinessTracker(apiName string) *readinessTracker {\n\ta.trackersMux.Lock()\n\tdefer a.trackersMux.Unlock()\n\ttracker, ok := a.readinessTrackers[apiName]\n\tif ok {\n\t\treturn tracker\n\t}\n\n\ttracker = newReadinessTracker()\n\ta.readinessTrackers[apiName] = tracker\n\n\treturn tracker\n}\n\nfunc (a *activator) addAPI(obj interface{}) {\n\tapiMetadata, err := getAPIMeta(obj)\n\tif err != nil {\n\t\ta.logger.Errorw(\"error during virtual service informer add callback\", zap.Error(err))\n\t\ttelemetry.Error(err)\n\t\treturn\n\t}\n\n\tif apiMetadata.apiKind != userconfig.RealtimeAPIKind {\n\t\treturn\n\t}\n\n\tapiName := apiMetadata.apiName\n\n\ta.activatorsMux.Lock()\n\tif a.apiActivators[apiName] == nil {\n\t\ta.logger.Debugw(\"adding new api activator\", zap.String(\"apiName\", apiName))\n\t\ta.apiActivators[apiName] = newAPIActivator(apiMetadata.maxQueueLength, apiMetadata.maxConcurrency)\n\t}\n\ta.activatorsMux.Unlock()\n\n\ta.reporter.AddAPI(apiName)\n}\n\nfunc (a *activator) updateAPI(oldObj interface{}, newObj interface{}) {\n\tapiMetadata, err := getAPIMeta(newObj)\n\tif err != nil {\n\t\ta.logger.Errorw(\"error during virtual service informer update callback\", zap.Error(err))\n\t\ttelemetry.Error(err)\n\t\treturn\n\t}\n\n\tif apiMetadata.apiKind != userconfig.RealtimeAPIKind {\n\t\treturn\n\t}\n\n\tapiName := apiMetadata.apiName\n\n\toldAPIMetatada, err := getAPIMeta(oldObj)\n\tif err != nil {\n\t\ta.logger.Errorw(\"error during virtual service informer update callback\", zap.Error(err))\n\t\ttelemetry.Error(err)\n\t\treturn\n\t}\n\n\tif oldAPIMetatada.maxConcurrency != apiMetadata.maxConcurrency || oldAPIMetatada.maxQueueLength != apiMetadata.maxQueueLength {\n\t\ta.logger.Debugw(\"updating api activator\", zap.String(\"apiName\", apiName))\n\n\t\ta.activatorsMux.Lock()\n\t\ta.apiActivators[apiName].updateQueueParams(apiMetadata.maxQueueLength, apiMetadata.maxConcurrency)\n\t\ta.activatorsMux.Unlock()\n\t}\n}\n\nfunc (a *activator) removeAPI(obj interface{}) {\n\tapiMetadata, err := getAPIMeta(obj)\n\tif err != nil {\n\t\ta.logger.Errorw(\"error during virtual service informer delete callback\", zap.Error(err))\n\t\ttelemetry.Error(err)\n\t\treturn\n\t}\n\n\tif apiMetadata.apiKind != userconfig.RealtimeAPIKind {\n\t\treturn\n\t}\n\n\ta.logger.Debugw(\"deleting api activator\", zap.String(\"apiName\", apiMetadata.apiName))\n\n\ta.activatorsMux.Lock()\n\tdelete(a.apiActivators, apiMetadata.apiName)\n\ta.activatorsMux.Unlock()\n\n\ta.reporter.RemoveAPI(apiMetadata.apiName)\n}\n\nfunc (a *activator) awakenAPI(apiName string) {\n\terr := a.autoscalerClient.Awaken(\n\t\tuserconfig.Resource{\n\t\t\tName: apiName,\n\t\t\tKind: userconfig.RealtimeAPIKind, // only realtime apis are supported as of now, so we can assume the kind\n\t\t},\n\t)\n\tif err != nil {\n\t\ta.logger.Errorw(\"failed to awake api\", zap.Error(err), zap.String(\"apiName\", apiName))\n\t}\n}\n\nfunc (a *activator) updateReadinessTracker(obj interface{}) {\n\tdeployment, ok := obj.(*kapps.Deployment)\n\tif !ok {\n\t\treturn\n\t}\n\n\tapi, err := getAPIMeta(obj)\n\tif err != nil {\n\t\ta.logger.Errorw(\"error during deployment informer callback\", zap.Error(err))\n\t\ttelemetry.Error(err)\n\t\treturn\n\t}\n\n\tif api.apiKind != userconfig.RealtimeAPIKind {\n\t\treturn\n\t}\n\n\ttracker := a.getOrCreateReadinessTracker(api.apiName)\n\ttracker.Update(deployment)\n\n\ta.logger.Debugw(\"updated readiness tracker\",\n\t\tzap.Bool(\"ready\", deployment.Status.ReadyReplicas > 0),\n\t\tzap.String(\"apiName\", api.apiName),\n\t)\n}\n\nfunc (a *activator) removeReadinessTracker(obj interface{}) {\n\tapi, err := getAPIMeta(obj)\n\tif err != nil {\n\t\ta.logger.Errorw(\"error during deployment informer callback\", zap.Error(err))\n\t\ttelemetry.Error(err)\n\t\treturn\n\t}\n\n\tif api.apiKind != userconfig.RealtimeAPIKind {\n\t\treturn\n\t}\n\n\ta.trackersMux.Lock()\n\tdefer a.trackersMux.Unlock()\n\tdelete(a.readinessTrackers, api.apiName)\n}\n"
  },
  {
    "path": "pkg/activator/activator_test.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage activator\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/proxy\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/stretchr/testify/require\"\n\t\"go.uber.org/zap\"\n)\n\ntype autoscalerClientMock struct{}\n\nfunc (m autoscalerClientMock) AddAPI(_ userconfig.Resource) error {\n\treturn nil\n}\n\nfunc (m autoscalerClientMock) Awaken(_ userconfig.Resource) error {\n\treturn nil\n}\n\nfunc newLogger(t *testing.T) *zap.SugaredLogger {\n\tt.Helper()\n\n\tconfig := zap.NewDevelopmentConfig()\n\tconfig.Level = zap.NewAtomicLevelAt(zap.ErrorLevel)\n\tlogger, err := config.Build()\n\trequire.NoError(t, err)\n\n\tlogr := logger.Sugar()\n\n\treturn logr\n}\n\nfunc TestActivator_Try(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\n\tapiName := \"test\"\n\tact := &activator{\n\t\tautoscalerClient: autoscalerClientMock{},\n\t\tapiActivators: map[string]*apiActivator{\n\t\t\tapiName: newAPIActivator(1, 1),\n\t\t},\n\t\treadinessTrackers: map[string]*readinessTracker{\n\t\t\tapiName: {ready: true},\n\t\t},\n\t\tlogger: log,\n\t}\n\n\tctx := context.Background()\n\tctx = context.WithValue(ctx, APINameCtxKey, apiName)\n\n\terrCh := make(chan error)\n\twaitCh := make(chan struct{})\n\n\tfor i := 0; i < 3; i++ {\n\t\tgo func() {\n\t\t\tctx, cancel := context.WithTimeout(ctx, time.Second)\n\t\t\tdefer cancel()\n\n\t\t\terrCh <- act.Try(ctx, func() error {\n\t\t\t\t<-waitCh\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}()\n\t}\n\n\terr := <-errCh\n\trequire.Error(t, err)\n\trequire.True(t, errors.Is(err, proxy.ErrRequestQueueFull))\n\n\tfor i := 0; i < 2; i++ {\n\t\twaitCh <- struct{}{}\n\t\trequire.NoError(t, <-errCh)\n\t}\n}\n"
  },
  {
    "path": "pkg/activator/api_activator.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage activator\n\nimport (\n\t\"context\"\n\t\"sync\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/proxy\"\n\tkapps \"k8s.io/api/apps/v1\"\n)\n\ntype apiActivator struct {\n\tbreaker *proxy.Breaker\n}\n\nfunc newAPIActivator(maxQueueLength, maxConcurrency int) *apiActivator {\n\tbreaker := proxy.NewBreaker(proxy.BreakerParams{\n\t\tQueueDepth:      maxQueueLength,\n\t\tMaxConcurrency:  maxConcurrency,\n\t\tInitialCapacity: maxConcurrency,\n\t})\n\n\treturn &apiActivator{breaker: breaker}\n}\n\n// try waits for the readinessTracker to be ready and then attempts to execute the passed callback.\n// If the readinessTracker does not reach a ready state, it will timeout.\nfunc (a *apiActivator) try(ctx context.Context, fn func() error, tracker *readinessTracker) error {\n\tvar execErr error\n\n\tif err := a.breaker.Maybe(ctx, func() {\n\t\tctx, cancel := context.WithTimeout(ctx, consts.WaitForReadyReplicasTimeout)\n\t\tdefer cancel()\n\n\t\tif !tracker.IsReady() {\n\t\tloop:\n\t\t\tfor {\n\t\t\t\tselect {\n\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\texecErr = errors.Wrap(ctx.Err(), \"no ready replicas available\")\n\t\t\t\t\treturn\n\t\t\t\tcase <-tracker.Wait():\n\t\t\t\t\tbreak loop\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\texecErr = fn()\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\treturn execErr\n}\n\n// updateQueueParams updates the breaker queue parameters (not thread safe)\nfunc (a *apiActivator) updateQueueParams(maxQueueLength, maxConcurrency int) {\n\ta.breaker.UpdateConcurrency(maxConcurrency)\n\ta.breaker.UpdateQueueLength(maxQueueLength)\n}\n\n// inFlight returns the amount of in-flight requests of the breaker\nfunc (a *apiActivator) inFlight() int64 {\n\treturn a.breaker.InFlight()\n}\n\ntype readinessTracker struct {\n\tmux   sync.RWMutex\n\tc     chan bool\n\tready bool\n}\n\nfunc newReadinessTracker() *readinessTracker {\n\treturn &readinessTracker{\n\t\tc: make(chan bool),\n\t}\n}\n\nfunc (t *readinessTracker) Update(deployment *kapps.Deployment) {\n\tt.mux.Lock()\n\tdefer t.mux.Unlock()\n\n\t// if changing from unready to ready state, wake up waiting goroutines\n\tif !t.ready && deployment.Status.ReadyReplicas > 0 {\n\t\tclose(t.c)            // wake up all of the goroutines waiting on this channel\n\t\tt.c = make(chan bool) // make a new channel ready to be used\n\t}\n\tt.ready = deployment.Status.ReadyReplicas > 0\n}\n\nfunc (t *readinessTracker) IsReady() bool {\n\tt.mux.RLock()\n\tdefer t.mux.RUnlock()\n\treturn t.ready\n}\n\nfunc (t *readinessTracker) Wait() chan bool {\n\tt.mux.RLock()\n\tdefer t.mux.RUnlock()\n\treturn t.c\n}\n"
  },
  {
    "path": "pkg/activator/api_activator_test.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage activator\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/proxy\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestApiActivator_Try(t *testing.T) {\n\tt.Parallel()\n\n\tact := newAPIActivator(1, 1)\n\n\terrCh := make(chan error)\n\twaitCh := make(chan struct{})\n\n\tfor i := 0; i < 3; i++ {\n\n\t\tgo func() {\n\t\t\terrCh <- act.try(context.Background(), func() error {\n\t\t\t\t<-waitCh\n\t\t\t\treturn nil\n\t\t\t}, &readinessTracker{ready: true})\n\t\t}()\n\t}\n\n\terr := <-errCh\n\trequire.Error(t, err)\n\trequire.True(t, errors.Is(err, proxy.ErrRequestQueueFull))\n\n\tfor i := 0; i < 2; i++ {\n\t\twaitCh <- struct{}{}\n\t\trequire.NoError(t, <-errCh)\n\t}\n}\n"
  },
  {
    "path": "pkg/activator/handler.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage activator\n\nimport (\n\t\"context\"\n\tstderrors \"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"net/url\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/proxy\"\n\t\"go.uber.org/zap\"\n)\n\ntype Handler struct {\n\tactivator Activator\n\tlogger    *zap.SugaredLogger\n}\n\nfunc NewHandler(act Activator, logger *zap.SugaredLogger) *Handler {\n\treturn &Handler{\n\t\tactivator: act,\n\t\tlogger:    logger,\n\t}\n}\n\nfunc (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tif hasCortexProbeHeader(r) {\n\t\tw.WriteHeader(http.StatusOK)\n\t\treturn\n\t}\n\n\tapiName := r.Header.Get(consts.CortexAPINameHeader)\n\tif apiName == \"\" {\n\t\thttp.Error(w, fmt.Sprintf(\"missing %s\", consts.CortexAPINameHeader), http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tctx := r.Context()\n\tctx = context.WithValue(ctx, APINameCtxKey, apiName)\n\n\tif err := h.activator.Try(ctx, func() error {\n\t\treturn h.proxyRequest(w, r)\n\t}); err != nil {\n\t\th.logger.Errorw(\"activator try error\", zap.Error(err))\n\n\t\tif stderrors.Is(err, context.DeadlineExceeded) || stderrors.Is(err, proxy.ErrRequestQueueFull) {\n\t\t\thttp.Error(w, err.Error(), http.StatusServiceUnavailable)\n\t\t} else {\n\t\t\tw.WriteHeader(http.StatusInternalServerError)\n\t\t\ttelemetry.Error(err)\n\t\t}\n\t}\n}\n\nfunc (h *Handler) proxyRequest(w http.ResponseWriter, r *http.Request) error {\n\ttarget := r.Header.Get(consts.CortexTargetServiceHeader)\n\tif target == \"\" {\n\t\treturn errors.ErrorUnexpected(\"missing header\", consts.CortexTargetServiceHeader)\n\t}\n\n\ttargetURL, err := url.Parse(target)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\t// delete activator specific headers\n\tr.Header.Del(consts.CortexAPINameHeader)\n\tr.Header.Del(consts.CortexTargetServiceHeader)\n\n\treverseProxy := httputil.NewSingleHostReverseProxy(targetURL)\n\treverseProxy.ServeHTTP(w, r)\n\n\treturn nil\n}\n\nfunc hasCortexProbeHeader(r *http.Request) bool {\n\treturn r.Header.Get(consts.CortexProbeHeader) != \"\"\n}\n"
  },
  {
    "path": "pkg/activator/handler_test.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage activator\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestActivatorHandler_ServeHTTP(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\n\tapiName := \"test\"\n\tact := &activator{\n\t\tautoscalerClient: autoscalerClientMock{},\n\t\tapiActivators: map[string]*apiActivator{\n\t\t\tapiName: newAPIActivator(1, 1),\n\t\t},\n\t\treadinessTrackers: map[string]*readinessTracker{\n\t\t\tapiName: {ready: true},\n\t\t},\n\t\tlogger: log,\n\t}\n\n\tah := NewHandler(act, log)\n\n\tvar callCount int\n\tserver := httptest.NewServer(\n\t\thttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\tcallCount++\n\t\t\tw.WriteHeader(http.StatusOK)\n\t\t}),\n\t)\n\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"POST\", \"http://fake.cortex.dev/api\", nil)\n\tr.Header.Set(consts.CortexAPINameHeader, apiName)\n\tr.Header.Set(consts.CortexTargetServiceHeader, server.URL)\n\n\tah.ServeHTTP(w, r)\n\n\trequire.Equal(t, http.StatusOK, w.Code)\n\trequire.Equal(t, 1, callCount)\n}\n"
  },
  {
    "path": "pkg/activator/helpers.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage activator\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"k8s.io/apimachinery/pkg/api/meta\"\n)\n\ntype apiMeta struct {\n\tapiName        string\n\tapiKind        userconfig.Kind\n\tlabels         map[string]string\n\tannotations    map[string]string\n\tmaxConcurrency int\n\tmaxQueueLength int\n}\n\nfunc getAPIMeta(obj interface{}) (apiMeta, error) {\n\tresource, err := meta.Accessor(obj)\n\tif err != nil {\n\t\treturn apiMeta{}, err\n\t}\n\n\tlabels := resource.GetLabels()\n\tapiKind, ok := labels[\"apiKind\"]\n\tif !ok {\n\t\treturn apiMeta{}, err\n\t}\n\n\tapiName, ok := labels[\"apiName\"]\n\tif !ok {\n\t\treturn apiMeta{}, errors.ErrorUnexpected(\"got a virtual service without apiName label\")\n\t}\n\n\tmaxQueueLength, maxConcurrency, err := userconfig.ConcurrencyFromAnnotations(resource)\n\tif err != nil {\n\t\treturn apiMeta{}, err\n\t}\n\n\treturn apiMeta{\n\t\tapiName:        apiName,\n\t\tapiKind:        userconfig.KindFromString(apiKind),\n\t\tlabels:         labels,\n\t\tannotations:    resource.GetAnnotations(),\n\t\tmaxConcurrency: maxConcurrency,\n\t\tmaxQueueLength: maxQueueLength,\n\t}, nil\n}\n"
  },
  {
    "path": "pkg/activator/request_stats.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage activator\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promauto\"\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n)\n\ntype PrometheusStatsReporter struct {\n\thandler          http.Handler\n\tinFlightRequests *prometheus.GaugeVec\n}\n\nfunc NewPrometheusStatsReporter() *PrometheusStatsReporter {\n\tinFlightRequestsGauge := promauto.NewGaugeVec(prometheus.GaugeOpts{\n\t\tName: \"cortex_in_flight_requests\",\n\t\tHelp: \"The number of in-flight requests for a cortex API\",\n\t}, []string{\"api_name\"})\n\n\treturn &PrometheusStatsReporter{\n\t\thandler:          promhttp.Handler(),\n\t\tinFlightRequests: inFlightRequestsGauge,\n\t}\n}\n\nfunc (r *PrometheusStatsReporter) AddAPI(apiName string) {\n\tr.inFlightRequests.WithLabelValues(apiName).Set(0)\n}\n\nfunc (r *PrometheusStatsReporter) RemoveAPI(apiName string) {\n\tr.inFlightRequests.DeleteLabelValues(apiName)\n}\n\nfunc (r *PrometheusStatsReporter) ServeHTTP(w http.ResponseWriter, req *http.Request) {\n\tr.handler.ServeHTTP(w, req)\n}\n"
  },
  {
    "path": "pkg/async-gateway/endpoint.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage gateway\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/async\"\n\t\"github.com/gorilla/mux\"\n\t\"go.uber.org/zap\"\n)\n\n// Endpoint wraps an async-gateway Service with HTTP logic\ntype Endpoint struct {\n\tservice Service\n\tlogger  *zap.SugaredLogger\n}\n\n// NewEndpoint creates and initializes a new Endpoint struct\nfunc NewEndpoint(svc Service, logger *zap.SugaredLogger) *Endpoint {\n\treturn &Endpoint{\n\t\tservice: svc,\n\t\tlogger:  logger,\n\t}\n}\n\n// CreateWorkload is a handler for the async-gateway service workload creation route\nfunc (e *Endpoint) CreateWorkload(w http.ResponseWriter, r *http.Request) {\n\trequestID := r.Header.Get(\"x-request-id\")\n\tif requestID == \"\" {\n\t\trespondPlainText(w, http.StatusBadRequest, \"error: missing x-request-id key in request header\")\n\t\treturn\n\t}\n\n\tapiName := r.Header.Get(consts.CortexAPINameHeader)\n\tif requestID == \"\" {\n\t\trespondPlainText(w, http.StatusBadRequest, fmt.Sprintf(\"error: missing %s key in request header\", consts.CortexAPINameHeader))\n\t\treturn\n\t}\n\tr.Header.Del(consts.CortexAPINameHeader)\n\n\tqueueURL := r.Header.Get(consts.CortexQueueURLHeader)\n\tif queueURL == \"\" {\n\t\trespondPlainText(w, http.StatusBadRequest, fmt.Sprintf(\"error: missing %s key in request header\", consts.CortexQueueURLHeader))\n\t\treturn\n\t}\n\tr.Header.Del(consts.CortexQueueURLHeader)\n\n\tbody := r.Body\n\tdefer func() {\n\t\t_ = r.Body.Close()\n\t}()\n\n\tlog := e.logger.With(zap.String(\"id\", requestID), zap.String(\"apiName\", apiName))\n\n\tid, err := e.service.CreateWorkload(requestID, apiName, queueURL, body, r.Header)\n\tif err != nil {\n\t\trespondPlainText(w, http.StatusInternalServerError, fmt.Sprintf(\"error: %v\", err))\n\t\tlogErrorWithTelemetry(log, errors.Wrap(err, \"failed to create workload\"))\n\t\treturn\n\t}\n\n\tif err = respondJSON(w, http.StatusOK, CreateWorkloadResponse{ID: id}); err != nil {\n\t\tlogErrorWithTelemetry(log, errors.Wrap(err, \"failed to encode json response\"))\n\t\treturn\n\t}\n}\n\n// GetWorkload is a handler for the async-gateway service workload retrieval route\nfunc (e *Endpoint) GetWorkload(w http.ResponseWriter, r *http.Request) {\n\tvars := mux.Vars(r)\n\tid, ok := vars[\"id\"]\n\tif !ok {\n\t\trespondPlainText(w, http.StatusBadRequest, \"error: missing request id in url path\")\n\t\treturn\n\t}\n\n\tapiName := r.Header.Get(consts.CortexAPINameHeader)\n\tif apiName == \"\" {\n\t\trespondPlainText(w, http.StatusBadRequest, fmt.Sprintf(\"error: missing %s key in request header\", consts.CortexAPINameHeader))\n\t\treturn\n\t}\n\tr.Header.Del(consts.CortexAPINameHeader)\n\n\tlog := e.logger.With(zap.String(\"id\", id), zap.String(\"apiName\", apiName))\n\n\tres, err := e.service.GetWorkload(id, apiName)\n\tif err != nil {\n\t\trespondPlainText(w, http.StatusInternalServerError, fmt.Sprintf(\"error: %v\", err))\n\t\tlogErrorWithTelemetry(log, errors.Wrap(err, \"failed to get workload\"))\n\t\treturn\n\t}\n\tif res.Status == async.StatusNotFound {\n\t\trespondPlainText(w, http.StatusNotFound, fmt.Sprintf(\"error: id %s not found\", res.ID))\n\t\tlogErrorWithTelemetry(log, errors.ErrorUnexpected(fmt.Sprintf(\"error: id %s not found\", res.ID)))\n\t\treturn\n\t}\n\n\tif err = respondJSON(w, http.StatusOK, res); err != nil {\n\t\tlogErrorWithTelemetry(log, errors.Wrap(err, \"failed to encode json response\"))\n\t\treturn\n\t}\n}\n\nfunc respondPlainText(w http.ResponseWriter, statusCode int, message string) {\n\tw.Header().Set(\"Content-Type\", \"text/plain\")\n\tw.WriteHeader(statusCode)\n\t_, _ = w.Write([]byte(message))\n}\n\nfunc respondJSON(w http.ResponseWriter, statusCode int, s interface{}) error {\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\tw.WriteHeader(statusCode)\n\treturn json.NewEncoder(w).Encode(s)\n}\n\nfunc logErrorWithTelemetry(log *zap.SugaredLogger, err error) {\n\ttelemetry.Error(err)\n\tlog.Error(err)\n}\n"
  },
  {
    "path": "pkg/async-gateway/queue.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage gateway\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\tawssqs \"github.com/aws/aws-sdk-go/service/sqs\"\n)\n\n// Queue is an interface to abstract communication with event queues\ntype Queue interface {\n\tSendMessage(message string, uniqueID string) error\n}\n\ntype sqs struct {\n\tqueueURL string\n\tclient   *awssqs.SQS\n}\n\n// NewSQS creates a new SQS client that satisfies the Queue interface\nfunc NewSQS(queueURL string, sess *session.Session) Queue {\n\tclient := awssqs.New(sess)\n\n\treturn &sqs{queueURL: queueURL, client: client}\n}\n\n// SendMessage sends a string\nfunc (q *sqs) SendMessage(message string, uniqueID string) error {\n\t_, err := q.client.SendMessage(&awssqs.SendMessageInput{\n\t\tMessageBody:            aws.String(message),\n\t\tMessageDeduplicationId: aws.String(uniqueID),\n\t\tMessageGroupId:         aws.String(uniqueID),\n\t\tQueueUrl:               aws.String(q.queueURL),\n\t})\n\treturn err\n}\n"
  },
  {
    "path": "pkg/async-gateway/service.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage gateway\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/types/async\"\n\t\"go.uber.org/zap\"\n)\n\n// Service provides an interface to the async-gateway business logic\ntype Service interface {\n\tCreateWorkload(id string, apiName string, queueURL string, payload io.Reader, headers http.Header) (string, error)\n\tGetWorkload(id string, apiName string) (GetWorkloadResponse, error)\n}\n\ntype service struct {\n\tlogger     *zap.SugaredLogger\n\tstorage    Storage\n\tclusterUID string\n\tsession    session.Session\n}\n\n// NewService creates a new async-gateway service\nfunc NewService(clusterUID string, storage Storage, logger *zap.SugaredLogger, session session.Session) Service {\n\treturn &service{\n\t\tlogger:     logger,\n\t\tstorage:    storage,\n\t\tclusterUID: clusterUID,\n\t\tsession:    session,\n\t}\n}\n\n// CreateWorkload enqueues an async workload request and uploads the request payload to S3\nfunc (s *service) CreateWorkload(id string, apiName string, queueURL string, payload io.Reader, headers http.Header) (string, error) {\n\tprefix := async.StoragePath(s.clusterUID, apiName)\n\tlog := s.logger.With(zap.String(\"id\", id), zap.String(\"apiName\", apiName))\n\n\tbuf := &bytes.Buffer{}\n\tif err := json.NewEncoder(buf).Encode(headers); err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to dump headers\")\n\t}\n\n\theadersPath := async.HeadersPath(prefix, id)\n\tlog.Debugw(\"uploading headers\", zap.String(\"path\", headersPath))\n\tif err := s.storage.Upload(headersPath, buf, \"application/json\"); err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to upload headers\")\n\t}\n\n\tcontentType := headers.Get(\"Content-Type\")\n\tpayloadPath := async.PayloadPath(prefix, id)\n\tlog.Debugw(\"uploading payload\", zap.String(\"path\", payloadPath))\n\tif err := s.storage.Upload(payloadPath, payload, contentType); err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to upload payload\")\n\t}\n\n\tlog.Debug(\"sending message to queue\")\n\tqueue := NewSQS(queueURL, &s.session)\n\tif err := queue.SendMessage(id, id); err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to send message to queue\")\n\t}\n\n\tstatusPath := fmt.Sprintf(\"%s/%s/status/%s\", prefix, id, async.StatusInQueue)\n\tlog.Debug(fmt.Sprintf(\"setting status to %s\", async.StatusInQueue))\n\tif err := s.storage.Upload(statusPath, strings.NewReader(\"\"), \"text/plain\"); err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to upload workload status\")\n\t}\n\n\treturn id, nil\n}\n\n// GetWorkload retrieves the status and result, if available, of a given workload\nfunc (s *service) GetWorkload(id string, apiName string) (GetWorkloadResponse, error) {\n\tlog := s.logger.With(zap.String(\"id\", id), zap.String(\"apiName\", apiName))\n\n\tst, err := s.getStatus(id, apiName)\n\tif err != nil {\n\t\treturn GetWorkloadResponse{}, err\n\t}\n\n\tif st != async.StatusCompleted {\n\t\treturn GetWorkloadResponse{\n\t\t\tID:     id,\n\t\t\tStatus: st,\n\t\t}, nil\n\t}\n\n\t// attempt to download user result\n\tprefix := async.StoragePath(s.clusterUID, apiName)\n\tresultPath := async.ResultPath(prefix, id)\n\tlog.Debug(\"downloading user result\", zap.String(\"path\", resultPath))\n\tresultBuf, err := s.storage.Download(resultPath)\n\tif err != nil {\n\t\treturn GetWorkloadResponse{}, err\n\t}\n\n\tvar userResponse UserResponse\n\tif err = json.Unmarshal(resultBuf, &userResponse); err != nil {\n\t\treturn GetWorkloadResponse{}, err\n\t}\n\n\tlog.Debug(\"getting workload timestamp\")\n\ttimestamp, err := s.storage.GetLastModified(resultPath)\n\tif err != nil {\n\t\treturn GetWorkloadResponse{}, err\n\t}\n\n\treturn GetWorkloadResponse{\n\t\tID:        id,\n\t\tStatus:    st,\n\t\tResult:    &userResponse,\n\t\tTimestamp: &timestamp,\n\t}, nil\n}\n\nfunc (s *service) getStatus(id string, apiName string) (async.Status, error) {\n\tprefix := async.StoragePath(s.clusterUID, apiName)\n\tlog := s.logger.With(zap.String(\"id\", id))\n\n\t// download workload status\n\tstatusPrefixPath := async.StatusPrefixPath(prefix, id)\n\tlog.Debug(\"checking status\", zap.String(\"path\", statusPrefixPath))\n\tfiles, err := s.storage.List(statusPrefixPath)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif len(files) == 0 {\n\t\treturn async.StatusNotFound, nil\n\t}\n\n\t// determine request status\n\tst := async.StatusInQueue\n\tfor _, file := range files {\n\t\tfileStatus := async.Status(file)\n\t\tif !fileStatus.Valid() {\n\t\t\tst = fileStatus\n\t\t\treturn \"\", fmt.Errorf(\"invalid workload status: %s\", st)\n\t\t}\n\t\tif fileStatus == async.StatusInProgress {\n\t\t\tst = fileStatus\n\t\t}\n\t\tif fileStatus == async.StatusCompleted || fileStatus == async.StatusFailed {\n\t\t\tst = fileStatus\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn st, nil\n}\n"
  },
  {
    "path": "pkg/async-gateway/storage.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage gateway\n\nimport (\n\t\"io\"\n\t\"path\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\tawss3 \"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/aws/aws-sdk-go/service/s3/s3manager\"\n)\n\n// Storage is an interface that abstracts cloud storage uploading\ntype Storage interface {\n\tUpload(key string, payload io.Reader, contentType string) error\n\tDownload(key string) ([]byte, error)\n\tList(key string) ([]string, error)\n\tGetLastModified(key string) (time.Time, error)\n}\n\ntype s3 struct {\n\tuploader   *s3manager.Uploader\n\tdownloader *s3manager.Downloader\n\tclient     *awss3.S3\n\tbucket     string\n}\n\n// NewS3 creates a new S3 client that satisfies the Storage interface\nfunc NewS3(sess *session.Session, bucket string) Storage {\n\tuploader := s3manager.NewUploader(sess)\n\tdownloader := s3manager.NewDownloader(sess)\n\tclient := awss3.New(sess)\n\treturn &s3{\n\t\tuploader:   uploader,\n\t\tbucket:     bucket,\n\t\tdownloader: downloader,\n\t\tclient:     client,\n\t}\n}\n\n// Upload uploads binary data to S3\nfunc (s *s3) Upload(key string, payload io.Reader, contentType string) error {\n\t_, err := s.uploader.Upload(&s3manager.UploadInput{\n\t\tKey:         aws.String(key),\n\t\tBucket:      aws.String(s.bucket),\n\t\tContentType: aws.String(contentType),\n\t\tBody:        payload,\n\t})\n\treturn err\n}\n\n// Download downloads a file from S3 into memory\nfunc (s *s3) Download(key string) ([]byte, error) {\n\tbuff := &aws.WriteAtBuffer{}\n\tinput := awss3.GetObjectInput{\n\t\tKey:    aws.String(key),\n\t\tBucket: aws.String(s.bucket),\n\t}\n\n\t_, err := s.downloader.Download(buff, &input)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn buff.Bytes(), nil\n}\n\n// List lists a set of files from a given S3 path.\n// Works only for one level deep sub-paths.\nfunc (s *s3) List(key string) ([]string, error) {\n\tif key != \"\" && !strings.HasSuffix(key, \"/\") {\n\t\tkey += \"/\"\n\t}\n\n\tresult, err := s.client.ListObjectsV2(&awss3.ListObjectsV2Input{\n\t\tPrefix: aws.String(key),\n\t\tBucket: aws.String(s.bucket),\n\t})\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfiles := []string{}\n\tfor _, obj := range result.Contents {\n\t\t_, file := path.Split(*obj.Key)\n\t\tfiles = append(files, file)\n\t}\n\treturn files, nil\n}\n\n// GetLastModified retrieves the last modified timestamp of an S3 object\nfunc (s *s3) GetLastModified(key string) (time.Time, error) {\n\tinput := awss3.GetObjectInput{\n\t\tKey:    aws.String(key),\n\t\tBucket: aws.String(s.bucket),\n\t}\n\tobj, err := s.client.GetObject(&input)\n\tif err != nil {\n\t\treturn time.Time{}, err\n\t}\n\n\treturn *obj.LastModified, nil\n}\n"
  },
  {
    "path": "pkg/async-gateway/types.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage gateway\n\nimport (\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/types/async\"\n)\n\n// UserResponse represents the user's API response, which has to be JSON serializable\ntype UserResponse = map[string]interface{}\n\n//CreateWorkloadResponse represents the response returned to the user on workload creation\ntype CreateWorkloadResponse struct {\n\tID string `json:\"id\"`\n}\n\n// GetWorkloadResponse represents the workload response that is returned to the user\ntype GetWorkloadResponse struct {\n\tID        string        `json:\"id\"`\n\tStatus    async.Status  `json:\"status\"`\n\tResult    *UserResponse `json:\"result,omitempty\"`\n\tTimestamp *time.Time    `json:\"timestamp,omitempty\"`\n}\n"
  },
  {
    "path": "pkg/autoscaler/async_scaler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage autoscaler\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tpromv1 \"github.com/prometheus/client_golang/api/prometheus/v1\"\n\t\"github.com/prometheus/common/model\"\n)\n\ntype AsyncScaler struct {\n\tk8s        *k8s.Client\n\tprometheus promv1.API\n}\n\nfunc NewAsyncScaler(k8sClient *k8s.Client, promClient promv1.API) *AsyncScaler {\n\treturn &AsyncScaler{\n\t\tk8s:        k8sClient,\n\t\tprometheus: promClient,\n\t}\n}\n\nfunc (s *AsyncScaler) Scale(apiName string, request int32) error {\n\tdeployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif deployment == nil {\n\t\treturn errors.ErrorUnexpected(\"unable to find k8s deployment\", apiName)\n\t}\n\n\tif deployment.Spec.Replicas != nil && *deployment.Spec.Replicas == request {\n\t\treturn nil\n\t}\n\n\tdeployment.Spec.Replicas = pointer.Int32(request)\n\n\tif _, err = s.k8s.UpdateDeployment(deployment); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (s *AsyncScaler) GetInFlightRequests(apiName string, window time.Duration) (*float64, error) {\n\twindowSeconds := int64(window.Seconds())\n\n\t// PromQL query:\n\t// \tsum(sum_over_time(cortex_async_in_flight{api_name=\"<apiName>\"}[60s])) /\n\t//\tsum(count_over_time(cortex_async_in_flight{api_name=\"<apiName>\"}[60s]))\n\tquery := fmt.Sprintf(\n\t\t\"sum(sum_over_time(cortex_async_in_flight{api_name=\\\"%s\\\"}[%ds])) / \"+\n\t\t\t\"max(count_over_time(cortex_async_in_flight{api_name=\\\"%s\\\"}[%ds]))\",\n\t\tapiName, windowSeconds,\n\t\tapiName, windowSeconds,\n\t)\n\n\tctx, cancel := context.WithTimeout(context.Background(), _prometheusQueryTimeoutSeconds*time.Second)\n\tdefer cancel()\n\n\tvaluesQuery, _, err := s.prometheus.Query(ctx, query, time.Now())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvalues, ok := valuesQuery.(model.Vector)\n\tif !ok {\n\t\treturn nil, errors.ErrorUnexpected(\"failed to convert prometheus metric to vector\")\n\t}\n\n\tif values.Len() != 0 {\n\t\treturn pointer.Float64(float64(values[0].Value)), nil\n\t}\n\treturn nil, nil\n}\n\nfunc (s *AsyncScaler) GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error) {\n\tdeployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif deployment == nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to find k8s deployment\", apiName)\n\t}\n\n\tautoscalingSpec, err := userconfig.AutoscalingFromAnnotations(deployment)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn autoscalingSpec, nil\n}\n\nfunc (s *AsyncScaler) CurrentRequestedReplicas(apiName string) (int32, error) {\n\tdeployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif deployment == nil {\n\t\treturn 0, errors.ErrorUnexpected(\"unable to find k8s deployment\", apiName)\n\t}\n\n\tif deployment.Spec.Replicas == nil {\n\t\treturn 0, errors.ErrorUnexpected(\"k8s deployment doesn't have the replicas field set\", apiName)\n\t}\n\n\treturn *deployment.Spec.Replicas, nil\n}\n"
  },
  {
    "path": "pkg/autoscaler/autoscaler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage autoscaler\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/cron\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tlibmath \"github.com/cortexlabs/cortex/pkg/lib/math\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"go.uber.org/zap\"\n)\n\nconst (\n\t_prometheusQueryTimeoutSeconds = 10\n)\n\ntype Scaler interface {\n\tScale(apiName string, request int32) error\n\tGetInFlightRequests(apiName string, window time.Duration) (*float64, error)\n\tGetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error)\n\tCurrentRequestedReplicas(apiName string) (int32, error)\n}\n\ntype Autoscaler struct {\n\tlogger  *zap.SugaredLogger\n\tcrons   map[string]cron.Cron\n\tscalers map[userconfig.Kind]Scaler\n\trecs    map[string]*recommendations\n}\n\nfunc New(logger *zap.SugaredLogger) *Autoscaler {\n\treturn &Autoscaler{\n\t\tlogger:  logger,\n\t\tcrons:   make(map[string]cron.Cron),\n\t\tscalers: make(map[userconfig.Kind]Scaler),\n\t\trecs:    make(map[string]*recommendations),\n\t}\n}\n\nfunc (a *Autoscaler) AddScaler(scaler Scaler, kind userconfig.Kind) {\n\ta.scalers[kind] = scaler\n}\n\nfunc (a *Autoscaler) Awaken(api userconfig.Resource) error {\n\tscaler, ok := a.scalers[api.Kind]\n\tif !ok {\n\t\treturn errors.ErrorUnexpected(\n\t\t\tfmt.Sprintf(\"autoscaler does not have a scaler for the %s kind\", api.Kind),\n\t\t)\n\t}\n\n\tlog := a.logger.With(\n\t\tzap.String(\"apiName\", api.Name),\n\t\tzap.String(\"apiKind\", api.Kind.String()),\n\t)\n\n\tcurrentRequestedReplicas, err := scaler.CurrentRequestedReplicas(api.Name)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to get current replicas\")\n\t}\n\n\tif currentRequestedReplicas > 0 {\n\t\treturn nil\n\t}\n\n\tlog.Infof(\"autoscaling awake event\")\n\tif err := scaler.Scale(api.Name, 1); err != nil {\n\t\treturn errors.Wrap(err, \"failed to scale api to one\")\n\t}\n\n\ta.recs[api.Name].add(1)\n\n\treturn nil\n}\n\nfunc (a *Autoscaler) AddAPI(api userconfig.Resource) error {\n\tif _, ok := a.crons[api.Name]; ok {\n\t\treturn nil\n\t}\n\n\terrorHandler := func(err error) {\n\t\tlog := a.logger.With(\n\t\t\tzap.String(\"apiName\", api.Name),\n\t\t\tzap.String(\"apiKind\", api.Kind.String()),\n\t\t)\n\n\t\tlog.Error(err)\n\t\ttelemetry.Error(err)\n\t}\n\n\tautoscaleFn, err := a.autoscaleFn(api)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ta.crons[api.Name] = cron.Run(autoscaleFn, errorHandler, spec.AutoscalingTickInterval)\n\n\treturn nil\n}\n\nfunc (a *Autoscaler) RemoveAPI(api userconfig.Resource) {\n\tlog := a.logger.With(\n\t\tzap.String(\"apiName\", api.Name),\n\t\tzap.String(\"apiKind\", api.Kind.String()),\n\t)\n\n\tif autoscalerCron, ok := a.crons[api.Name]; ok {\n\t\tautoscalerCron.Cancel()\n\t\tdelete(a.crons, api.Name)\n\t}\n\n\tdelete(a.recs, api.Name)\n\tlog.Info(\"autoscaler stop\")\n}\n\nfunc (a *Autoscaler) Stop() {\n\tfor apiName, apiCron := range a.crons {\n\t\tapiCron.Cancel()\n\t\tdelete(a.crons, apiName)\n\t}\n}\n\nfunc (a *Autoscaler) autoscaleFn(api userconfig.Resource) (func() error, error) {\n\tlog := a.logger.With(\n\t\tzap.String(\"apiName\", api.Name),\n\t\tzap.String(\"apiKind\", api.Kind.String()),\n\t)\n\n\tscaler, ok := a.scalers[api.Kind]\n\tif !ok {\n\t\treturn nil, errors.ErrorUnexpected(\n\t\t\tfmt.Sprintf(\"autoscaler does not have a scaler for the %s kind\", api.Kind),\n\t\t)\n\t}\n\n\tlog.Info(\"autoscaler init\")\n\n\tvar startTime time.Time\n\ta.recs[api.Name] = newRecommendations()\n\n\treturn func() error {\n\t\tautoscalingSpec, err := scaler.GetAutoscalingSpec(api.Name)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to get autoscaling spec\")\n\t\t}\n\n\t\tcurrentRequestedReplicas, err := scaler.CurrentRequestedReplicas(api.Name)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to get current replicas\")\n\t\t}\n\n\t\tif startTime.IsZero() {\n\t\t\tstartTime = time.Now()\n\t\t}\n\n\t\tavgInFlight, err := scaler.GetInFlightRequests(api.Name, autoscalingSpec.Window)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to get in-flight requests\")\n\t\t}\n\t\tif avgInFlight == nil {\n\t\t\tlog.Debug(\"autoscaler tick: metrics not available yet\")\n\t\t\treturn nil\n\t\t}\n\n\t\trawRecommendation := *avgInFlight / *autoscalingSpec.TargetInFlight\n\t\trecommendation := int32(math.Ceil(rawRecommendation))\n\n\t\tif rawRecommendation < float64(currentRequestedReplicas) && rawRecommendation > float64(currentRequestedReplicas)*(1-autoscalingSpec.DownscaleTolerance) {\n\t\t\trecommendation = currentRequestedReplicas\n\t\t}\n\n\t\tif rawRecommendation > float64(currentRequestedReplicas) && rawRecommendation < float64(currentRequestedReplicas)*(1+autoscalingSpec.UpscaleTolerance) {\n\t\t\trecommendation = currentRequestedReplicas\n\t\t}\n\n\t\t// always allow subtraction of 1\n\t\tdownscaleFactorFloor := libmath.MinInt32(currentRequestedReplicas-1, int32(math.Ceil(float64(currentRequestedReplicas)*autoscalingSpec.MaxDownscaleFactor)))\n\t\tif recommendation < downscaleFactorFloor {\n\t\t\trecommendation = downscaleFactorFloor\n\t\t}\n\n\t\t// always allow addition of 1\n\t\tupscaleFactorCeil := libmath.MaxInt32(currentRequestedReplicas+1, int32(math.Ceil(float64(currentRequestedReplicas)*autoscalingSpec.MaxUpscaleFactor)))\n\t\tif recommendation > upscaleFactorCeil {\n\t\t\trecommendation = upscaleFactorCeil\n\t\t}\n\n\t\tif recommendation < autoscalingSpec.MinReplicas {\n\t\t\trecommendation = autoscalingSpec.MinReplicas\n\t\t}\n\n\t\tif recommendation > autoscalingSpec.MaxReplicas {\n\t\t\trecommendation = autoscalingSpec.MaxReplicas\n\t\t}\n\n\t\trecs := a.recs[api.Name]\n\n\t\t// Rule of thumb: any modifications that don't consider historical recommendations should be performed before\n\t\t// recording the recommendation, any modifications that use historical recommendations should be performed after\n\t\trecs.add(recommendation)\n\n\t\t// This is just for garbage collection\n\t\trecs.deleteOlderThan(libtime.MaxDuration(autoscalingSpec.DownscaleStabilizationPeriod, autoscalingSpec.UpscaleStabilizationPeriod))\n\n\t\trequest := recommendation\n\t\tvar downscaleStabilizationFloor *int32\n\t\tvar upscaleStabilizationCeil *int32\n\n\t\tif request < currentRequestedReplicas {\n\t\t\tdownscaleStabilizationFloor = recs.maxSince(autoscalingSpec.DownscaleStabilizationPeriod)\n\t\t\tif downscaleStabilizationFloor != nil {\n\t\t\t\tdownscaleStabilizationFloor = pointer.Int32(libmath.MinInt32(*downscaleStabilizationFloor, currentRequestedReplicas))\n\t\t\t}\n\t\t\tif time.Since(startTime) < autoscalingSpec.DownscaleStabilizationPeriod {\n\t\t\t\trequest = currentRequestedReplicas\n\t\t\t} else if downscaleStabilizationFloor != nil && request < *downscaleStabilizationFloor {\n\t\t\t\trequest = *downscaleStabilizationFloor\n\t\t\t}\n\t\t}\n\t\tif request > currentRequestedReplicas {\n\t\t\tupscaleStabilizationCeil = recs.minSince(autoscalingSpec.UpscaleStabilizationPeriod)\n\t\t\tif upscaleStabilizationCeil != nil {\n\t\t\t\tupscaleStabilizationCeil = pointer.Int32(libmath.MaxInt32(*upscaleStabilizationCeil, currentRequestedReplicas))\n\t\t\t}\n\t\t\tif time.Since(startTime) < autoscalingSpec.UpscaleStabilizationPeriod {\n\t\t\t\trequest = currentRequestedReplicas\n\t\t\t} else if upscaleStabilizationCeil != nil && request > *upscaleStabilizationCeil {\n\t\t\t\trequest = *upscaleStabilizationCeil\n\t\t\t}\n\t\t}\n\n\t\tlog.Debugw(\"autoscaler tick\",\n\t\t\t\"autoscaling\", map[string]interface{}{\n\t\t\t\t\"avg_in_flight\":                  *avgInFlight,\n\t\t\t\t\"target_in_flight\":               *autoscalingSpec.TargetInFlight,\n\t\t\t\t\"raw_recommendation\":             rawRecommendation,\n\t\t\t\t\"current_replicas\":               currentRequestedReplicas,\n\t\t\t\t\"downscale_tolerance\":            autoscalingSpec.DownscaleTolerance,\n\t\t\t\t\"upscale_tolerance\":              autoscalingSpec.UpscaleTolerance,\n\t\t\t\t\"max_downscale_factor\":           autoscalingSpec.MaxDownscaleFactor,\n\t\t\t\t\"downscale_factor_floor\":         downscaleFactorFloor,\n\t\t\t\t\"max_upscale_factor\":             autoscalingSpec.MaxUpscaleFactor,\n\t\t\t\t\"upscale_factor_ceil\":            upscaleFactorCeil,\n\t\t\t\t\"min_replicas\":                   autoscalingSpec.MinReplicas,\n\t\t\t\t\"max_replicas\":                   autoscalingSpec.MaxReplicas,\n\t\t\t\t\"recommendation\":                 recommendation,\n\t\t\t\t\"downscale_stabilization_period\": autoscalingSpec.DownscaleStabilizationPeriod.Seconds(),\n\t\t\t\t\"downscale_stabilization_floor\":  downscaleStabilizationFloor,\n\t\t\t\t\"upscale_stabilization_period\":   autoscalingSpec.UpscaleStabilizationPeriod.Seconds(),\n\t\t\t\t\"upscale_stabilization_ceil\":     upscaleStabilizationCeil,\n\t\t\t\t\"request\":                        request,\n\t\t\t},\n\t\t)\n\n\t\tif currentRequestedReplicas != request {\n\t\t\tlog.Infof(\"autoscaling event: %d -> %d\", currentRequestedReplicas, request)\n\t\t\tif err = scaler.Scale(api.Name, request); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\treturn nil\n\t}, nil\n}\n"
  },
  {
    "path": "pkg/autoscaler/autoscaler_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage autoscaler\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/cron\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/stretchr/testify/require\"\n\t\"go.uber.org/zap\"\n)\n\nfunc newLogger(t *testing.T) *zap.SugaredLogger {\n\tt.Helper()\n\n\tconfig := zap.NewDevelopmentConfig()\n\tconfig.Level = zap.NewAtomicLevelAt(zap.ErrorLevel)\n\tlogger, err := config.Build()\n\trequire.NoError(t, err)\n\n\tlogr := logger.Sugar()\n\n\treturn logr\n}\n\nfunc generateRecommendationTimeline(t *testing.T, recs []int32, interval time.Duration) *recommendations {\n\tt.Helper()\n\n\tstartTime := time.Now()\n\trecsTimeline := map[time.Time]int32{}\n\n\tfor i := range recs {\n\t\ttimestamp := startTime.Add(time.Duration(i) * interval)\n\t\trecsTimeline[timestamp] = recs[i]\n\t}\n\n\treturn &recommendations{\n\t\ttimeline: recsTimeline,\n\t}\n}\n\nfunc TestAutoscaler_AutoscaleFn(t *testing.T) {\n\tt.Parallel()\n\tlog := newLogger(t)\n\n\tinterval := 250 * time.Millisecond\n\n\tcases := []struct {\n\t\tname                   string\n\t\tautoscalingSpec        userconfig.Autoscaling\n\t\tinFlight               float64\n\t\tcurrentReplicas        int32\n\t\trecommendationTimeline []int32\n\t\texpectedRequest        *int32\n\t}{\n\t\t{\n\t\t\tname: \"no scale below zero within stabilization period\",\n\t\t\tautoscalingSpec: userconfig.Autoscaling{\n\t\t\t\tMinReplicas:                  0,\n\t\t\t\tMaxReplicas:                  5,\n\t\t\t\tInitReplicas:                 0,\n\t\t\t\tTargetInFlight:               pointer.Float64(1),\n\t\t\t\tWindow:                       4 * interval,\n\t\t\t\tDownscaleStabilizationPeriod: time.Second,\n\t\t\t\tUpscaleStabilizationPeriod:   time.Second,\n\t\t\t\tMaxDownscaleFactor:           0.75,\n\t\t\t\tMaxUpscaleFactor:             1.5,\n\t\t\t\tDownscaleTolerance:           0.05,\n\t\t\t\tUpscaleTolerance:             0.05,\n\t\t\t},\n\t\t\tinFlight:        0,\n\t\t\tcurrentReplicas: 1,\n\t\t\texpectedRequest: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"downscale no stabilization\",\n\t\t\tautoscalingSpec: userconfig.Autoscaling{\n\t\t\t\tMinReplicas:                  1,\n\t\t\t\tMaxReplicas:                  5,\n\t\t\t\tInitReplicas:                 1,\n\t\t\t\tTargetInFlight:               pointer.Float64(1),\n\t\t\t\tWindow:                       4 * interval,\n\t\t\t\tDownscaleStabilizationPeriod: 0,\n\t\t\t\tUpscaleStabilizationPeriod:   0,\n\t\t\t\tMaxDownscaleFactor:           0.5,\n\t\t\t\tMaxUpscaleFactor:             1.5,\n\t\t\t\tDownscaleTolerance:           0.05,\n\t\t\t\tUpscaleTolerance:             0.05,\n\t\t\t},\n\t\t\tinFlight:               2,\n\t\t\tcurrentReplicas:        5,\n\t\t\trecommendationTimeline: []int32{5, 2, 2, 2},\n\t\t\texpectedRequest:        pointer.Int32(3),\n\t\t},\n\t\t{\n\t\t\tname: \"downscale with stabilization\",\n\t\t\tautoscalingSpec: userconfig.Autoscaling{\n\t\t\t\tMinReplicas:                  1,\n\t\t\t\tMaxReplicas:                  5,\n\t\t\t\tInitReplicas:                 1,\n\t\t\t\tTargetInFlight:               pointer.Float64(1),\n\t\t\t\tWindow:                       4 * interval,\n\t\t\t\tDownscaleStabilizationPeriod: time.Second,\n\t\t\t\tUpscaleStabilizationPeriod:   time.Second,\n\t\t\t\tMaxDownscaleFactor:           0.5,\n\t\t\t\tMaxUpscaleFactor:             1.5,\n\t\t\t\tDownscaleTolerance:           0.05,\n\t\t\t\tUpscaleTolerance:             0.05,\n\t\t\t},\n\t\t\tinFlight:               5,\n\t\t\tcurrentReplicas:        1,\n\t\t\trecommendationTimeline: []int32{5, 5, 2, 2},\n\t\t\texpectedRequest:        nil,\n\t\t},\n\t\t{\n\t\t\tname: \"upscale no stabilization\",\n\t\t\tautoscalingSpec: userconfig.Autoscaling{\n\t\t\t\tMinReplicas:                  1,\n\t\t\t\tMaxReplicas:                  5,\n\t\t\t\tInitReplicas:                 1,\n\t\t\t\tTargetInFlight:               pointer.Float64(1),\n\t\t\t\tWindow:                       4 * interval,\n\t\t\t\tDownscaleStabilizationPeriod: 0,\n\t\t\t\tUpscaleStabilizationPeriod:   0,\n\t\t\t\tMaxDownscaleFactor:           0.5,\n\t\t\t\tMaxUpscaleFactor:             1.5,\n\t\t\t\tDownscaleTolerance:           0.05,\n\t\t\t\tUpscaleTolerance:             0.05,\n\t\t\t},\n\t\t\tinFlight:        3,\n\t\t\tcurrentReplicas: 1,\n\t\t\texpectedRequest: pointer.Int32(2),\n\t\t},\n\t\t{\n\t\t\tname: \"upscale with stabilization\",\n\t\t\tautoscalingSpec: userconfig.Autoscaling{\n\t\t\t\tMinReplicas:                  1,\n\t\t\t\tMaxReplicas:                  5,\n\t\t\t\tInitReplicas:                 1,\n\t\t\t\tTargetInFlight:               pointer.Float64(1),\n\t\t\t\tWindow:                       4 * interval,\n\t\t\t\tDownscaleStabilizationPeriod: time.Second,\n\t\t\t\tUpscaleStabilizationPeriod:   time.Second,\n\t\t\t\tMaxDownscaleFactor:           0.5,\n\t\t\t\tMaxUpscaleFactor:             1.5,\n\t\t\t\tDownscaleTolerance:           0.05,\n\t\t\t\tUpscaleTolerance:             0.05,\n\t\t\t},\n\t\t\tinFlight:               5,\n\t\t\tcurrentReplicas:        2,\n\t\t\trecommendationTimeline: []int32{2, 2, 2, 5},\n\t\t\texpectedRequest:        nil,\n\t\t},\n\t\t{\n\t\t\tname: \"no upscale below current replicas\",\n\t\t\tautoscalingSpec: userconfig.Autoscaling{\n\t\t\t\tMinReplicas:                  0,\n\t\t\t\tMaxReplicas:                  5,\n\t\t\t\tInitReplicas:                 0,\n\t\t\t\tTargetInFlight:               pointer.Float64(1),\n\t\t\t\tWindow:                       4 * interval,\n\t\t\t\tDownscaleStabilizationPeriod: time.Second,\n\t\t\t\tUpscaleStabilizationPeriod:   time.Second,\n\t\t\t\tMaxDownscaleFactor:           0.75,\n\t\t\t\tMaxUpscaleFactor:             1.5,\n\t\t\t\tDownscaleTolerance:           0.05,\n\t\t\t\tUpscaleTolerance:             0.05,\n\t\t\t},\n\t\t\tinFlight:               3,\n\t\t\tcurrentReplicas:        2,\n\t\t\trecommendationTimeline: []int32{0, 1, 2, 3},\n\t\t\texpectedRequest:        nil,\n\t\t},\n\t}\n\n\tfor i, tt := range cases {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tt.Parallel()\n\n\t\t\tvar latestRequest *int32\n\n\t\t\tscalerMock := &ScalerFunc{\n\t\t\t\tScaleFunc: func(apiName string, request int32) error {\n\t\t\t\t\tlatestRequest = pointer.Int32(request)\n\t\t\t\t\treturn nil\n\t\t\t\t},\n\t\t\t\tGetInFlightRequestsFunc: func(apiName string, window time.Duration) (*float64, error) {\n\t\t\t\t\treturn pointer.Float64(tt.inFlight), nil\n\t\t\t\t},\n\t\t\t\tGetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) {\n\t\t\t\t\treturn &cases[i].autoscalingSpec, nil\n\t\t\t\t},\n\t\t\t\tCurrentRequestedReplicasFunc: func(apiName string) (int32, error) {\n\t\t\t\t\treturn tt.currentReplicas, nil\n\t\t\t\t},\n\t\t\t}\n\n\t\t\tautoScaler := &Autoscaler{\n\t\t\t\tlogger:  log,\n\t\t\t\tcrons:   make(map[string]cron.Cron),\n\t\t\t\tscalers: make(map[userconfig.Kind]Scaler),\n\t\t\t\trecs:    make(map[string]*recommendations),\n\t\t\t}\n\t\t\tautoScaler.AddScaler(scalerMock, userconfig.RealtimeAPIKind)\n\n\t\t\tapiName := \"test\"\n\t\t\tapi := userconfig.Resource{\n\t\t\t\tName: apiName,\n\t\t\t\tKind: userconfig.RealtimeAPIKind,\n\t\t\t}\n\n\t\t\tautoScaler.recs[apiName] = generateRecommendationTimeline(t, tt.recommendationTimeline, interval)\n\t\t\tautoscaleFn, err := autoScaler.autoscaleFn(api)\n\t\t\trequire.NoError(t, err)\n\n\t\t\ttime.Sleep(interval)\n\n\t\t\terr = autoscaleFn()\n\t\t\trequire.NoError(t, err)\n\n\t\t\trequire.Equal(t, tt.expectedRequest, latestRequest)\n\t\t})\n\t}\n\n}\n\nfunc TestAutoscaler_Awake(t *testing.T) {\n\tt.Parallel()\n\tlog := newLogger(t)\n\n\tmux := sync.RWMutex{}\n\tvar latestRequest int32\n\n\tdownscaleStabilizationPeriod := 3 * time.Second\n\tscalerMock := &ScalerFunc{\n\t\tScaleFunc: func(apiName string, request int32) error {\n\t\t\tmux.Lock()\n\t\t\tdefer mux.Unlock()\n\n\t\t\tlatestRequest = request\n\t\t\treturn nil\n\t\t},\n\t\tGetInFlightRequestsFunc: func(apiName string, window time.Duration) (*float64, error) {\n\t\t\treturn pointer.Float64(0), nil\n\t\t},\n\t\tGetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) {\n\t\t\treturn &userconfig.Autoscaling{\n\t\t\t\tMinReplicas:                  0,\n\t\t\t\tMaxReplicas:                  1,\n\t\t\t\tInitReplicas:                 1,\n\t\t\t\tTargetInFlight:               pointer.Float64(1),\n\t\t\t\tWindow:                       500 * time.Millisecond,\n\t\t\t\tDownscaleStabilizationPeriod: downscaleStabilizationPeriod,\n\t\t\t\tMaxDownscaleFactor:           0.75,\n\t\t\t\tMaxUpscaleFactor:             1.5,\n\t\t\t}, nil\n\t\t},\n\t\tCurrentRequestedReplicasFunc: func(apiName string) (int32, error) {\n\t\t\treturn 0, nil\n\t\t},\n\t}\n\n\tautoScaler := &Autoscaler{\n\t\tlogger:  log,\n\t\tcrons:   make(map[string]cron.Cron),\n\t\tscalers: make(map[userconfig.Kind]Scaler),\n\t\trecs:    make(map[string]*recommendations),\n\t}\n\tautoScaler.AddScaler(scalerMock, userconfig.RealtimeAPIKind)\n\n\tapiName := \"test\"\n\tapi := userconfig.Resource{\n\t\tName: apiName,\n\t\tKind: userconfig.RealtimeAPIKind,\n\t}\n\n\tautoscaleFn, err := autoScaler.autoscaleFn(api)\n\trequire.NoError(t, err)\n\n\tticker := time.NewTicker(250 * time.Millisecond)\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ticker.C:\n\t\t\t\terr := autoscaleFn()\n\t\t\t\trequire.NoError(t, err)\n\t\t\t}\n\t\t}\n\t}()\n\n\terr = autoScaler.Awaken(api)\n\trequire.NoError(t, err)\n\n\trequire.Never(t, func() bool {\n\t\tmux.RLock()\n\t\tdefer mux.RUnlock()\n\t\treturn latestRequest != 1\n\t}, downscaleStabilizationPeriod, time.Second)\n}\n\nfunc TestAutoscaler_MinReplicas(t *testing.T) {\n\tt.Parallel()\n\tlog := newLogger(t)\n\n\tmux := sync.RWMutex{}\n\tvar latestRequest int32\n\n\tminReplicas := int32(5)\n\tmaxReplicas := int32(10)\n\n\tscalerMock := &ScalerFunc{\n\t\tScaleFunc: func(apiName string, request int32) error {\n\t\t\tmux.Lock()\n\t\t\tdefer mux.Unlock()\n\n\t\t\tlatestRequest = request\n\t\t\treturn nil\n\t\t},\n\t\tGetInFlightRequestsFunc: func(apiName string, window time.Duration) (*float64, error) {\n\t\t\treturn pointer.Float64(0), nil\n\t\t},\n\t\tGetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) {\n\t\t\treturn &userconfig.Autoscaling{\n\t\t\t\tMinReplicas:        minReplicas,\n\t\t\t\tMaxReplicas:        maxReplicas,\n\t\t\t\tInitReplicas:       minReplicas,\n\t\t\t\tTargetInFlight:     pointer.Float64(1),\n\t\t\t\tWindow:             500 * time.Millisecond,\n\t\t\t\tMaxDownscaleFactor: 0.75,\n\t\t\t\tMaxUpscaleFactor:   1.5,\n\t\t\t}, nil\n\t\t},\n\t\tCurrentRequestedReplicasFunc: func(apiName string) (int32, error) {\n\t\t\treturn minReplicas + 1, nil\n\t\t},\n\t}\n\n\tautoScaler := &Autoscaler{\n\t\tlogger:  log,\n\t\tcrons:   make(map[string]cron.Cron),\n\t\tscalers: make(map[userconfig.Kind]Scaler),\n\t\trecs:    make(map[string]*recommendations),\n\t}\n\tautoScaler.AddScaler(scalerMock, userconfig.RealtimeAPIKind)\n\n\tapiName := \"test\"\n\tapi := userconfig.Resource{\n\t\tName: apiName,\n\t\tKind: userconfig.RealtimeAPIKind,\n\t}\n\n\tautoscaleFn, err := autoScaler.autoscaleFn(api)\n\trequire.NoError(t, err)\n\n\tticker := time.NewTicker(250 * time.Millisecond)\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ticker.C:\n\t\t\t\terr := autoscaleFn()\n\t\t\t\trequire.NoError(t, err)\n\t\t\t}\n\t\t}\n\t}()\n\n\trequire.Never(t, func() bool {\n\t\tmux.RLock()\n\t\tdefer mux.RUnlock()\n\t\treturn latestRequest < minReplicas\n\t}, 3*time.Second, time.Second)\n}\n\nfunc TestAutoscaler_MaxReplicas(t *testing.T) {\n\tt.Parallel()\n\tlog := newLogger(t)\n\n\tmux := sync.RWMutex{}\n\tvar latestRequest int32\n\n\tminReplicas := int32(5)\n\tmaxReplicas := int32(10)\n\n\tscalerMock := &ScalerFunc{\n\t\tScaleFunc: func(apiName string, request int32) error {\n\t\t\tmux.Lock()\n\t\t\tdefer mux.Unlock()\n\n\t\t\tlatestRequest = request\n\t\t\treturn nil\n\t\t},\n\t\tGetInFlightRequestsFunc: func(apiName string, window time.Duration) (*float64, error) {\n\t\t\treturn pointer.Float64(20), nil\n\t\t},\n\t\tGetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) {\n\t\t\treturn &userconfig.Autoscaling{\n\t\t\t\tMinReplicas:        minReplicas,\n\t\t\t\tMaxReplicas:        maxReplicas,\n\t\t\t\tInitReplicas:       minReplicas,\n\t\t\t\tTargetInFlight:     pointer.Float64(1),\n\t\t\t\tWindow:             500 * time.Millisecond,\n\t\t\t\tMaxDownscaleFactor: 0.75,\n\t\t\t\tMaxUpscaleFactor:   1.5,\n\t\t\t}, nil\n\t\t},\n\t\tCurrentRequestedReplicasFunc: func(apiName string) (int32, error) {\n\t\t\treturn minReplicas, nil\n\t\t},\n\t}\n\n\tautoScaler := &Autoscaler{\n\t\tlogger:  log,\n\t\tcrons:   make(map[string]cron.Cron),\n\t\tscalers: make(map[userconfig.Kind]Scaler),\n\t\trecs:    make(map[string]*recommendations),\n\t}\n\tautoScaler.AddScaler(scalerMock, userconfig.RealtimeAPIKind)\n\n\tapiName := \"test\"\n\tapi := userconfig.Resource{\n\t\tName: apiName,\n\t\tKind: userconfig.RealtimeAPIKind,\n\t}\n\n\tautoscaleFn, err := autoScaler.autoscaleFn(api)\n\trequire.NoError(t, err)\n\n\tticker := time.NewTicker(250 * time.Millisecond)\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ticker.C:\n\t\t\t\terr := autoscaleFn()\n\t\t\t\trequire.NoError(t, err)\n\t\t\t}\n\t\t}\n\t}()\n\n\trequire.Never(t, func() bool {\n\t\tmux.RLock()\n\t\tdefer mux.RUnlock()\n\t\treturn latestRequest > maxReplicas\n\t}, 3*time.Second, time.Second)\n}\n"
  },
  {
    "path": "pkg/autoscaler/client.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage autoscaler\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\ntype Client interface {\n\tAwaken(api userconfig.Resource) error\n}\n\ntype client struct {\n\thttpClient *http.Client\n\tendpoint   string\n}\n\nfunc NewClient(endpoint string) Client {\n\treturn &client{\n\t\thttpClient: &http.Client{},\n\t\tendpoint:   endpoint,\n\t}\n}\n\nfunc (c *client) Awaken(api userconfig.Resource) error {\n\tpayload, err := json.Marshal(api)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tresponse, err := c.httpClient.Post(\n\t\turls.Join(c.endpoint, \"/awaken\"), \"application/json\", bytes.NewBuffer(payload),\n\t)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() { _ = response.Body.Close() }()\n\n\tif response.StatusCode != http.StatusOK {\n\t\tbodyBytes, _ := ioutil.ReadAll(response.Body)\n\t\terrMsg := fmt.Sprintf(\"failed to awake api (status code %d)\", response.StatusCode)\n\t\tif bodyBytes != nil {\n\t\t\terrMsg = errMsg + fmt.Sprintf(\": %s\", string(bodyBytes))\n\t\t}\n\n\t\treturn fmt.Errorf(errMsg)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/autoscaler/handler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage autoscaler\n\nimport (\n\t\"encoding/json\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\ntype Handler struct {\n\tautoscaler *Autoscaler\n}\n\nfunc NewHandler(autoscaler *Autoscaler) *Handler {\n\treturn &Handler{autoscaler: autoscaler}\n}\n\nfunc (h *Handler) Awaken(w http.ResponseWriter, r *http.Request) {\n\tvar api userconfig.Resource\n\tif err := json.NewDecoder(r.Body).Decode(&api); err != nil {\n\t\thttp.Error(w, \"failed to json decode request body\", http.StatusBadRequest)\n\t\treturn\n\t}\n\tdefer func() { _ = r.Body.Close() }()\n\n\tif err := h.autoscaler.Awaken(api); err != nil {\n\t\thttp.Error(w, errors.Wrap(err, \"failed to awaken api\").Error(), http.StatusInternalServerError)\n\t\ttelemetry.Error(err)\n\t\treturn\n\t}\n\n\tw.WriteHeader(http.StatusOK)\n}\n"
  },
  {
    "path": "pkg/autoscaler/realtime_scaler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage autoscaler\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tpromv1 \"github.com/prometheus/client_golang/api/prometheus/v1\"\n\t\"github.com/prometheus/common/model\"\n\t\"go.uber.org/zap\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tctrlclient \"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\ntype RealtimeScaler struct {\n\tk8s        *k8s.Client\n\tprometheus promv1.API\n\tlogger     *zap.SugaredLogger\n}\n\nfunc NewRealtimeScaler(k8sClient *k8s.Client, promClient promv1.API, logger *zap.SugaredLogger) *RealtimeScaler {\n\treturn &RealtimeScaler{\n\t\tk8s:        k8sClient,\n\t\tprometheus: promClient,\n\t\tlogger:     logger,\n\t}\n}\n\nfunc (s *RealtimeScaler) Scale(apiName string, request int32) error {\n\tctx := context.Background()\n\n\t// we use the controller-runtime client to make use of the cache mechanism\n\tvar deployment kapps.Deployment\n\terr := s.k8s.Get(ctx, ctrlclient.ObjectKey{\n\t\tNamespace: s.k8s.Namespace,\n\t\tName:      workloads.K8sName(apiName),\n\t}, &deployment)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to get deployment\")\n\t}\n\n\tif deployment.Spec.Replicas == nil {\n\t\treturn errors.Wrap(err, \"k8s deployment doesn't have the replicas field set\")\n\t}\n\n\tcurrent := *deployment.Spec.Replicas\n\tif current == request {\n\t\treturn nil\n\t}\n\n\tif request == 0 {\n\t\tif err = s.routeToActivator(&deployment); err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to re-route traffic to activator\")\n\t\t}\n\t}\n\n\tdeployment.Spec.Replicas = pointer.Int32(request)\n\n\tif err = s.k8s.Update(ctx, &deployment); err != nil {\n\t\treturn errors.Wrap(err, \"failed to update deployment\")\n\t}\n\n\tif current == 0 && request > 0 {\n\t\tgo func() {\n\t\t\tif err := s.routeToService(&deployment); err != nil {\n\t\t\t\ts.logger.Errorw(\"failed to re-route traffic to API\",\n\t\t\t\t\tzap.Error(err), zap.String(\"apiName\", apiName),\n\t\t\t\t)\n\t\t\t\ttelemetry.Error(err)\n\t\t\t}\n\t\t}()\n\n\t}\n\n\treturn nil\n}\n\nfunc (s *RealtimeScaler) GetInFlightRequests(apiName string, window time.Duration) (*float64, error) {\n\twindowSeconds := int64(window.Seconds())\n\n\t// PromQL query:\n\t// \tsum(sum_over_time(cortex_in_flight_requests{api_name=\"<apiName>\"}[60s])) /\n\t//\tsum(count_over_time(cortex_in_flight_requests{api_name=\"<apiName>\", container!=\"activator\"}[60s]))\n\tquery := fmt.Sprintf(\n\t\t\"sum(sum_over_time(cortex_in_flight_requests{api_name=\\\"%s\\\"}[%ds])) / \"+\n\t\t\t\"max(count_over_time(cortex_in_flight_requests{api_name=\\\"%s\\\", container!=\\\"activator\\\"}[%ds]))\",\n\t\tapiName, windowSeconds,\n\t\tapiName, windowSeconds,\n\t)\n\n\tctx, cancel := context.WithTimeout(context.Background(), _prometheusQueryTimeoutSeconds*time.Second)\n\tdefer cancel()\n\n\tvaluesQuery, _, err := s.prometheus.Query(ctx, query, time.Now())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvalues, ok := valuesQuery.(model.Vector)\n\tif !ok {\n\t\treturn nil, errors.ErrorUnexpected(\"failed to convert prometheus metric to vector\")\n\t}\n\n\t// no values available\n\tif values.Len() == 0 {\n\t\treturn nil, nil\n\t}\n\n\tavgInflightRequests := float64(values[0].Value)\n\n\treturn &avgInflightRequests, nil\n}\n\nfunc (s *RealtimeScaler) GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error) {\n\tdeployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName))\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"failed to get deployment\")\n\t}\n\n\tif deployment == nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to find k8s deployment\", apiName)\n\t}\n\n\tautoscalingSpec, err := userconfig.AutoscalingFromAnnotations(deployment)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn autoscalingSpec, nil\n}\n\nfunc (s *RealtimeScaler) CurrentRequestedReplicas(apiName string) (int32, error) {\n\tctx := context.Background()\n\n\t// we use the controller-runtime client to make use of the cache mechanism\n\tvar deployment kapps.Deployment\n\terr := s.k8s.Get(ctx, ctrlclient.ObjectKey{\n\t\tNamespace: s.k8s.Namespace,\n\t\tName:      workloads.K8sName(apiName),\n\t}, &deployment)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, \"failed to get deployment\")\n\t}\n\n\tif deployment.Spec.Replicas == nil {\n\t\treturn 0, errors.Wrap(err, \"k8s deployment doesn't have the replicas field set\")\n\t}\n\n\treturn *deployment.Spec.Replicas, nil\n}\n\nfunc (s *RealtimeScaler) routeToService(deployment *kapps.Deployment) error {\n\tctx := context.Background()\n\tvs, err := s.k8s.GetVirtualService(deployment.Name)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to get virtual service\")\n\t}\n\n\tif len(vs.Spec.Http) < 1 {\n\t\treturn errors.ErrorUnexpected(\"virtual service does not have any http entries\")\n\t}\n\n\tif err = s.waitForReadyReplicas(ctx, deployment); err != nil {\n\t\treturn errors.Wrap(err, \"no ready replicas available\")\n\t}\n\n\tfor i := range vs.Spec.Http {\n\t\tif len(vs.Spec.Http[i].Route) != 2 {\n\t\t\treturn errors.ErrorUnexpected(\"virtual service does not have the required number of 2 http routes\")\n\t\t}\n\n\t\tvs.Spec.Http[i].Route[0].Weight = 100 // service traffic\n\t\tvs.Spec.Http[i].Route[1].Weight = 0   // activator traffic\n\t}\n\n\tvsClient := s.k8s.IstioClientSet().NetworkingV1beta1().VirtualServices(s.k8s.Namespace)\n\tif _, err = vsClient.Update(ctx, vs, kmeta.UpdateOptions{}); err != nil {\n\t\treturn errors.Wrap(err, \"failed to update virtual service\")\n\t}\n\n\treturn nil\n}\n\nfunc (s *RealtimeScaler) routeToActivator(deployment *kapps.Deployment) error {\n\tctx := context.Background()\n\tvs, err := s.k8s.GetVirtualService(deployment.Name)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to get virtual service\")\n\t}\n\n\tif len(vs.Spec.Http) < 1 {\n\t\treturn errors.ErrorUnexpected(\"virtual service does not have any http entries\")\n\t}\n\n\tfor i := range vs.Spec.Http {\n\t\tif len(vs.Spec.Http[i].Route) != 2 {\n\t\t\treturn errors.ErrorUnexpected(\"virtual service does not have the required number of 2 http routes\")\n\t\t}\n\n\t\tvs.Spec.Http[i].Route[0].Weight = 0   // service traffic\n\t\tvs.Spec.Http[i].Route[1].Weight = 100 // activator traffic\n\t}\n\n\tvsClient := s.k8s.IstioClientSet().NetworkingV1beta1().VirtualServices(s.k8s.Namespace)\n\tif _, err = vsClient.Update(ctx, vs, kmeta.UpdateOptions{}); err != nil {\n\t\treturn errors.Wrap(err, \"failed to update virtual service\")\n\t}\n\n\treturn nil\n}\n\nfunc (s *RealtimeScaler) waitForReadyReplicas(ctx context.Context, deployment *kapps.Deployment) error {\n\twatcher, err := s.k8s.ClientSet().AppsV1().Deployments(s.k8s.Namespace).Watch(\n\t\tctx,\n\t\tkmeta.ListOptions{\n\t\t\tFieldSelector: fmt.Sprintf(\"metadata.name=%s\", deployment.Name),\n\t\t\tWatch:         true,\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"could not create deployment watcher\")\n\t}\n\n\tdefer watcher.Stop()\n\n\tctx, cancel := context.WithTimeout(ctx, consts.WaitForReadyReplicasTimeout)\n\tdefer cancel()\n\n\tfor {\n\t\tselect {\n\t\tcase event := <-watcher.ResultChan():\n\t\t\tdeploy, ok := event.Object.(*kapps.Deployment)\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif deploy.Status.ReadyReplicas > 0 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\tcase <-ctx.Done():\n\t\t\treturn ctx.Err()\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/autoscaler/recommendations.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage autoscaler\n\nimport (\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n)\n\ntype recommendations struct {\n\tmux      sync.RWMutex\n\ttimeline map[time.Time]int32\n}\n\nfunc newRecommendations() *recommendations {\n\treturn &recommendations{\n\t\ttimeline: make(map[time.Time]int32),\n\t}\n}\n\nfunc (r *recommendations) add(rec int32) {\n\tr.mux.Lock()\n\tdefer r.mux.Unlock()\n\n\tr.timeline[time.Now()] = rec\n}\n\nfunc (r *recommendations) deleteOlderThan(period time.Duration) {\n\tr.mux.Lock()\n\tdefer r.mux.Unlock()\n\n\tfor t := range r.timeline {\n\t\tif time.Since(t) > period {\n\t\t\tdelete(r.timeline, t)\n\t\t}\n\t}\n}\n\n// Returns nil if no recommendations in the period\nfunc (r *recommendations) maxSince(period time.Duration) *int32 {\n\tr.mux.RLock()\n\tdefer r.mux.RUnlock()\n\n\tmax := int32(math.MinInt32)\n\tfoundRecommendation := false\n\n\tfor t, rec := range r.timeline {\n\t\tif time.Since(t) <= period && rec > max {\n\t\t\tmax = rec\n\t\t\tfoundRecommendation = true\n\t\t}\n\t}\n\n\tif !foundRecommendation {\n\t\treturn nil\n\t}\n\n\treturn &max\n}\n\n// Returns nil if no recommendations in the period\nfunc (r *recommendations) minSince(period time.Duration) *int32 {\n\tr.mux.RLock()\n\tdefer r.mux.RUnlock()\n\n\tmin := int32(math.MaxInt32)\n\tfoundRecommendation := false\n\n\tfor t, rec := range r.timeline {\n\t\tif time.Since(t) <= period && rec < min {\n\t\t\tmin = rec\n\t\t\tfoundRecommendation = true\n\t\t}\n\t}\n\n\tif !foundRecommendation {\n\t\treturn nil\n\t}\n\n\treturn &min\n}\n"
  },
  {
    "path": "pkg/autoscaler/scaler_func.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage autoscaler\n\nimport (\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\ntype ScalerFunc struct {\n\tScaleFunc                    func(apiName string, request int32) error\n\tGetInFlightRequestsFunc      func(apiName string, window time.Duration) (*float64, error)\n\tGetAutoscalingSpecFunc       func(apiName string) (*userconfig.Autoscaling, error)\n\tCurrentRequestedReplicasFunc func(apiName string) (int32, error)\n}\n\nfunc (s *ScalerFunc) Scale(apiName string, request int32) error {\n\tif s.ScaleFunc == nil {\n\t\treturn nil\n\t}\n\n\treturn s.ScaleFunc(apiName, request)\n}\n\nfunc (s *ScalerFunc) GetInFlightRequests(apiName string, window time.Duration) (*float64, error) {\n\tif s.GetInFlightRequestsFunc == nil {\n\t\treturn nil, nil\n\t}\n\n\treturn s.GetInFlightRequestsFunc(apiName, window)\n}\n\nfunc (s *ScalerFunc) GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error) {\n\tif s.GetAutoscalingSpecFunc == nil {\n\t\treturn nil, nil\n\t}\n\n\treturn s.GetAutoscalingSpecFunc(apiName)\n}\n\nfunc (s *ScalerFunc) CurrentRequestedReplicas(apiName string) (int32, error) {\n\tif s.CurrentRequestedReplicasFunc == nil {\n\t\treturn 0, nil\n\t}\n\n\treturn s.CurrentRequestedReplicasFunc(apiName)\n}\n"
  },
  {
    "path": "pkg/config/config.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage config\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/DataDog/datadog-go/statsd\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\tcr \"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/hash\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\tpromapi \"github.com/prometheus/client_golang/api\"\n\tpromv1 \"github.com/prometheus/client_golang/api/prometheus/v1\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\tutilruntime \"k8s.io/apimachinery/pkg/util/runtime\"\n\tclientgoscheme \"k8s.io/client-go/kubernetes/scheme\"\n)\n\nvar (\n\tOperatorMetadata *clusterconfig.OperatorMetadata\n\n\tClusterConfig *clusterconfig.Config\n\n\tAWS             *aws.Client\n\tK8s             *k8s.Client\n\tK8sIstio        *k8s.Client\n\tK8sAllNamspaces *k8s.Client\n\tMetricsClient   *statsd.Client\n\tPrometheus      promv1.API\n\tscheme          = runtime.NewScheme()\n)\n\nfunc init() {\n\tutilruntime.Must(clientgoscheme.AddToScheme(scheme))\n\tutilruntime.Must(batch.AddToScheme(scheme))\n}\n\nfunc InitConfigs(clusterConfig *clusterconfig.Config, operatorMetadata *clusterconfig.OperatorMetadata) {\n\tClusterConfig = clusterConfig\n\tOperatorMetadata = operatorMetadata\n}\n\nfunc getClusterConfigFromConfigMap() (clusterconfig.Config, error) {\n\tconfigMapData, _, err := K8s.GetConfigMapData(\"cluster-config\")\n\tif err != nil {\n\t\treturn clusterconfig.Config{}, err\n\t}\n\tclusterConfig := clusterconfig.Config{}\n\terr = cr.ParseYAMLBytes(&clusterConfig, clusterconfig.FullConfigValidation, []byte(configMapData[\"cluster.yaml\"]))\n\tif err != nil {\n\t\treturn clusterconfig.Config{}, err\n\t}\n\n\treturn clusterConfig, nil\n}\n\nfunc Init() error {\n\tvar err error\n\n\tclusterConfigPath := os.Getenv(\"CORTEX_CLUSTER_CONFIG_PATH\")\n\tif clusterConfigPath == \"\" {\n\t\tclusterConfigPath = consts.DefaultInClusterConfigPath\n\t}\n\n\tclusterConfig, err := clusterconfig.NewForFile(clusterConfigPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tClusterConfig = clusterConfig\n\n\tAWS, err = aws.NewForRegion(clusterConfig.Region)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\taccountID, hashedAccountID, err := AWS.CheckCredentials()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tclusterConfig.AccountID = accountID\n\n\tOperatorMetadata = &clusterconfig.OperatorMetadata{\n\t\tAPIVersion:          consts.CortexVersion,\n\t\tOperatorID:          hashedAccountID,\n\t\tClusterID:           hash.String(clusterConfig.ClusterName + clusterConfig.Region + hashedAccountID),\n\t\tIsOperatorInCluster: strings.ToLower(os.Getenv(\"CORTEX_OPERATOR_IN_CLUSTER\")) != \"false\",\n\t}\n\n\tif K8s, err = k8s.New(consts.DefaultNamespace, OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil {\n\t\treturn err\n\t}\n\n\tif K8sIstio, err = k8s.New(consts.IstioNamespace, OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil {\n\t\treturn err\n\t}\n\n\tif !OperatorMetadata.IsOperatorInCluster {\n\t\tcc, err := getClusterConfigFromConfigMap()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tclusterConfig.Bucket = cc.Bucket\n\t\tclusterConfig.ClusterUID = cc.ClusterUID\n\t}\n\n\texists, err := AWS.DoesBucketExist(clusterConfig.Bucket)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif !exists {\n\t\treturn errors.ErrorUnexpected(\"the specified bucket does not exist\", clusterConfig.Bucket)\n\t}\n\n\terr = telemetry.Init(telemetry.Config{\n\t\tEnabled: clusterConfig.Telemetry,\n\t\tUserID:  OperatorMetadata.OperatorID,\n\t\tProperties: map[string]string{\n\t\t\t\"cluster_id\":  OperatorMetadata.ClusterID,\n\t\t\t\"operator_id\": OperatorMetadata.OperatorID,\n\t\t},\n\t\tEnvironment: \"operator\",\n\t\tLogErrors:   true,\n\t\tBackoffMode: telemetry.BackoffDuplicateMessages,\n\t})\n\tif err != nil {\n\t\tfmt.Println(errors.Message(err))\n\t}\n\n\tprometheusURL := os.Getenv(\"CORTEX_PROMETHEUS_URL\")\n\tif len(prometheusURL) == 0 {\n\t\tprometheusURL = fmt.Sprintf(\"http://prometheus.%s:9090\", consts.PrometheusNamespace)\n\t}\n\n\tpromClient, err := promapi.NewClient(promapi.Config{\n\t\tAddress: prometheusURL,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tPrometheus = promv1.NewAPI(promClient)\n\tif K8sAllNamspaces, err = k8s.New(\"\", OperatorMetadata.IsOperatorInCluster, nil, scheme); err != nil {\n\t\treturn err\n\t}\n\n\tif OperatorMetadata.IsOperatorInCluster {\n\t\tMetricsClient, err = statsd.New(fmt.Sprintf(\"prometheus-statsd-exporter.%s:9125\", consts.PrometheusNamespace))\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(errors.WithStack(err), \"unable to initialize metrics client\")\n\t\t}\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/consts/consts.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage consts\n\nimport (\n\t\"os\"\n\t\"time\"\n\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n)\n\nvar (\n\tCortexVersion      = \"master\" // CORTEX_VERSION\n\tCortexVersionMinor = \"master\" // CORTEX_VERSION_MINOR\n\n\tDefaultNamespace    = \"default\"\n\tKubeSystemNamespace = \"kube-system\"\n\tIstioNamespace      = \"istio-system\"\n\tPrometheusNamespace = \"prometheus\"\n\tLoggingNamespace    = \"logging\"\n\n\tDefaultMaxQueueLength = int64(100)\n\tDefaultMaxConcurrency = int64(1)\n\n\tDefaultUserPodPortInt32 = int32(8080)\n\n\tProxyPortStr   = \"8888\"\n\tProxyPortInt32 = int32(8888)\n\n\tActivatorName      = \"activator\"\n\tActivatorPortInt32 = int32(8000)\n\n\tAdminPortName  = \"admin\"\n\tAdminPortStr   = \"15000\"\n\tAdminPortInt32 = int32(15000)\n\n\tAuthHeader = \"X-Cortex-Authorization\"\n\n\tCortexProxyCPU    = kresource.MustParse(\"100m\")\n\tCortexProxyMem    = kresource.MustParse(\"100Mi\")\n\tCortexDequeuerCPU = kresource.MustParse(\"100m\")\n\tCortexDequeuerMem = kresource.MustParse(\"100Mi\")\n\n\t/*\n\t\tCPU Pod Reservations:\n\t\t- FluentBit 100\n\t\t- NodeExporter 50 (it has two containers)\n\t\t- KubeProxy 100\n\t\t- AWS cni 10\n\t*/\n\tCortexCPUPodReserved = kresource.MustParse(\"260m\")\n\t/*\n\t\tCPU Node Reservations:\n\t\t- Reserved (150 + 150) see generate_eks.py for details\n\t*/\n\tCortexCPUK8sReserved = kresource.MustParse(\"300m\")\n\n\t/*\n\t\tMemory Pod Reservations:\n\t\t- FluentBit 150\n\t\t- NodeExporter 200 (it has two containers)\n\t*/\n\tCortexMemPodReserved = kresource.MustParse(\"350Mi\")\n\t/*\n\t\tMemory Node Reservations:\n\t\t- Reserved (300 + 300 + 200) see generate_eks.py for details\n\t*/\n\tCortexMemK8sReserved = kresource.MustParse(\"800Mi\")\n\n\tDefaultInClusterConfigPath   = \"/configs/cluster/cluster.yaml\"\n\tMaxBucketLifecycleRules      = 100\n\tAsyncWorkloadsExpirationDays = int64(7)\n\n\tReservedContainerPorts = []int32{\n\t\tProxyPortInt32,\n\t\tAdminPortInt32,\n\t}\n\tReservedContainerNames = []string{\n\t\t\"dequeuer\",\n\t\t\"proxy\",\n\t}\n\n\tUserAgentKey             = \"User-Agent\"\n\tKubeProbeUserAgentPrefix = \"kube-probe/\"\n\n\tCortexAPINameHeader       = \"X-Cortex-API-Name\"\n\tCortexTargetServiceHeader = \"X-Cortex-Target-Service\"\n\tCortexProbeHeader         = \"X-Cortex-Probe\"\n\tCortexOriginHeader        = \"X-Cortex-Origin\"\n\tCortexQueueURLHeader      = \"X-Cortex-Queue-URL\"\n\n\tWaitForReadyReplicasTimeout = 20 * time.Minute\n)\n\nfunc DefaultRegistry() string {\n\tif registryOverride := os.Getenv(\"CORTEX_DEV_DEFAULT_IMAGE_REGISTRY\"); registryOverride != \"\" {\n\t\treturn registryOverride\n\t}\n\treturn \"quay.io/cortexlabs\"\n}\n"
  },
  {
    "path": "pkg/crds/Makefile",
    "content": "#!make\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Image URL to use all building/pushing image targets\nIMG ?= controller:latest\n# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)\nCRD_OPTIONS ?= \"crd:trivialVersions=true,preserveUnknownFields=false,crdVersions=v1\"\n# Cortex cluster config path. Defaults to the dev config path\nCLUSTER_CONFIG ?= \"../../dev/config/cluster.yaml\"\n\n# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)\nifeq (,$(shell go env GOBIN))\nGOBIN=$(shell go env GOPATH)/bin\nelse\nGOBIN=$(shell go env GOBIN)\nendif\n\n# Setting SHELL to bash allows bash commands to be executed by recipes.\n# This is a requirement for 'setup-envtest.sh' in the test target.\n# Options are set to exit when a recipe line exits non-zero or a piped command fails.\nSHELL = /usr/bin/env bash -o pipefail\n.SHELLFLAGS = -ec\n\nall: build\n\n##@ General\n\n# The help target prints out all targets with their descriptions organized\n# beneath their categories. The categories are represented by '##@' and the\n# target descriptions by '##'. The awk commands is responsible for reading the\n# entire set of makefiles included in this invocation, looking for lines of the\n# file as xyz: ## something, and then pretty-format the target and help. Then,\n# if there's a line with ##@ something, that gets pretty-printed as a category.\n# More info on the usage of ANSI control characters for terminal formatting:\n# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters\n# More info on the awk command:\n# http://linuxcommand.org/lc3_adv_awk.php\n\nhelp: ## Display this help.\n\t@awk 'BEGIN {FS = \":.*##\"; printf \"\\nUsage:\\n  make \\033[36m<target>\\033[0m\\n\"} /^[a-zA-Z_0-9-]+:.*?##/ { printf \"  \\033[36m%-15s\\033[0m %s\\n\", $$1, $$2 } /^##@/ { printf \"\\n\\033[1m%s\\033[0m\\n\", substr($$0, 5) } ' $(MAKEFILE_LIST)\n\n##@ Development\n\nmanifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.\n\t$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths=\"./...\" output:crd:artifacts:config=config/crd/bases\n\ngenerate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.\n\t$(CONTROLLER_GEN) object:headerFile=\"hack/boilerplate.go.txt\" paths=\"./...\"\n\nfmt: ## Run go fmt against code.\n\tgo fmt ./...\n\nvet: ## Run go vet against code.\n\tgo vet ./...\n\nENVTEST_ASSETS_DIR=$(shell pwd)/testbin\ntest: manifests generate fmt vet ## Run tests.\n\tmkdir -p ${ENVTEST_ASSETS_DIR}\n\ttest -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh\n\tsource ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out\n\n##@ Build\n\nbuild: generate fmt vet ## Build manager binary.\n\tgo build -o bin/manager main.go\n\nrun: manifests generate fmt vet ## Run a controller from your host.\n\tbash ./hack/run_manager.sh ${CLUSTER_CONFIG}\n\ndocker-build: ## Build docker image with the manager.\n\tdocker build -f ../../images/controller-manager/Dockerfile -t ${IMG} ../..\n\ndocker-push: ## Push docker image with the manager.\n\tdocker push ${IMG}\n\n##@ Deployment\n\ninstall: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.\n\t$(KUSTOMIZE) build config/crd | kubectl apply -f -\n\nuninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.\n\t$(KUSTOMIZE) build config/crd | kubectl delete -f -\n\ndeploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.\n\tcd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}\n\t$(KUSTOMIZE) build config/default | kubectl apply -f -\n\nundeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.\n\t$(KUSTOMIZE) build config/default | kubectl delete -f -\n\n\nCONTROLLER_GEN = $(shell pwd)/bin/controller-gen\ncontroller-gen: ## Download controller-gen locally if necessary.\n\t$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1)\n\nKUSTOMIZE = $(shell pwd)/bin/kustomize\nkustomize: ## Download kustomize locally if necessary.\n\t$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7)\n\n# go-get-tool will 'go get' any package $2 and install it to $1.\nPROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))\ndefine go-get-tool\n@[ -f $(1) ] || { \\\nset -e ;\\\nTMP_DIR=$$(mktemp -d) ;\\\ncd $$TMP_DIR ;\\\ngo mod init tmp ;\\\necho \"Downloading $(2)\" ;\\\nGOBIN=$(PROJECT_DIR)/bin go get $(2) ;\\\nrm -rf $$TMP_DIR ;\\\n}\nendef\n"
  },
  {
    "path": "pkg/crds/PROJECT",
    "content": "domain: cortex.dev\nlayout:\n- go.kubebuilder.io/v3\nmultigroup: true\nprojectName: operator\nrepo: github.com/cortexlabs/cortex\nresources:\n- api:\n    crdVersion: v1\n    namespaced: true\n  controller: true\n  domain: cortex.dev\n  group: batch\n  kind: BatchJob\n  path: github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\n  version: v1alpha1\nversion: \"3\"\n"
  },
  {
    "path": "pkg/crds/apis/batch/v1alpha1/batchjob_metrics.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage v1alpha1\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/types/metrics\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\tpromv1 \"github.com/prometheus/client_golang/api/prometheus/v1\"\n\t\"github.com/prometheus/common/model\"\n)\n\nconst (\n\t_metricsRequestTimeoutSeconds = 10\n)\n\n// GetMetrics retrieves the BatchJob metrics from prometheus\nfunc GetMetrics(promAPIv1 promv1.API, jobKey spec.JobKey, t time.Time) (*metrics.BatchMetrics, error) {\n\tvar (\n\t\tjobBatchesSucceeded float64\n\t\tjobBatchesFailed    float64\n\t\tavgTimePerBatch     *float64\n\t)\n\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tjobBatchesSucceeded, err = getSucceededBatchesForJobMetric(promAPIv1, jobKey, t)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tjobBatchesFailed, err = getFailedBatchesForJobMetric(promAPIv1, jobKey, t)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tavgTimePerBatch, err = getAvgTimePerBatchMetric(promAPIv1, jobKey, t)\n\t\t\treturn err\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &metrics.BatchMetrics{\n\t\tSucceeded:           int(jobBatchesSucceeded),\n\t\tFailed:              int(jobBatchesFailed),\n\t\tAverageTimePerBatch: avgTimePerBatch,\n\t}, nil\n}\n\nfunc getSucceededBatchesForJobMetric(promAPIv1 promv1.API, jobKey spec.JobKey, t time.Time) (float64, error) {\n\tquery := fmt.Sprintf(\n\t\t\"sum(cortex_batch_succeeded{api_name=\\\"%s\\\", job_id=\\\"%s\\\"})\",\n\t\tjobKey.APIName, jobKey.ID,\n\t)\n\n\tvalues, err := queryPrometheusVec(promAPIv1, query, t)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif values.Len() == 0 {\n\t\treturn 0, nil\n\t}\n\n\tsucceededBatches := float64(values[0].Value)\n\treturn succeededBatches, nil\n}\n\nfunc getFailedBatchesForJobMetric(promAPIv1 promv1.API, jobKey spec.JobKey, t time.Time) (float64, error) {\n\tquery := fmt.Sprintf(\n\t\t\"sum(cortex_batch_failed{api_name=\\\"%s\\\", job_id=\\\"%s\\\"})\",\n\t\tjobKey.APIName, jobKey.ID,\n\t)\n\n\tvalues, err := queryPrometheusVec(promAPIv1, query, t)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif values.Len() == 0 {\n\t\treturn 0, nil\n\t}\n\n\tfailedBatches := float64(values[0].Value)\n\treturn failedBatches, nil\n}\n\nfunc getAvgTimePerBatchMetric(promAPIv1 promv1.API, jobKey spec.JobKey, t time.Time) (*float64, error) {\n\tquery := fmt.Sprintf(\n\t\t\"sum(cortex_time_per_batch_sum{api_name=\\\"%s\\\", job_id=\\\"%s\\\"}) / sum(cortex_time_per_batch_count{api_name=\\\"%s\\\", job_id=\\\"%s\\\"})\",\n\t\tjobKey.APIName, jobKey.ID,\n\t\tjobKey.APIName, jobKey.ID,\n\t)\n\n\tvalues, err := queryPrometheusVec(promAPIv1, query, t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif values.Len() == 0 {\n\t\treturn nil, nil\n\t}\n\n\tavgTimePerBatch := float64(values[0].Value)\n\treturn &avgTimePerBatch, nil\n}\n\nfunc queryPrometheusVec(promAPIv1 promv1.API, query string, t time.Time) (model.Vector, error) {\n\tctx, cancel := context.WithTimeout(context.Background(), _metricsRequestTimeoutSeconds*time.Second)\n\tdefer cancel()\n\n\tvaluesQuery, _, err := promAPIv1.Query(ctx, query, t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvalues, ok := valuesQuery.(model.Vector)\n\tif !ok {\n\t\treturn nil, errors.ErrorUnexpected(\"failed to convert metric to vector\")\n\t}\n\n\treturn values, nil\n}\n"
  },
  {
    "path": "pkg/crds/apis/batch/v1alpha1/batchjob_types.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage v1alpha1\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\n// BatchJobSpec defines the desired state of BatchJob\ntype BatchJobSpec struct {\n\t// +kubebuilder:validation:Required\n\t// Reference to a cortex BatchAPI name\n\tAPIName string `json:\"api_name,omitempty\"`\n\n\t// +kubebuilder:validation:Required\n\t// Reference to a cortex BatchAPI apiID\n\tAPIID string `json:\"api_id,omitempty\"`\n\n\t// +kubebuilder:validation:Optional\n\t// +kubebuilder:default=1\n\t// Number of workers for the batch job\n\tWorkers int32 `json:\"workers,omitempty\"`\n\n\t// +kubebuilder:validation:Optional\n\t// YAML content of the user config\n\tConfig *string `json:\"config,omitempty\"`\n\n\t// +kubebuilder:validation:Optional\n\t// Duration until a batch job times out\n\tTimeout *kmeta.Duration `json:\"timeout,omitempty\"`\n\n\t// +kubebuilder:validation:Optional\n\t// Configuration for the dead letter queue\n\tDeadLetterQueue *DeadLetterQueueSpec `json:\"dead_letter_queue,omitempty\"`\n\n\t// +kubebuilder:validation:Optional\n\t// Compute resource requirements\n\tResources *kcore.ResourceRequirements `json:\"resources,omitempty\"`\n\n\t// +kubebuilder:validation:Optional\n\t// +nullable\n\t// Node groups selector\n\tNodeGroups []string `json:\"node_groups\"`\n\n\t// +kubebuilder:validation:Optional\n\t// +nullable\n\t// Readiness probes for the job (container name -> probe)\n\tProbes map[string]kcore.Probe `json:\"probes\"`\n\n\t// +kubebuilder:validation:Optional\n\t// Time to live for the resource. The controller will clean-up resources\n\t// that reached a final state when the TTL time is exceeded.\n\tTTL *kmeta.Duration `json:\"ttl,omitempty\"`\n}\n\n// DeadLetterQueueSpec defines the desired state for the dead letter queue in a BatchJob\ntype DeadLetterQueueSpec struct {\n\t// +kubebuilder:validation:Required\n\t// arn of the dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo\n\tARN string `json:\"arn,omitempty\"`\n\n\t// +kubebuilder:validation:Optional\n\t// +kubebuilder:default=1\n\t// +kubebuilder:validation:Minimum=1\n\t// Number of times a batch is allowed to be handled by a worker before it is considered to be failed\n\t// and transferred to the dead letter queue (must be >= 1)\n\tMaxReceiveCount int32 `json:\"max_receive_count,omitempty\"`\n}\n\n// BatchJobStatus defines the observed state of BatchJob\ntype BatchJobStatus struct {\n\t// Job ID\n\tID string `json:\"id,omitempty\"`\n\n\t// Processing ending timestamp\n\tEndTime *kmeta.Time `json:\"end_time,omitempty\"`\n\n\t// URL for the used SQS queue\n\tQueueURL string `json:\"queue_url,omitempty\"`\n\n\t// Total batch count\n\tTotalBatchCount int `json:\"total_batch_count,omitempty\"`\n\n\t// +kubebuilder:validation:Type=string\n\t// Status of the batch job\n\tStatus status.JobCode `json:\"status,omitempty\"`\n\n\t// Detailed worker counts with respective status\n\tWorkerCounts *status.WorkerCounts `json:\"worker_counts,omitempty\"`\n}\n\n// EnqueuingStatus is an enum for the different possible enqueuing status\ntype EnqueuingStatus string\n\n// Possible EnqueuingStatus states\nconst (\n\tEnqueuingNotStarted EnqueuingStatus = \"not_started\"\n\tEnqueuingInProgress EnqueuingStatus = \"in_progress\"\n\tEnqueuingDone       EnqueuingStatus = \"done\"\n\tEnqueuingFailed     EnqueuingStatus = \"failed\"\n)\n\n// +kubebuilder:object:root=true\n// +kubebuilder:subresource:status\n// +kubebuilder:printcolumn:JSONPath=\".status.status\",name=\"Status\",type=\"string\"\n// +kubebuilder:printcolumn:JSONPath=\".status.queue_url\",name=\"Queue URL\",type=\"string\"\n\n// BatchJob is the Schema for the batchjobs API\ntype BatchJob struct {\n\tkmeta.TypeMeta   `json:\",inline\"`\n\tkmeta.ObjectMeta `json:\"metadata,omitempty\"`\n\n\tSpec   BatchJobSpec   `json:\"spec,omitempty\"`\n\tStatus BatchJobStatus `json:\"status,omitempty\"`\n}\n\n// +kubebuilder:object:root=true\n\n// BatchJobList contains a list of BatchJob\ntype BatchJobList struct {\n\tkmeta.TypeMeta `json:\",inline\"`\n\tkmeta.ListMeta `json:\"metadata,omitempty\"`\n\tItems          []BatchJob `json:\"items\"`\n}\n\nfunc init() {\n\tSchemeBuilder.Register(&BatchJob{}, &BatchJobList{})\n}\n"
  },
  {
    "path": "pkg/crds/apis/batch/v1alpha1/groupversion_info.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n// Package v1alpha1 contains API Schema definitions for the batch v1alpha1 API group\n// +kubebuilder:object:generate=true\n// +groupName=batch.cortex.dev\npackage v1alpha1\n\nimport (\n\t\"k8s.io/apimachinery/pkg/runtime/schema\"\n\t\"sigs.k8s.io/controller-runtime/pkg/scheme\"\n)\n\nvar (\n\t// GroupVersion is group version used to register these objects\n\tGroupVersion = schema.GroupVersion{Group: \"batch.cortex.dev\", Version: \"v1alpha1\"}\n\n\t// SchemeBuilder is used to add go types to the GroupVersionKind scheme\n\tSchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}\n\n\t// AddToScheme adds the types in this group-version to the given scheme.\n\tAddToScheme = SchemeBuilder.AddToScheme\n)\n"
  },
  {
    "path": "pkg/crds/apis/batch/v1alpha1/zz_generated.deepcopy.go",
    "content": "//go:build !ignore_autogenerated\n// +build !ignore_autogenerated\n\n/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n// Code generated by controller-gen. DO NOT EDIT.\n\npackage v1alpha1\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\truntime \"k8s.io/apimachinery/pkg/runtime\"\n)\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *BatchJob) DeepCopyInto(out *BatchJob) {\n\t*out = *in\n\tout.TypeMeta = in.TypeMeta\n\tin.ObjectMeta.DeepCopyInto(&out.ObjectMeta)\n\tin.Spec.DeepCopyInto(&out.Spec)\n\tin.Status.DeepCopyInto(&out.Status)\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchJob.\nfunc (in *BatchJob) DeepCopy() *BatchJob {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(BatchJob)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.\nfunc (in *BatchJob) DeepCopyObject() runtime.Object {\n\tif c := in.DeepCopy(); c != nil {\n\t\treturn c\n\t}\n\treturn nil\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *BatchJobList) DeepCopyInto(out *BatchJobList) {\n\t*out = *in\n\tout.TypeMeta = in.TypeMeta\n\tin.ListMeta.DeepCopyInto(&out.ListMeta)\n\tif in.Items != nil {\n\t\tin, out := &in.Items, &out.Items\n\t\t*out = make([]BatchJob, len(*in))\n\t\tfor i := range *in {\n\t\t\t(*in)[i].DeepCopyInto(&(*out)[i])\n\t\t}\n\t}\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchJobList.\nfunc (in *BatchJobList) DeepCopy() *BatchJobList {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(BatchJobList)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.\nfunc (in *BatchJobList) DeepCopyObject() runtime.Object {\n\tif c := in.DeepCopy(); c != nil {\n\t\treturn c\n\t}\n\treturn nil\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *BatchJobSpec) DeepCopyInto(out *BatchJobSpec) {\n\t*out = *in\n\tif in.Config != nil {\n\t\tin, out := &in.Config, &out.Config\n\t\t*out = new(string)\n\t\t**out = **in\n\t}\n\tif in.Timeout != nil {\n\t\tin, out := &in.Timeout, &out.Timeout\n\t\t*out = new(v1.Duration)\n\t\t**out = **in\n\t}\n\tif in.DeadLetterQueue != nil {\n\t\tin, out := &in.DeadLetterQueue, &out.DeadLetterQueue\n\t\t*out = new(DeadLetterQueueSpec)\n\t\t**out = **in\n\t}\n\tif in.Resources != nil {\n\t\tin, out := &in.Resources, &out.Resources\n\t\t*out = new(corev1.ResourceRequirements)\n\t\t(*in).DeepCopyInto(*out)\n\t}\n\tif in.NodeGroups != nil {\n\t\tin, out := &in.NodeGroups, &out.NodeGroups\n\t\t*out = make([]string, len(*in))\n\t\tcopy(*out, *in)\n\t}\n\tif in.Probes != nil {\n\t\tin, out := &in.Probes, &out.Probes\n\t\t*out = make(map[string]corev1.Probe, len(*in))\n\t\tfor key, val := range *in {\n\t\t\t(*out)[key] = *val.DeepCopy()\n\t\t}\n\t}\n\tif in.TTL != nil {\n\t\tin, out := &in.TTL, &out.TTL\n\t\t*out = new(v1.Duration)\n\t\t**out = **in\n\t}\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchJobSpec.\nfunc (in *BatchJobSpec) DeepCopy() *BatchJobSpec {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(BatchJobSpec)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *BatchJobStatus) DeepCopyInto(out *BatchJobStatus) {\n\t*out = *in\n\tif in.EndTime != nil {\n\t\tin, out := &in.EndTime, &out.EndTime\n\t\t*out = (*in).DeepCopy()\n\t}\n\tif in.WorkerCounts != nil {\n\t\tin, out := &in.WorkerCounts, &out.WorkerCounts\n\t\t*out = new(status.WorkerCounts)\n\t\t**out = **in\n\t}\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchJobStatus.\nfunc (in *BatchJobStatus) DeepCopy() *BatchJobStatus {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(BatchJobStatus)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n\n// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.\nfunc (in *DeadLetterQueueSpec) DeepCopyInto(out *DeadLetterQueueSpec) {\n\t*out = *in\n}\n\n// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeadLetterQueueSpec.\nfunc (in *DeadLetterQueueSpec) DeepCopy() *DeadLetterQueueSpec {\n\tif in == nil {\n\t\treturn nil\n\t}\n\tout := new(DeadLetterQueueSpec)\n\tin.DeepCopyInto(out)\n\treturn out\n}\n"
  },
  {
    "path": "pkg/crds/config/crd/bases/batch.cortex.dev_batchjobs.yaml",
    "content": "---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    controller-gen.kubebuilder.io/version: v0.4.1\n  creationTimestamp: null\n  name: batchjobs.batch.cortex.dev\nspec:\n  group: batch.cortex.dev\n  names:\n    kind: BatchJob\n    listKind: BatchJobList\n    plural: batchjobs\n    singular: batchjob\n  scope: Namespaced\n  versions:\n  - additionalPrinterColumns:\n    - jsonPath: .status.status\n      name: Status\n      type: string\n    - jsonPath: .status.queue_url\n      name: Queue URL\n      type: string\n    name: v1alpha1\n    schema:\n      openAPIV3Schema:\n        description: BatchJob is the Schema for the batchjobs API\n        properties:\n          apiVersion:\n            description: 'APIVersion defines the versioned schema of this representation\n              of an object. Servers should convert recognized schemas to the latest\n              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n            type: string\n          kind:\n            description: 'Kind is a string value representing the REST resource this\n              object represents. Servers may infer this from the endpoint the client\n              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n            type: string\n          metadata:\n            type: object\n          spec:\n            description: BatchJobSpec defines the desired state of BatchJob\n            properties:\n              api_id:\n                description: Reference to a cortex BatchAPI apiID\n                type: string\n              api_name:\n                description: Reference to a cortex BatchAPI name\n                type: string\n              config:\n                description: YAML content of the user config\n                type: string\n              dead_letter_queue:\n                description: Configuration for the dead letter queue\n                properties:\n                  arn:\n                    description: arn of the dead letter queue e.g. arn:aws:sqs:us-west-2:123456789:failed.fifo\n                    type: string\n                  max_receive_count:\n                    default: 1\n                    description: Number of times a batch is allowed to be handled\n                      by a worker before it is considered to be failed and transferred\n                      to the dead letter queue (must be >= 1)\n                    format: int32\n                    minimum: 1\n                    type: integer\n                type: object\n              node_groups:\n                description: Node groups selector\n                items:\n                  type: string\n                nullable: true\n                type: array\n              probes:\n                additionalProperties:\n                  description: Probe describes a health check to be performed against\n                    a container to determine whether it is alive or ready to receive\n                    traffic.\n                  properties:\n                    exec:\n                      description: One and only one of the following should be specified.\n                        Exec specifies the action to take.\n                      properties:\n                        command:\n                          description: Command is the command line to execute inside\n                            the container, the working directory for the command  is\n                            root ('/') in the container's filesystem. The command\n                            is simply exec'd, it is not run inside a shell, so traditional\n                            shell instructions ('|', etc) won't work. To use a shell,\n                            you need to explicitly call out to that shell. Exit status\n                            of 0 is treated as live/healthy and non-zero is unhealthy.\n                          items:\n                            type: string\n                          type: array\n                      type: object\n                    failureThreshold:\n                      description: Minimum consecutive failures for the probe to be\n                        considered failed after having succeeded. Defaults to 3. Minimum\n                        value is 1.\n                      format: int32\n                      type: integer\n                    httpGet:\n                      description: HTTPGet specifies the http request to perform.\n                      properties:\n                        host:\n                          description: Host name to connect to, defaults to the pod\n                            IP. You probably want to set \"Host\" in httpHeaders instead.\n                          type: string\n                        httpHeaders:\n                          description: Custom headers to set in the request. HTTP\n                            allows repeated headers.\n                          items:\n                            description: HTTPHeader describes a custom header to be\n                              used in HTTP probes\n                            properties:\n                              name:\n                                description: The header field name\n                                type: string\n                              value:\n                                description: The header field value\n                                type: string\n                            required:\n                            - name\n                            - value\n                            type: object\n                          type: array\n                        path:\n                          description: Path to access on the HTTP server.\n                          type: string\n                        port:\n                          anyOf:\n                          - type: integer\n                          - type: string\n                          description: Name or number of the port to access on the\n                            container. Number must be in the range 1 to 65535. Name\n                            must be an IANA_SVC_NAME.\n                          x-kubernetes-int-or-string: true\n                        scheme:\n                          description: Scheme to use for connecting to the host. Defaults\n                            to HTTP.\n                          type: string\n                      required:\n                      - port\n                      type: object\n                    initialDelaySeconds:\n                      description: 'Number of seconds after the container has started\n                        before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      format: int32\n                      type: integer\n                    periodSeconds:\n                      description: How often (in seconds) to perform the probe. Default\n                        to 10 seconds. Minimum value is 1.\n                      format: int32\n                      type: integer\n                    successThreshold:\n                      description: Minimum consecutive successes for the probe to\n                        be considered successful after having failed. Defaults to\n                        1. Must be 1 for liveness and startup. Minimum value is 1.\n                      format: int32\n                      type: integer\n                    tcpSocket:\n                      description: 'TCPSocket specifies an action involving a TCP\n                        port. TCP hooks not yet supported TODO: implement a realistic\n                        TCP lifecycle hook'\n                      properties:\n                        host:\n                          description: 'Optional: Host name to connect to, defaults\n                            to the pod IP.'\n                          type: string\n                        port:\n                          anyOf:\n                          - type: integer\n                          - type: string\n                          description: Number or name of the port to access on the\n                            container. Number must be in the range 1 to 65535. Name\n                            must be an IANA_SVC_NAME.\n                          x-kubernetes-int-or-string: true\n                      required:\n                      - port\n                      type: object\n                    timeoutSeconds:\n                      description: 'Number of seconds after which the probe times\n                        out. Defaults to 1 second. Minimum value is 1. More info:\n                        https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'\n                      format: int32\n                      type: integer\n                  type: object\n                description: Readiness probes for the job (container name -> probe)\n                nullable: true\n                type: object\n              resources:\n                description: Compute resource requirements\n                properties:\n                  limits:\n                    additionalProperties:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                      x-kubernetes-int-or-string: true\n                    description: 'Limits describes the maximum amount of compute resources\n                      allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                    type: object\n                  requests:\n                    additionalProperties:\n                      anyOf:\n                      - type: integer\n                      - type: string\n                      pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n                      x-kubernetes-int-or-string: true\n                    description: 'Requests describes the minimum amount of compute\n                      resources required. If Requests is omitted for a container,\n                      it defaults to Limits if that is explicitly specified, otherwise\n                      to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'\n                    type: object\n                type: object\n              timeout:\n                description: Duration until a batch job times out\n                type: string\n              ttl:\n                description: Time to live for the resource. The controller will clean-up\n                  resources that reached a final state when the TTL time is exceeded.\n                type: string\n              workers:\n                default: 1\n                description: Number of workers for the batch job\n                format: int32\n                type: integer\n            type: object\n          status:\n            description: BatchJobStatus defines the observed state of BatchJob\n            properties:\n              end_time:\n                description: Processing ending timestamp\n                format: date-time\n                type: string\n              id:\n                description: Job ID\n                type: string\n              queue_url:\n                description: URL for the used SQS queue\n                type: string\n              status:\n                description: Status of the batch job\n                type: string\n              total_batch_count:\n                description: Total batch count\n                type: integer\n              worker_counts:\n                description: Detailed worker counts with respective status\n                properties:\n                  creating:\n                    format: int32\n                    type: integer\n                  err_image_pull:\n                    format: int32\n                    type: integer\n                  failed:\n                    format: int32\n                    type: integer\n                  killed:\n                    format: int32\n                    type: integer\n                  killed_oom:\n                    format: int32\n                    type: integer\n                  not_ready:\n                    format: int32\n                    type: integer\n                  pending:\n                    format: int32\n                    type: integer\n                  ready:\n                    format: int32\n                    type: integer\n                  stalled:\n                    format: int32\n                    type: integer\n                  succeeded:\n                    format: int32\n                    type: integer\n                  terminating:\n                    format: int32\n                    type: integer\n                  unknown:\n                    format: int32\n                    type: integer\n                type: object\n            type: object\n        type: object\n    served: true\n    storage: true\n    subresources:\n      status: {}\nstatus:\n  acceptedNames:\n    kind: \"\"\n    plural: \"\"\n  conditions: []\n  storedVersions: []\n"
  },
  {
    "path": "pkg/crds/config/crd/kustomization.yaml",
    "content": "# This kustomization.yaml is not intended to be run by itself,\n# since it depends on service name and namespace that are out of this kustomize package.\n# It should be run by config/default\nresources:\n- bases/batch.cortex.dev_batchjobs.yaml\n#+kubebuilder:scaffold:crdkustomizeresource\n\npatchesStrategicMerge:\n# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.\n# patches here are for enabling the conversion webhook for each CRD\n#- patches/webhook_in_batchjobs.yaml\n#+kubebuilder:scaffold:crdkustomizewebhookpatch\n\n# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.\n# patches here are for enabling the CA injection for each CRD\n#- patches/cainjection_in_batchjobs.yaml\n#+kubebuilder:scaffold:crdkustomizecainjectionpatch\n\n# the following config is for teaching kustomize how to do kustomization for CRDs.\nconfigurations:\n- kustomizeconfig.yaml\n"
  },
  {
    "path": "pkg/crds/config/crd/kustomizeconfig.yaml",
    "content": "# This file is for teaching kustomize how to substitute name and namespace reference in CRD\nnameReference:\n- kind: Service\n  version: v1\n  fieldSpecs:\n  - kind: CustomResourceDefinition\n    version: v1\n    group: apiextensions.k8s.io\n    path: spec/conversion/webhook/clientConfig/service/name\n\nnamespace:\n- kind: CustomResourceDefinition\n  version: v1\n  group: apiextensions.k8s.io\n  path: spec/conversion/webhook/clientConfig/service/namespace\n  create: false\n\nvarReference:\n- path: metadata/annotations\n"
  },
  {
    "path": "pkg/crds/config/crd/patches/cainjection_in_batchjobs.yaml",
    "content": "# The following patch adds a directive for certmanager to inject CA into the CRD\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  annotations:\n    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)\n  name: batchjobs.batch.cortex.dev\n"
  },
  {
    "path": "pkg/crds/config/crd/patches/webhook_in_batchjobs.yaml",
    "content": "# The following patch enables a conversion webhook for the CRD\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n  name: batchjobs.batch.cortex.dev\nspec:\n  conversion:\n    strategy: Webhook\n    webhook:\n      clientConfig:\n        service:\n          namespace: system\n          name: webhook-service\n          path: /convert\n"
  },
  {
    "path": "pkg/crds/config/default/kustomization.yaml",
    "content": "# Adds namespace to all resources.\nnamespace: default\n\n# Value of this field is prepended to the\n# names of all resources, e.g. a deployment named\n# \"wordpress\" becomes \"alices-wordpress\".\n# Note that it should also match with the prefix (text before '-') of the namespace\n# field above.\nnamePrefix: operator-\n\n# Labels to add to all resources and selectors.\n#commonLabels:\n#  someName: someValue\n\nbases:\n- ../crd\n- ../rbac\n- ../manager\n# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in\n# crd/kustomization.yaml\n#- ../webhook\n# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.\n#- ../certmanager\n# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.\n#- ../prometheus\n\npatchesStrategicMerge:\n# Protect the /metrics endpoint by putting it behind auth.\n# If you want your controller-manager to expose the /metrics\n# endpoint w/o any authn/z, please comment the following line.\n#- manager_auth_proxy_patch.yaml\n\n# Mount the controller config file for loading manager configurations\n# through a ComponentConfig type\n#- manager_config_patch.yaml\n\n# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in\n# crd/kustomization.yaml\n#- manager_webhook_patch.yaml\n\n# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.\n# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.\n# 'CERTMANAGER' needs to be enabled to use ca injection\n#- webhookcainjection_patch.yaml\n\n# the following config is for teaching kustomize how to do var substitution\nvars:\n# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.\n#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR\n#  objref:\n#    kind: Certificate\n#    group: cert-manager.io\n#    version: v1\n#    name: serving-cert # this name should match the one in certificate.yaml\n#  fieldref:\n#    fieldpath: metadata.namespace\n#- name: CERTIFICATE_NAME\n#  objref:\n#    kind: Certificate\n#    group: cert-manager.io\n#    version: v1\n#    name: serving-cert # this name should match the one in certificate.yaml\n#- name: SERVICE_NAMESPACE # namespace of the service\n#  objref:\n#    kind: Service\n#    version: v1\n#    name: webhook-service\n#  fieldref:\n#    fieldpath: metadata.namespace\n#- name: SERVICE_NAME\n#  objref:\n#    kind: Service\n#    version: v1\n#    name: webhook-service\n"
  },
  {
    "path": "pkg/crds/config/default/manager_auth_proxy_patch.yaml",
    "content": "# This patch inject a sidecar container which is a HTTP proxy for the\n# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: controller-manager\n  namespace: system\nspec:\n  template:\n    spec:\n      containers:\n      - name: kube-rbac-proxy\n        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0\n        args:\n        - \"--secure-listen-address=0.0.0.0:8443\"\n        - \"--upstream=http://127.0.0.1:8080/\"\n        - \"--logtostderr=true\"\n        - \"--v=10\"\n        ports:\n        - containerPort: 8443\n          name: https\n      - name: manager\n        args:\n        - \"--health-probe-bind-address=:8081\"\n        - \"--metrics-bind-address=127.0.0.1:8080\"\n        - \"--leader-elect\"\n"
  },
  {
    "path": "pkg/crds/config/default/manager_config_patch.yaml",
    "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: controller-manager\n  namespace: system\nspec:\n  template:\n    spec:\n      containers:\n      - name: manager\n        args:\n        - \"--config=controller_manager_config.yaml\"\n        volumeMounts:\n        - name: manager-config\n          mountPath: /controller_manager_config.yaml\n          subPath: controller_manager_config.yaml\n      volumes:\n      - name: manager-config\n        configMap:\n          name: manager-config\n"
  },
  {
    "path": "pkg/crds/config/manager/controller_manager_config.yaml",
    "content": "apiVersion: controller-runtime.sigs.k8s.io/v1alpha1\nkind: ControllerManagerConfig\nhealth:\n  healthProbeBindAddress: :8081\nmetrics:\n  bindAddress: 127.0.0.1:8080\nwebhook:\n  port: 9443\nleaderElection:\n  leaderElect: true\n  resourceName: 7cc92962.cortex.dev\n"
  },
  {
    "path": "pkg/crds/config/manager/kustomization.yaml",
    "content": "resources:\n- manager.yaml\n\ngeneratorOptions:\n  disableNameSuffixHash: true\n\nconfigMapGenerator:\n- files:\n  - controller_manager_config.yaml\n  name: manager-config\napiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"
  },
  {
    "path": "pkg/crds/config/manager/manager.yaml",
    "content": "---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n  name: controller-manager\n  namespace: system\n  labels:\n    control-plane: controller-manager\nspec:\n  selector:\n    matchLabels:\n      control-plane: controller-manager\n  replicas: 1\n  strategy:\n    rollingUpdate:\n      maxSurge: 0\n  template:\n    metadata:\n      labels:\n        control-plane: controller-manager\n    spec:\n      securityContext:\n        runAsNonRoot: true\n      containers:\n      - name: manager\n        command:\n        - /manager\n        args:\n        - \"--config=/mnt/cluster.yaml\"\n        - \"--leader-elect\"\n        image: controller:latest\n        imagePullPolicy: Always\n        securityContext:\n          allowPrivilegeEscalation: false\n        env:\n          - name: CORTEX_OPERATOR_IN_CLUSTER\n            value: \"true\"\n        livenessProbe:\n          httpGet:\n            path: /healthz\n            port: 8081\n          initialDelaySeconds: 15\n          periodSeconds: 20\n        readinessProbe:\n          httpGet:\n            path: /readyz\n            port: 8081\n          initialDelaySeconds: 5\n          periodSeconds: 10\n        resources:\n          limits:\n            cpu: 300m\n            memory: 100Mi\n          requests:\n            cpu: 100m\n            memory: 80Mi\n        envFrom:\n        - configMapRef:\n            name: env-vars\n        volumeMounts:\n          - mountPath: /mnt/cluster.yaml\n            name: cluster-config\n            subPath: cluster.yaml\n      volumes:\n        - name: cluster-config\n          configMap:\n            name: cluster-config\n      serviceAccountName: controller-manager\n      terminationGracePeriodSeconds: 10\n"
  },
  {
    "path": "pkg/crds/config/prometheus/kustomization.yaml",
    "content": "resources:\n- monitor.yaml\n"
  },
  {
    "path": "pkg/crds/config/prometheus/monitor.yaml",
    "content": "# Prometheus Monitor Service (Metrics)\napiVersion: monitoring.coreos.com/v1\nkind: ServiceMonitor\nmetadata:\n  labels:\n    control-plane: controller-manager\n  name: controller-manager-metrics-monitor\n  namespace: system\nspec:\n  endpoints:\n    - path: /metrics\n      port: https\n      scheme: https\n      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\n      tlsConfig:\n        insecureSkipVerify: true\n  selector:\n    matchLabels:\n      control-plane: controller-manager\n"
  },
  {
    "path": "pkg/crds/config/rbac/auth_proxy_client_clusterrole.yaml",
    "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: metrics-reader\nrules:\n- nonResourceURLs:\n  - \"/metrics\"\n  verbs:\n  - get\n"
  },
  {
    "path": "pkg/crds/config/rbac/auth_proxy_role.yaml",
    "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: proxy-role\nrules:\n- apiGroups:\n  - authentication.k8s.io\n  resources:\n  - tokenreviews\n  verbs:\n  - create\n- apiGroups:\n  - authorization.k8s.io\n  resources:\n  - subjectaccessreviews\n  verbs:\n  - create\n"
  },
  {
    "path": "pkg/crds/config/rbac/auth_proxy_role_binding.yaml",
    "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: proxy-rolebinding\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: proxy-role\nsubjects:\n- kind: ServiceAccount\n  name: controller-manager\n  namespace: system\n"
  },
  {
    "path": "pkg/crds/config/rbac/auth_proxy_service.yaml",
    "content": "apiVersion: v1\nkind: Service\nmetadata:\n  labels:\n    control-plane: controller-manager\n  name: controller-manager-metrics-service\n  namespace: system\nspec:\n  ports:\n  - name: https\n    port: 8443\n    targetPort: https\n  selector:\n    control-plane: controller-manager\n"
  },
  {
    "path": "pkg/crds/config/rbac/batchjob_editor_role.yaml",
    "content": "# permissions for end users to edit batchjobs.\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: batchjob-editor-role\nrules:\n- apiGroups:\n  - batch.cortex.dev\n  resources:\n  - batchjobs\n  verbs:\n  - create\n  - delete\n  - get\n  - list\n  - patch\n  - update\n  - watch\n- apiGroups:\n  - batch.cortex.dev\n  resources:\n  - batchjobs/status\n  verbs:\n  - get\n"
  },
  {
    "path": "pkg/crds/config/rbac/batchjob_viewer_role.yaml",
    "content": "# permissions for end users to view batchjobs.\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  name: batchjob-viewer-role\nrules:\n- apiGroups:\n  - batch.cortex.dev\n  resources:\n  - batchjobs\n  verbs:\n  - get\n  - list\n  - watch\n- apiGroups:\n  - batch.cortex.dev\n  resources:\n  - batchjobs/status\n  verbs:\n  - get\n"
  },
  {
    "path": "pkg/crds/config/rbac/kustomization.yaml",
    "content": "resources:\n# All RBAC will be applied under this service account in\n# the deployment namespace. You may comment out this resource\n# if your manager will use a service account that exists at\n# runtime. Be sure to update RoleBinding and ClusterRoleBinding\n# subjects if changing service account names.\n- service_account.yaml\n- role.yaml\n- role_binding.yaml\n- leader_election_role.yaml\n- leader_election_role_binding.yaml\n# Comment the following 4 lines if you want to disable\n# the auth proxy (https://github.com/brancz/kube-rbac-proxy)\n# which protects your /metrics endpoint.\n- auth_proxy_service.yaml\n- auth_proxy_role.yaml\n- auth_proxy_role_binding.yaml\n- auth_proxy_client_clusterrole.yaml\n"
  },
  {
    "path": "pkg/crds/config/rbac/leader_election_role.yaml",
    "content": "# permissions to do leader election.\napiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n  name: leader-election-role\nrules:\n- apiGroups:\n  - \"\"\n  resources:\n  - configmaps\n  verbs:\n  - get\n  - list\n  - watch\n  - create\n  - update\n  - patch\n  - delete\n- apiGroups:\n  - coordination.k8s.io\n  resources:\n  - leases\n  verbs:\n  - get\n  - list\n  - watch\n  - create\n  - update\n  - patch\n  - delete\n- apiGroups:\n  - \"\"\n  resources:\n  - events\n  verbs:\n  - create\n  - patch\n"
  },
  {
    "path": "pkg/crds/config/rbac/leader_election_role_binding.yaml",
    "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n  name: leader-election-rolebinding\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: Role\n  name: leader-election-role\nsubjects:\n- kind: ServiceAccount\n  name: controller-manager\n  namespace: system\n"
  },
  {
    "path": "pkg/crds/config/rbac/role.yaml",
    "content": "---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n  creationTimestamp: null\n  name: manager-role\nrules:\n- apiGroups:\n  - \"\"\n  resources:\n  - configmaps\n  verbs:\n  - create\n  - get\n  - list\n  - watch\n- apiGroups:\n  - \"\"\n  resources:\n  - pods\n  verbs:\n  - get\n  - list\n  - watch\n- apiGroups:\n  - batch\n  resources:\n  - jobs\n  verbs:\n  - create\n  - get\n  - list\n  - patch\n  - update\n  - watch\n- apiGroups:\n  - batch.cortex.dev\n  resources:\n  - batchjobs\n  verbs:\n  - create\n  - delete\n  - get\n  - list\n  - patch\n  - update\n  - watch\n- apiGroups:\n  - batch.cortex.dev\n  resources:\n  - batchjobs/finalizers\n  verbs:\n  - update\n- apiGroups:\n  - batch.cortex.dev\n  resources:\n  - batchjobs/status\n  verbs:\n  - get\n  - patch\n  - update\n"
  },
  {
    "path": "pkg/crds/config/rbac/role_binding.yaml",
    "content": "apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n  name: manager-rolebinding\nroleRef:\n  apiGroup: rbac.authorization.k8s.io\n  kind: ClusterRole\n  name: manager-role\nsubjects:\n- kind: ServiceAccount\n  name: controller-manager\n  namespace: system\n"
  },
  {
    "path": "pkg/crds/config/rbac/service_account.yaml",
    "content": "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: controller-manager\n  namespace: system\n"
  },
  {
    "path": "pkg/crds/config/samples/batch_v1alpha1_batchjob.yaml",
    "content": "apiVersion: batch.cortex.dev/v1alpha1\nkind: BatchJob\nmetadata:\n  name: \"123456\"\nspec:\n  api_name: \"image-classifier\"\n  api_id: \"123456\"\n  workers: 1\n  config: |\n    dest_s3_dir: s3://abc/123\n  ttl: \"10s\"\n"
  },
  {
    "path": "pkg/crds/controllers/batch/batchjob_controller.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchcontrollers\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/crds/controllers\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/go-logr/logr\"\n\tpromv1 \"github.com/prometheus/client_golang/api/prometheus/v1\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\tctrl \"sigs.k8s.io/controller-runtime\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\nconst (\n\t_sqsFinalizer                 = \"sqs.finalizers.batch.cortex.dev\"\n\t_s3Finalizer                  = \"s3.finalizers.batch.cortex.dev\"\n\t_completedTimestampAnnotation = \"batch.cortex.dev/completed_timestamp\"\n)\n\n// BatchJobReconciler reconciles a BatchJob object\ntype BatchJobReconciler struct {\n\tclient.Client\n\tConfig        BatchJobReconcilerConfig\n\tLog           logr.Logger\n\tAWS           *awslib.Client\n\tClusterConfig *clusterconfig.Config\n\tPrometheus    promv1.API\n\tScheme        *runtime.Scheme\n}\n\n// +kubebuilder:rbac:groups=batch.cortex.dev,resources=batchjobs,verbs=get;list;watch;create;update;patch;delete\n// +kubebuilder:rbac:groups=batch.cortex.dev,resources=batchjobs/status,verbs=get;update;patch\n// +kubebuilder:rbac:groups=batch.cortex.dev,resources=batchjobs/finalizers,verbs=update\n// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch\n// +kubebuilder:rbac:groups=\"\",resources=pods,verbs=get;list;watch\n// +kubebuilder:rbac:groups=\"\",resources=configmaps,verbs=create;get;list;watch\n\n// Reconcile is part of the main kubernetes reconciliation loop which aims to\n// move the current state of the cluster closer to the desired state.\nfunc (r *BatchJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {\n\tlog := r.Log.WithValues(\"cortex.labels\",\n\t\tmap[string]string{\n\t\t\t\"batchjob\": req.NamespacedName.String(),\n\t\t\t\"apiKind\":  userconfig.BatchAPIKind.String(),\n\t\t},\n\t)\n\n\t// Step 1: get resource from request\n\tbatchJob := batch.BatchJob{}\n\tlog.V(1).Info(\"retrieving resource\")\n\tif err := r.Get(ctx, req.NamespacedName, &batchJob); err != nil {\n\t\tif !kerrors.IsNotFound(err) {\n\t\t\tlog.Error(err, \"failed to retrieve resource\")\n\t\t}\n\t\treturn ctrl.Result{}, client.IgnoreNotFound(err)\n\t}\n\n\tlog = r.Log.WithValues(\"cortex.labels\",\n\t\tmap[string]string{\n\t\t\t\"batchjob\": req.NamespacedName.String(),\n\t\t\t\"apiKind\":  userconfig.BatchAPIKind.String(),\n\t\t\t\"apiName\":  batchJob.Spec.APIName,\n\t\t\t\"apiID\":    batchJob.Spec.APIID,\n\t\t\t\"jobID\":    batchJob.Name,\n\t\t},\n\t)\n\t// Step 2: create finalizer or handle deletion\n\tif batchJob.ObjectMeta.DeletionTimestamp.IsZero() {\n\t\t// The object is not being deleted, so we add our finalizer if it does not exist yet,\n\t\tif !slices.HasString(batchJob.ObjectMeta.Finalizers, _sqsFinalizer) &&\n\t\t\t!slices.HasString(batchJob.ObjectMeta.Finalizers, _s3Finalizer) {\n\t\t\tlog.V(1).Info(\"adding finalizers\")\n\t\t\tbatchJob.ObjectMeta.Finalizers = append(batchJob.ObjectMeta.Finalizers, _sqsFinalizer, _s3Finalizer)\n\t\t\tif err := r.Update(ctx, &batchJob); err != nil {\n\t\t\t\tlog.Error(err, \"failed to add finalizers to resource\")\n\t\t\t\treturn ctrl.Result{}, err\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// The object is being deleted\n\t\tif slices.HasString(batchJob.ObjectMeta.Finalizers, _sqsFinalizer) {\n\t\t\t// our finalizer is present, so lets handle any external dependency\n\t\t\tlog.V(1).Info(\"deleting SQS queue\")\n\t\t\tif err := r.deleteSQSQueue(batchJob); err != nil {\n\t\t\t\tlog.Error(err, \"failed to delete SQS queue\")\n\t\t\t\treturn ctrl.Result{}, err\n\t\t\t}\n\n\t\t\tlog.V(1).Info(\"removing sqs finalizer\")\n\t\t\t// remove our finalizer from the list and update it.\n\t\t\tbatchJob.ObjectMeta.Finalizers = slices.RemoveString(batchJob.ObjectMeta.Finalizers, _sqsFinalizer)\n\t\t\tif err := r.Update(ctx, &batchJob); err != nil {\n\t\t\t\tlog.Error(err, \"failed to remove sqs finalizer from resource\")\n\t\t\t\treturn ctrl.Result{}, err\n\t\t\t}\n\t\t\treturn ctrl.Result{}, nil // return here because the status update will trigger another reconcile\n\t\t}\n\n\t\tif slices.HasString(batchJob.ObjectMeta.Finalizers, _s3Finalizer) {\n\t\t\tlog.V(1).Info(\"persisting job to S3\")\n\t\t\tif err := r.persistJobToS3(batchJob); err != nil {\n\t\t\t\tlog.Error(err, \"failed to persist job to S3\")\n\t\t\t\treturn ctrl.Result{}, err\n\t\t\t}\n\n\t\t\tlog.V(1).Info(\"removing S3 finalizer\")\n\t\t\tbatchJob.ObjectMeta.Finalizers = slices.RemoveString(batchJob.ObjectMeta.Finalizers, _s3Finalizer)\n\t\t\tif err := r.Update(ctx, &batchJob); err != nil {\n\t\t\t\tlog.Error(err, \"failed to remove S3 finalizer from resource\")\n\t\t\t\treturn ctrl.Result{}, err\n\t\t\t}\n\t\t\treturn ctrl.Result{}, nil // return here because the status update will trigger another reconcile\n\t\t}\n\n\t\treturn ctrl.Result{}, nil\n\t}\n\n\t// Step 3: Update Status\n\tlog.V(1).Info(\"checking if queue exists\")\n\tqueueExists, err := r.checkIfQueueExists(batchJob)\n\tif err != nil {\n\t\tlog.Error(err, \"failed to check if queue exists\")\n\t\treturn ctrl.Result{}, err\n\t}\n\n\tlog.V(1).Info(\"getting configmap\")\n\tconfigMap, err := r.getConfigMap(ctx, batchJob)\n\tif err != nil && !kerrors.IsNotFound(err) {\n\t\tlog.Error(err, \"failed to get configmap\")\n\t\treturn ctrl.Result{}, err\n\t}\n\n\tlog.V(1).Info(\"getting worker job\")\n\tworkerJob, err := r.getWorkerJob(ctx, batchJob)\n\tif err != nil && !kerrors.IsNotFound(err) {\n\t\tlog.Error(err, \"failed to get worker job\")\n\t\treturn ctrl.Result{}, err\n\t}\n\n\tlog.V(1).Info(\"checking enqueuing status\")\n\tenqueuerJob, enqueuingStatus, err := r.checkEnqueuingStatus(ctx, batchJob)\n\tif err != nil {\n\t\tlog.Error(err, \"failed to check enqueuing status\")\n\t\treturn ctrl.Result{}, err\n\t}\n\n\tvar totalBatchCount int\n\tif enqueuingStatus == batch.EnqueuingDone {\n\t\ttotalBatchCount, err = r.Config.GetTotalBatchCount(r, batchJob)\n\t}\n\n\tworkerJobExists := workerJob != nil\n\tconfigMapExists := configMap != nil\n\tstatusInfo := batchJobStatusInfo{\n\t\tQueueExists:     queueExists,\n\t\tEnqueuingStatus: enqueuingStatus,\n\t\tEnqueuerJob:     enqueuerJob,\n\t\tWorkerJob:       workerJob,\n\t\tTotalBatchCount: totalBatchCount,\n\t}\n\n\tlog.V(1).Info(\"status data successfully acquired\",\n\t\t\"queueExists\", queueExists,\n\t\t\"configMapExists\", configMapExists,\n\t\t\"enqueuingStatus\", enqueuingStatus,\n\t\t\"workerJobExists\", workerJobExists,\n\t)\n\n\tlog.V(1).Info(\"updating status\")\n\tif err = r.updateStatus(ctx, &batchJob, statusInfo); err != nil {\n\t\tif controllers.IsOptimisticLockError(err) {\n\t\t\tlog.Info(\"conflict during status update, retrying\")\n\t\t\treturn ctrl.Result{Requeue: true}, nil\n\t\t}\n\t\tlog.Error(err, \"failed to update status\")\n\t\treturn ctrl.Result{}, err\n\t}\n\n\tlog.V(1).Info(\"current job status\", \"jobStatus\", batchJob.Status.Status)\n\n\t// Step 4: Add a completion timestamp annotation if job is in a completed state\n\tvar completedTimestamp *time.Time\n\tif batchJob.Status.Status.IsCompleted() {\n\t\tcompletedTimestampStr, completedTimestampExists := batchJob.Annotations[_completedTimestampAnnotation]\n\t\tif !completedTimestampExists {\n\t\t\tif err = r.updateCompletedTimestamp(ctx, &batchJob); err != nil {\n\t\t\t\tlog.Error(err, \"failed to update completed timestamp annotation\")\n\t\t\t\treturn ctrl.Result{}, err\n\t\t\t}\n\t\t\treturn ctrl.Result{}, nil\n\t\t}\n\n\t\tts, err := time.Parse(time.RFC3339, completedTimestampStr)\n\t\tif err != nil {\n\t\t\tlog.Error(err, \"failed to parse completed timestamp string\")\n\t\t\treturn ctrl.Result{}, err\n\t\t}\n\t\tcompletedTimestamp = &ts\n\t}\n\n\t// Step 5: Create resources\n\tvar queueURL string\n\tif !queueExists {\n\t\tlog.Info(\"creating queue\")\n\t\tqueueURL, err = r.createQueue(batchJob)\n\t\tif err != nil {\n\t\t\tlog.Error(err, \"failed to create queue\")\n\t\t\treturn ctrl.Result{}, err\n\t\t}\n\t} else {\n\t\tqueueURL = r.getQueueURL(batchJob)\n\t}\n\n\tswitch enqueuingStatus {\n\tcase batch.EnqueuingNotStarted:\n\t\tlog.Info(\"enqueuing payload\")\n\t\tif err = r.enqueuePayload(ctx, batchJob, queueURL); err != nil {\n\t\t\tlog.Error(err, \"failed to start enqueuing the payload\")\n\t\t\treturn ctrl.Result{}, err\n\t\t}\n\t\treturn ctrl.Result{}, nil\n\tcase batch.EnqueuingInProgress:\n\t\t// wait for enqueuing process to be reach a final state (done|failed)\n\t\treturn ctrl.Result{}, nil\n\tcase batch.EnqueuingFailed:\n\t\tlog.Info(\"failed to enqueue payload\")\n\tcase batch.EnqueuingDone:\n\t\tif !configMapExists {\n\t\t\tlog.V(1).Info(\"creating worker configmap\")\n\t\t\tif err = r.createWorkerConfigMap(ctx, batchJob, queueURL); err != nil {\n\t\t\t\tlog.Error(err, \"failed to create worker configmap\")\n\t\t\t\treturn ctrl.Result{}, err\n\t\t\t}\n\n\t\t}\n\t\tif !workerJobExists {\n\t\t\tlog.Info(\"creating worker job\")\n\t\t\tif err = r.createWorkerJob(ctx, batchJob, queueURL); err != nil {\n\t\t\t\tlog.Error(err, \"failed to create worker job\")\n\t\t\t\treturn ctrl.Result{}, err\n\t\t\t}\n\t\t}\n\t}\n\n\t// Step 6: Delete self if TTL is enabled and reached a final state\n\tif batchJob.Spec.TTL != nil && completedTimestamp != nil {\n\t\tafterFinishedDuration := time.Since(*completedTimestamp)\n\t\tif afterFinishedDuration >= batchJob.Spec.TTL.Duration {\n\t\t\tif err = r.Delete(ctx, &batchJob); err != nil {\n\t\t\t\treturn ctrl.Result{}, client.IgnoreNotFound(err)\n\t\t\t}\n\t\t\tlog.Info(\"TTL exceeded, deleting resource\")\n\t\t\treturn ctrl.Result{}, nil\n\t\t}\n\t\tlog.V(1).Info(\"scheduling reconciliation requeue\", \"time\", batchJob.Spec.TTL.Duration)\n\t\treturn ctrl.Result{RequeueAfter: batchJob.Spec.TTL.Duration}, nil\n\t}\n\n\treturn ctrl.Result{}, nil\n}\n\n// SetupWithManager sets up the controller with the Manager.\nfunc (r *BatchJobReconciler) SetupWithManager(mgr ctrl.Manager) error {\n\treturn ctrl.NewControllerManagedBy(mgr).\n\t\tFor(&batch.BatchJob{}).\n\t\tOwns(&kbatch.Job{}).\n\t\tComplete(r)\n}\n"
  },
  {
    "path": "pkg/crds/controllers/batch/batchjob_controller_config.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchcontrollers\n\nimport (\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/types/metrics\"\n)\n\n// BatchJobReconcilerConfig reconciler config for the BatchJob kind. Allows for mocking specific methods\ntype BatchJobReconcilerConfig struct {\n\tGetTotalBatchCount func(r *BatchJobReconciler, batchJob batch.BatchJob) (int, error)\n\tGetMetrics         func(r *BatchJobReconciler, batchJob batch.BatchJob) (metrics.BatchMetrics, error)\n\tSaveJobMetrics     func(r *BatchJobReconciler, batchJob batch.BatchJob) error\n\tSaveJobStatus      func(r *BatchJobReconciler, batchJob batch.BatchJob) error\n}\n\n// ApplyDefaults sets the defaults for BatchJobReconcilerConfig\nfunc (c BatchJobReconcilerConfig) ApplyDefaults() BatchJobReconcilerConfig {\n\tif c.GetTotalBatchCount == nil {\n\t\tc.GetTotalBatchCount = getTotalBatchCount\n\t}\n\n\tif c.GetMetrics == nil {\n\t\tc.GetMetrics = getMetrics\n\t}\n\n\tif c.SaveJobMetrics == nil {\n\t\tc.SaveJobMetrics = saveJobMetrics\n\t}\n\n\tif c.SaveJobStatus == nil {\n\t\tc.SaveJobStatus = saveJobStatus\n\t}\n\n\treturn c\n}\n"
  },
  {
    "path": "pkg/crds/controllers/batch/batchjob_controller_helpers.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchcontrollers\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/metrics\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\t\"github.com/cortexlabs/yaml\"\n\tcache \"github.com/patrickmn/go-cache\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tctrl \"sigs.k8s.io/controller-runtime\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\nconst (\n\t_enqueuerContainerName  = \"enqueuer\"\n\t_deadlineExceededReason = \"DeadlineExceeded\"\n\t_cacheDuration          = 60 * time.Second\n)\n\nvar totalBatchCountCache, apiSpecCache *cache.Cache\n\nfunc init() {\n\ttotalBatchCountCache = cache.New(_cacheDuration, _cacheDuration)\n\tapiSpecCache = cache.New(_cacheDuration, _cacheDuration)\n}\n\ntype batchJobStatusInfo struct {\n\tQueueExists     bool\n\tEnqueuingStatus batch.EnqueuingStatus\n\tEnqueuerJob     *kbatch.Job\n\tWorkerJob       *kbatch.Job\n\tTotalBatchCount int\n}\n\nfunc (r *BatchJobReconciler) getConfigMap(ctx context.Context, batchJob batch.BatchJob) (*kcore.ConfigMap, error) {\n\tvar configMap kcore.ConfigMap\n\terr := r.Get(ctx, client.ObjectKey{\n\t\tNamespace: batchJob.Namespace,\n\t\tName:      batchJob.Spec.APIName + \"-\" + batchJob.Name,\n\t}, &configMap)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, err\n\t}\n\n\treturn &configMap, nil\n}\n\nfunc (r *BatchJobReconciler) checkIfQueueExists(batchJob batch.BatchJob) (bool, error) {\n\tqueueName := r.getQueueName(batchJob)\n\tinput := &sqs.GetQueueUrlInput{\n\t\tQueueName: aws.String(queueName),\n\t}\n\t_, err := r.AWS.SQS().GetQueueUrl(input)\n\tif err != nil {\n\t\tif aerr, ok := err.(awserr.Error); ok {\n\t\t\tif aerr.Code() == sqs.ErrCodeQueueDoesNotExist {\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t}\n\t\treturn false, err\n\t}\n\treturn true, nil\n}\n\nfunc (r *BatchJobReconciler) createQueue(batchJob batch.BatchJob) (string, error) {\n\tqueueName := r.getQueueName(batchJob)\n\n\ttags := map[string]string{\n\t\tclusterconfig.ClusterNameTag: r.ClusterConfig.ClusterName,\n\t\t\"apiName\":                    batchJob.Spec.APIName,\n\t\t\"apiID\":                      batchJob.Spec.APIID,\n\t\t\"jobID\":                      batchJob.Name,\n\t}\n\n\tattributes := map[string]string{\n\t\tsqs.QueueAttributeNameFifoQueue:         \"true\",\n\t\tsqs.QueueAttributeNameVisibilityTimeout: \"60\",\n\t}\n\n\tif batchJob.Spec.DeadLetterQueue != nil {\n\t\tredrivePolicy := map[string]string{\n\t\t\t\"deadLetterTargetArn\": batchJob.Spec.DeadLetterQueue.ARN,\n\t\t\t\"maxReceiveCount\":     s.Int32(batchJob.Spec.DeadLetterQueue.MaxReceiveCount),\n\t\t}\n\n\t\tredrivePolicyJSONBytes, err := libjson.Marshal(redrivePolicy)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tattributes[sqs.QueueAttributeNameRedrivePolicy] = string(redrivePolicyJSONBytes)\n\t}\n\n\tinput := &sqs.CreateQueueInput{\n\t\tAttributes: aws.StringMap(attributes),\n\t\tQueueName:  aws.String(queueName),\n\t\tTags:       aws.StringMap(tags),\n\t}\n\toutput, err := r.AWS.SQS().CreateQueue(input)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn *output.QueueUrl, nil\n}\n\nfunc (r *BatchJobReconciler) getQueueURL(batchJob batch.BatchJob) string {\n\t// e.g. https://sqs.<region>.amazonaws.com/<account_id>/<queue_name>\n\treturn fmt.Sprintf(\n\t\t\"https://sqs.%s.amazonaws.com/%s/%s\",\n\t\tr.ClusterConfig.Region, r.ClusterConfig.AccountID, r.getQueueName(batchJob),\n\t)\n}\n\nfunc (r *BatchJobReconciler) getQueueName(batchJob batch.BatchJob) string {\n\t// cx_<hash of cluster name>_b_<api_name>_<job_id>.fifo\n\treturn clusterconfig.SQSNamePrefix(r.ClusterConfig.ClusterName) + \"b\" +\n\t\tclusterconfig.SQSQueueDelimiter + batchJob.Spec.APIName +\n\t\tclusterconfig.SQSQueueDelimiter + batchJob.Name + \".fifo\"\n}\n\nfunc (r *BatchJobReconciler) checkEnqueuingStatus(ctx context.Context, batchJob batch.BatchJob) (*kbatch.Job, batch.EnqueuingStatus, error) {\n\tvar enqueuerJob kbatch.Job\n\tif err := r.Get(ctx,\n\t\tclient.ObjectKey{\n\t\t\tNamespace: batchJob.Namespace,\n\t\t\tName:      batchJob.Spec.APIName + \"-\" + batchJob.Name + \"-enqueuer\",\n\t\t},\n\t\t&enqueuerJob,\n\t); err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, batch.EnqueuingNotStarted, nil\n\t\t}\n\t\treturn nil, \"\", err\n\t}\n\n\tenqueuerStatus := enqueuerJob.Status\n\tswitch {\n\tcase enqueuerStatus.Failed > 0:\n\t\treturn &enqueuerJob, batch.EnqueuingFailed, nil\n\tcase enqueuerStatus.Succeeded > 0:\n\t\treturn &enqueuerJob, batch.EnqueuingDone, nil\n\tcase enqueuerStatus.Active > 0:\n\t\treturn &enqueuerJob, batch.EnqueuingInProgress, nil\n\t}\n\n\treturn &enqueuerJob, batch.EnqueuingInProgress, nil\n}\n\nfunc (r *BatchJobReconciler) enqueuePayload(ctx context.Context, batchJob batch.BatchJob, queueURL string) error {\n\tenqueuerJob, err := r.desiredEnqueuerJob(batchJob, queueURL)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif err = r.Create(ctx, enqueuerJob); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (r *BatchJobReconciler) createWorkerConfigMap(ctx context.Context, batchJob batch.BatchJob, queueURL string) error {\n\tapiSpec, err := r.getAPISpec(batchJob)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to get API spec\")\n\t}\n\n\tjobSpec, err := r.ConvertControllerBatchToJobSpec(batchJob, *apiSpec, queueURL)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to convert controller batch job to operator batch job\")\n\t}\n\n\tconfigMapConfig := workloads.ConfigMapConfig{\n\t\tBatchJob: &jobSpec,\n\t\tProbes:   batchJob.Spec.Probes,\n\t}\n\n\tconfigMapData, err := configMapConfig.GenerateConfigMapData()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to generate config map data\")\n\t}\n\n\tconfigMap, err := r.desiredConfigMap(batchJob, configMapData)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to get desired configmap spec\")\n\t}\n\n\tif err := r.Create(ctx, configMap); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (r *BatchJobReconciler) createWorkerJob(ctx context.Context, batchJob batch.BatchJob, queueURL string) error {\n\tapiSpec, err := r.getAPISpec(batchJob)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to get API spec\")\n\t}\n\n\tjobSpec, err := r.uploadJobSpec(batchJob, *apiSpec, queueURL)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to upload job spec\")\n\t}\n\n\tworkerJob, err := r.desiredWorkerJob(batchJob, *apiSpec, *jobSpec)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to get desired worker job\")\n\t}\n\n\tif err = r.Create(ctx, workerJob); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (r *BatchJobReconciler) desiredEnqueuerJob(batchJob batch.BatchJob, queueURL string) (*kbatch.Job, error) {\n\tjob := k8s.Job(\n\t\t&k8s.JobSpec{\n\t\t\tName:        batchJob.Spec.APIName + \"-\" + batchJob.Name + \"-enqueuer\",\n\t\t\tNamespace:   batchJob.Namespace,\n\t\t\tParallelism: 1,\n\t\t\tLabels: map[string]string{\n\t\t\t\t\"apiKind\":          userconfig.BatchAPIKind.String(),\n\t\t\t\t\"apiName\":          batchJob.Spec.APIName,\n\t\t\t\t\"apiID\":            batchJob.Spec.APIID,\n\t\t\t\t\"jobID\":            batchJob.Name,\n\t\t\t\t\"cortex.dev/api\":   \"true\",\n\t\t\t\t\"cortex.dev/batch\": \"enqueuer\",\n\t\t\t},\n\t\t\tPodSpec: k8s.PodSpec{\n\t\t\t\tLabels: map[string]string{\n\t\t\t\t\t\"apiKind\":          userconfig.BatchAPIKind.String(),\n\t\t\t\t\t\"apiName\":          batchJob.Spec.APIName,\n\t\t\t\t\t\"apiID\":            batchJob.Spec.APIID,\n\t\t\t\t\t\"jobID\":            batchJob.Name,\n\t\t\t\t\t\"cortex.dev/api\":   \"true\",\n\t\t\t\t\t\"cortex.dev/batch\": \"enqueuer\",\n\t\t\t\t},\n\t\t\t\tAnnotations: map[string]string{\n\t\t\t\t\t\"traffic.sidecar.istio.io/excludeOutboundIPRanges\": \"0.0.0.0/0\",\n\t\t\t\t\t\"cluster-autoscaler.kubernetes.io/safe-to-evict\":   \"false\",\n\t\t\t\t},\n\t\t\t\tK8sPodSpec: kcore.PodSpec{\n\t\t\t\t\tRestartPolicy: kcore.RestartPolicyNever,\n\t\t\t\t\tContainers: []kcore.Container{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName:  _enqueuerContainerName,\n\t\t\t\t\t\t\tImage: r.ClusterConfig.ImageEnqueuer,\n\t\t\t\t\t\t\tArgs: []string{\n\t\t\t\t\t\t\t\t\"-cluster-uid\", r.ClusterConfig.ClusterUID,\n\t\t\t\t\t\t\t\t\"-region\", r.ClusterConfig.Region,\n\t\t\t\t\t\t\t\t\"-bucket\", r.ClusterConfig.Bucket,\n\t\t\t\t\t\t\t\t\"-queue\", queueURL,\n\t\t\t\t\t\t\t\t\"-apiName\", batchJob.Spec.APIName,\n\t\t\t\t\t\t\t\t\"-jobID\", batchJob.Name,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tEnv:             workloads.BaseEnvVars,\n\t\t\t\t\t\t\tEnvFrom:         workloads.BaseClusterEnvVars(),\n\t\t\t\t\t\t\tImagePullPolicy: kcore.PullAlways,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tNodeSelector:       workloads.NodeSelectors(),\n\t\t\t\t\tTolerations:        workloads.GenerateResourceTolerations(),\n\t\t\t\t\tAffinity:           workloads.GenerateNodeAffinities(batchJob.Spec.NodeGroups),\n\t\t\t\t\tServiceAccountName: workloads.ServiceAccountName,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t)\n\n\tif err := ctrl.SetControllerReference(&batchJob, job, r.Scheme); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn job, nil\n}\n\nfunc (r *BatchJobReconciler) desiredWorkerJob(batchJob batch.BatchJob, apiSpec spec.API, jobSpec spec.BatchJob) (*kbatch.Job, error) {\n\tcontainers, volumes := workloads.BatchContainers(apiSpec, &jobSpec)\n\n\tjob := k8s.Job(\n\t\t&k8s.JobSpec{\n\t\t\tName:        batchJob.Spec.APIName + \"-\" + batchJob.Name,\n\t\t\tNamespace:   batchJob.Namespace,\n\t\t\tParallelism: batchJob.Spec.Workers,\n\t\t\tLabels: map[string]string{\n\t\t\t\t\"apiKind\":          userconfig.BatchAPIKind.String(),\n\t\t\t\t\"apiName\":          batchJob.Spec.APIName,\n\t\t\t\t\"apiID\":            batchJob.Spec.APIID,\n\t\t\t\t\"specID\":           apiSpec.SpecID,\n\t\t\t\t\"podID\":            apiSpec.PodID,\n\t\t\t\t\"jobID\":            batchJob.Name,\n\t\t\t\t\"cortex.dev/api\":   \"true\",\n\t\t\t\t\"cortex.dev/batch\": \"worker\",\n\t\t\t},\n\t\t\tPodSpec: k8s.PodSpec{\n\t\t\t\tLabels: map[string]string{\n\t\t\t\t\t\"apiKind\":          userconfig.BatchAPIKind.String(),\n\t\t\t\t\t\"apiName\":          batchJob.Spec.APIName,\n\t\t\t\t\t\"apiID\":            batchJob.Spec.APIID,\n\t\t\t\t\t\"specID\":           apiSpec.SpecID,\n\t\t\t\t\t\"podID\":            apiSpec.PodID,\n\t\t\t\t\t\"jobID\":            batchJob.Name,\n\t\t\t\t\t\"cortex.dev/api\":   \"true\",\n\t\t\t\t\t\"cortex.dev/batch\": \"worker\",\n\t\t\t\t},\n\t\t\t\tAnnotations: map[string]string{\n\t\t\t\t\t\"traffic.sidecar.istio.io/excludeOutboundIPRanges\": \"0.0.0.0/0\",\n\t\t\t\t\t\"cluster-autoscaler.kubernetes.io/safe-to-evict\":   \"false\",\n\t\t\t\t},\n\t\t\t\tK8sPodSpec: kcore.PodSpec{\n\t\t\t\t\tInitContainers: []kcore.Container{\n\t\t\t\t\t\tworkloads.KubexitInitContainer(),\n\t\t\t\t\t},\n\t\t\t\t\tContainers:         containers,\n\t\t\t\t\tVolumes:            volumes,\n\t\t\t\t\tRestartPolicy:      kcore.RestartPolicyNever,\n\t\t\t\t\tNodeSelector:       workloads.NodeSelectors(),\n\t\t\t\t\tAffinity:           workloads.GenerateNodeAffinities(batchJob.Spec.NodeGroups),\n\t\t\t\t\tTolerations:        workloads.GenerateResourceTolerations(),\n\t\t\t\t\tServiceAccountName: workloads.ServiceAccountName,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t)\n\n\tif batchJob.Spec.Timeout != nil {\n\t\tjob.Spec.ActiveDeadlineSeconds = pointer.Int64(int64(batchJob.Spec.Timeout.Seconds()))\n\t}\n\n\tif err := ctrl.SetControllerReference(&batchJob, job, r.Scheme); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn job, nil\n}\n\nfunc (r *BatchJobReconciler) desiredConfigMap(batchJob batch.BatchJob, data map[string]string) (*kcore.ConfigMap, error) {\n\tconfigMap := k8s.ConfigMap(&k8s.ConfigMapSpec{\n\t\tName: batchJob.Spec.APIName + \"-\" + batchJob.Name,\n\t\tData: data,\n\t\tLabels: map[string]string{\n\t\t\t\"apiKind\":          userconfig.BatchAPIKind.String(),\n\t\t\t\"apiName\":          batchJob.Spec.APIName,\n\t\t\t\"apiID\":            batchJob.Spec.APIID,\n\t\t\t\"jobID\":            batchJob.Name,\n\t\t\t\"cortex.dev/api\":   \"true\",\n\t\t\t\"cortex.dev/batch\": \"config\",\n\t\t},\n\t})\n\tconfigMap.Namespace = batchJob.Namespace\n\n\tif err := ctrl.SetControllerReference(&batchJob, configMap, r.Scheme); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn configMap, nil\n}\n\nfunc (r *BatchJobReconciler) getAPISpec(batchJob batch.BatchJob) (*spec.API, error) {\n\tapiSpecKey := spec.Key(batchJob.Spec.APIName, batchJob.Spec.APIID, r.ClusterConfig.ClusterUID)\n\tcachedAPISpec, found := apiSpecCache.Get(apiSpecKey)\n\n\tvar apiSpec spec.API\n\tif found {\n\t\tapiSpec = cachedAPISpec.(spec.API)\n\t\treturn &apiSpec, nil\n\t}\n\n\tapiSpecBytes, err := r.AWS.ReadBytesFromS3(r.ClusterConfig.Bucket, apiSpecKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := json.Unmarshal(apiSpecBytes, &apiSpec); err != nil {\n\t\treturn nil, err\n\t}\n\n\tapiSpecCache.Set(apiSpecKey, apiSpec, _cacheDuration)\n\n\treturn &apiSpec, nil\n}\n\nfunc (r *BatchJobReconciler) getWorkerJob(ctx context.Context, batchJob batch.BatchJob) (*kbatch.Job, error) {\n\tworkerName := batchJob.Spec.APIName + \"-\" + batchJob.Name\n\n\tvar job kbatch.Job\n\terr := r.Get(ctx, client.ObjectKey{Namespace: batchJob.Namespace, Name: workerName}, &job)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, err\n\t}\n\n\treturn &job, nil\n}\n\nfunc (r *BatchJobReconciler) getWorkerJobPods(ctx context.Context, batchJob batch.BatchJob) ([]kcore.Pod, error) {\n\tworkerJobPods := kcore.PodList{}\n\tif err := r.List(ctx, &workerJobPods,\n\t\tclient.InNamespace(consts.DefaultNamespace),\n\t\tclient.MatchingLabels{\n\t\t\t\"jobID\":            batchJob.Name,\n\t\t\t\"apiName\":          batchJob.Spec.APIName,\n\t\t\t\"apiID\":            batchJob.Spec.APIID,\n\t\t\t\"cortex.dev/batch\": \"worker\",\n\t\t},\n\t); err != nil {\n\t\treturn nil, err\n\t}\n\treturn workerJobPods.Items, nil\n}\n\nfunc (r *BatchJobReconciler) updateStatus(ctx context.Context, batchJob *batch.BatchJob, statusInfo batchJobStatusInfo) error {\n\tbatchJob.Status.ID = batchJob.Name\n\n\tif statusInfo.QueueExists {\n\t\tbatchJob.Status.QueueURL = r.getQueueURL(*batchJob)\n\t}\n\n\tswitch statusInfo.EnqueuingStatus {\n\tcase batch.EnqueuingNotStarted:\n\t\tbatchJob.Status.Status = status.JobPending\n\tcase batch.EnqueuingInProgress:\n\t\tbatchJob.Status.Status = status.JobEnqueuing\n\tcase batch.EnqueuingFailed:\n\t\tbatchJob.Status.Status = status.JobEnqueueFailed\n\t\tbatchJob.Status.EndTime = statusInfo.EnqueuerJob.Status.CompletionTime\n\tcase batch.EnqueuingDone:\n\t\tbatchJob.Status.TotalBatchCount = statusInfo.TotalBatchCount\n\t}\n\n\tworkerJobPods, err := r.getWorkerJobPods(ctx, *batchJob)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to retrieve worker pods\")\n\t}\n\n\tworker := statusInfo.WorkerJob\n\tif worker != nil {\n\t\tbatchJob.Status.EndTime = worker.Status.CompletionTime // assign right away, because it's a pointer\n\n\t\tif batchJob.Status.EndTime == nil {\n\t\t\tcompletedTimestampStr, completedTimestampExists := batchJob.Annotations[_completedTimestampAnnotation]\n\t\t\tif completedTimestampExists {\n\t\t\t\tts, err := time.Parse(time.RFC3339, completedTimestampStr)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn errors.Wrap(err, \"failed to parse completed timestamp string\")\n\t\t\t\t}\n\t\t\t\tcompletedTime := v1.NewTime(ts)\n\t\t\t\tbatchJob.Status.EndTime = &completedTime\n\t\t\t}\n\t\t}\n\n\t\tif worker.Status.Failed == batchJob.Spec.Workers {\n\t\t\tbatchJobStatus := status.JobWorkerError\n\t\t\tfor _, condition := range worker.Status.Conditions {\n\t\t\t\tif condition.Reason == _deadlineExceededReason {\n\t\t\t\t\tbatchJobStatus = status.JobTimedOut\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor i := range workerJobPods {\n\t\t\t\tif k8s.WasPodOOMKilled(&workerJobPods[i]) {\n\t\t\t\t\tbatchJobStatus = status.JobWorkerOOM\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tbatchJob.Status.Status = batchJobStatus\n\t\t} else if worker.Status.Succeeded == batchJob.Spec.Workers {\n\t\t\tbatchJob.Status.Status = status.JobSucceeded\n\n\t\t\tjobMetrics, err := r.Config.GetMetrics(r, *batchJob)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tif jobMetrics.Failed > 0 {\n\t\t\t\tbatchJob.Status.Status = status.JobCompletedWithFailures\n\t\t\t}\n\n\t\t} else if worker.Status.Active > 0 {\n\t\t\tbatchJob.Status.Status = status.JobRunning\n\t\t}\n\n\t\tworkerCounts := getReplicaCounts(workerJobPods)\n\t\tbatchJob.Status.WorkerCounts = &workerCounts\n\t}\n\n\tif err := r.Status().Update(ctx, batchJob); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (r *BatchJobReconciler) deleteSQSQueue(batchJob batch.BatchJob) error {\n\tqueueURL := r.getQueueURL(batchJob)\n\tinput := sqs.DeleteQueueInput{QueueUrl: aws.String(queueURL)}\n\tif _, err := r.AWS.SQS().DeleteQueue(&input); err != nil {\n\t\tif awsErr, ok := err.(awserr.Error); ok {\n\t\t\tif awsErr.Code() == sqs.ErrCodeQueueDoesNotExist {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (r *BatchJobReconciler) uploadJobSpec(batchJob batch.BatchJob, api spec.API, queueURL string) (*spec.BatchJob, error) {\n\tjobSpec, err := r.ConvertControllerBatchToJobSpec(batchJob, api, queueURL)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err = r.AWS.UploadJSONToS3(&jobSpec, r.ClusterConfig.Bucket, r.jobSpecKey(batchJob)); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &jobSpec, nil\n}\n\nfunc (r *BatchJobReconciler) ConvertControllerBatchToJobSpec(batchJob batch.BatchJob, api spec.API, queueURL string) (spec.BatchJob, error) {\n\tvar deadLetterQueue *spec.SQSDeadLetterQueue\n\tif batchJob.Spec.DeadLetterQueue != nil {\n\t\tdeadLetterQueue = &spec.SQSDeadLetterQueue{\n\t\t\tARN:             batchJob.Spec.DeadLetterQueue.ARN,\n\t\t\tMaxReceiveCount: int(batchJob.Spec.DeadLetterQueue.MaxReceiveCount),\n\t\t}\n\t}\n\n\tvar config map[string]interface{}\n\tif batchJob.Spec.Config != nil {\n\t\terr := yaml.Unmarshal([]byte(*batchJob.Spec.Config), &config)\n\t\tif err != nil {\n\t\t\treturn spec.BatchJob{}, errors.Wrap(err, \"failed to unmarshal job spec config\")\n\t\t}\n\t}\n\n\tvar timeout *int\n\tif batchJob.Spec.Timeout != nil {\n\t\ttimeout = pointer.Int(int(batchJob.Spec.Timeout.Seconds()))\n\t}\n\n\ttotalBatchCount, err := r.Config.GetTotalBatchCount(r, batchJob)\n\tif err != nil {\n\t\treturn spec.BatchJob{}, errors.Wrap(err, \"failed to get total batch count\")\n\t}\n\n\treturn spec.BatchJob{\n\t\tJobKey: spec.JobKey{\n\t\t\tID:      batchJob.Status.ID,\n\t\t\tAPIName: batchJob.Spec.APIName,\n\t\t\tKind:    userconfig.BatchAPIKind,\n\t\t},\n\t\tRuntimeBatchJobConfig: spec.RuntimeBatchJobConfig{\n\t\t\tWorkers:            int(batchJob.Spec.Workers),\n\t\t\tSQSDeadLetterQueue: deadLetterQueue,\n\t\t\tConfig:             config,\n\t\t\tTimeout:            timeout,\n\t\t},\n\t\tAPIID:           api.ID,\n\t\tSQSUrl:          queueURL,\n\t\tStartTime:       batchJob.CreationTimestamp.Time,\n\t\tTotalBatchCount: totalBatchCount,\n\t}, nil\n}\n\nfunc (r *BatchJobReconciler) jobSpecKey(batchJob batch.BatchJob) string {\n\t// e.g. <cluster uid>/jobs/<job_api_kind>/<cortex version>/<api_name>/<job_id>/spec.json\n\treturn filepath.Join(\n\t\tr.ClusterConfig.ClusterUID,\n\t\t\"jobs\",\n\t\tuserconfig.BatchAPIKind.String(),\n\t\tconsts.CortexVersion,\n\t\tbatchJob.Spec.APIName,\n\t\tbatchJob.Name,\n\t\t\"spec.json\",\n\t)\n}\n\nfunc (r *BatchJobReconciler) updateCompletedTimestamp(ctx context.Context, batchJob *batch.BatchJob) error {\n\tts := time.Now().Format(time.RFC3339)\n\tif batchJob.Annotations != nil {\n\t\tbatchJob.Annotations[_completedTimestampAnnotation] = ts\n\t} else {\n\t\tbatchJob.Annotations = map[string]string{\n\t\t\t_completedTimestampAnnotation: ts,\n\t\t}\n\t}\n\tif err := r.Update(ctx, batchJob); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (r *BatchJobReconciler) persistJobToS3(batchJob batch.BatchJob) error {\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\treturn r.Config.SaveJobMetrics(r, batchJob)\n\t\t},\n\t\tfunc() error {\n\t\t\treturn r.Config.SaveJobStatus(r, batchJob)\n\t\t},\n\t)\n}\n\nfunc getTotalBatchCount(r *BatchJobReconciler, batchJob batch.BatchJob) (int, error) {\n\tkey := spec.JobBatchCountKey(r.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, batchJob.Spec.APIName, batchJob.Name)\n\tcachedTotalBatchCount, found := totalBatchCountCache.Get(key)\n\tvar totalBatchCount int\n\tif !found {\n\t\ttotalBatchCountBytes, err := r.AWS.ReadBytesFromS3(r.ClusterConfig.Bucket, key)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\ttotalBatchCount, err = strconv.Atoi(string(totalBatchCountBytes))\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t} else {\n\t\ttotalBatchCount = cachedTotalBatchCount.(int)\n\t}\n\n\ttotalBatchCountCache.Set(key, totalBatchCount, _cacheDuration)\n\n\treturn totalBatchCount, nil\n}\n\nfunc getMetrics(r *BatchJobReconciler, batchJob batch.BatchJob) (metrics.BatchMetrics, error) {\n\tendTime := time.Now()\n\tif batchJob.Status.EndTime != nil {\n\t\tendTime = batchJob.Status.EndTime.Time\n\t}\n\n\tjobMetrics, err := batch.GetMetrics(r.Prometheus, spec.JobKey{\n\t\tID:      batchJob.Name,\n\t\tAPIName: batchJob.Spec.APIName,\n\t\tKind:    userconfig.BatchAPIKind,\n\t}, endTime)\n\tif err != nil {\n\t\treturn metrics.BatchMetrics{}, err\n\t}\n\n\treturn *jobMetrics, nil\n}\n\nfunc saveJobMetrics(r *BatchJobReconciler, batchJob batch.BatchJob) error {\n\tjobMetrics, err := r.Config.GetMetrics(r, batchJob)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tkey := spec.JobMetricsKey(r.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, batchJob.Spec.APIName, batchJob.Name)\n\tif err = r.AWS.UploadJSONToS3(&jobMetrics, r.ClusterConfig.Bucket, key); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc saveJobStatus(r *BatchJobReconciler, batchJob batch.BatchJob) error {\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tstoppedStatusKey := filepath.Join(\n\t\t\t\tspec.JobAPIPrefix(r.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, batchJob.Spec.APIName),\n\t\t\t\tbatchJob.Name,\n\t\t\t\tstatus.JobStopped.String(),\n\t\t\t)\n\t\t\treturn r.AWS.UploadStringToS3(\"\", r.ClusterConfig.Bucket, stoppedStatusKey)\n\n\t\t},\n\t\tfunc() error {\n\t\t\tjobStatus := batchJob.Status.Status.String()\n\t\t\tkey := filepath.Join(\n\t\t\t\tspec.JobAPIPrefix(r.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, batchJob.Spec.APIName),\n\t\t\t\tbatchJob.Name,\n\t\t\t\tjobStatus,\n\t\t\t)\n\t\t\treturn r.AWS.UploadStringToS3(\"\", r.ClusterConfig.Bucket, key)\n\t\t},\n\t)\n}\n\nfunc getReplicaCounts(workerJobPods []kcore.Pod) status.WorkerCounts {\n\tworkerCounts := status.WorkerCounts{}\n\tfor i := range workerJobPods {\n\t\tswitch k8s.GetPodStatus(&workerJobPods[i]) {\n\t\tcase k8s.PodStatusPending:\n\t\t\tworkerCounts.Pending++\n\t\tcase k8s.PodStatusStalled:\n\t\t\tworkerCounts.Stalled++\n\t\tcase k8s.PodStatusCreating:\n\t\t\tworkerCounts.Creating++\n\t\tcase k8s.PodStatusNotReady:\n\t\t\tworkerCounts.NotReady++\n\t\tcase k8s.PodStatusErrImagePull:\n\t\t\tworkerCounts.ErrImagePull++\n\t\tcase k8s.PodStatusTerminating:\n\t\t\tworkerCounts.Terminating++\n\t\tcase k8s.PodStatusFailed:\n\t\t\tworkerCounts.Failed++\n\t\tcase k8s.PodStatusKilled:\n\t\t\tworkerCounts.Killed++\n\t\tcase k8s.PodStatusKilledOOM:\n\t\t\tworkerCounts.KilledOOM++\n\t\tcase k8s.PodStatusSucceeded:\n\t\t\tworkerCounts.Succeeded++\n\t\tcase k8s.PodStatusUnknown:\n\t\t\tworkerCounts.Unknown++\n\t\t}\n\t}\n\treturn workerCounts\n}\n"
  },
  {
    "path": "pkg/crds/controllers/batch/batchjob_controller_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchcontrollers_test\n\nimport (\n\t\"context\"\n\t\"strings\"\n\t\"time\"\n\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/random\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t. \"github.com/onsi/ginkgo\"\n\t. \"github.com/onsi/gomega\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tktypes \"k8s.io/apimachinery/pkg/types\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\nfunc uploadTestAPISpec(apiName string, apiID string) error {\n\tapiSpec := spec.API{\n\t\tAPI: &userconfig.API{\n\t\t\tResource: userconfig.Resource{\n\t\t\t\tName: apiName,\n\t\t\t\tKind: userconfig.BatchAPIKind,\n\t\t\t},\n\t\t\tPod: &userconfig.Pod{\n\t\t\t\tPort: pointer.Int32(8080),\n\t\t\t\tContainers: []*userconfig.Container{\n\t\t\t\t\t{\n\t\t\t\t\t\tName:    \"api\",\n\t\t\t\t\t\tImage:   \"quay.io/cortexlabs/batch-container-test:master\",\n\t\t\t\t\t\tCommand: []string{\"/bin/run\"},\n\t\t\t\t\t\tCompute: &userconfig.Compute{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tID:           apiID,\n\t\tSpecID:       random.String(5),\n\t\tPodID:        random.String(5),\n\t\tDeploymentID: random.String(5),\n\t}\n\tapiSpecKey := spec.Key(apiName, apiID, clusterConfig.ClusterUID)\n\tif err := awsClient.UploadJSONToS3(apiSpec, clusterConfig.Bucket, apiSpecKey); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc deleteTestAPISpec(apiName string, apiID string) error {\n\tapiSpecKey := spec.Key(apiName, apiID, clusterConfig.ClusterUID)\n\tif err := awsClient.DeleteS3File(clusterConfig.Bucket, apiSpecKey); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nvar _ = Describe(\"BatchJob controller\", func() {\n\n\t// Define utility constants for object names and testing timeouts/durations and intervals.\n\tconst (\n\t\tAPIName           = \"test-api\"\n\t\tBatchJobNamespace = \"default\"\n\t\ttimeout           = time.Second * 10\n\t\tinterval          = time.Millisecond * 250\n\t)\n\n\tvar (\n\t\trandomJobID string\n\t\trandomAPIID string\n\t)\n\n\tContext(\"Reconciliation\", func() {\n\t\tBeforeEach(func() {\n\t\t\t// ensures the tests can be ran in rapid succession by avoiding the time limits of SQS queue creation\n\t\t\trandomJobID = strings.ToLower(random.String(5))\n\t\t\trandomAPIID = random.Digits(5)\n\n\t\t\tExpect(uploadTestAPISpec(APIName, randomAPIID)).To(Succeed())\n\t\t})\n\n\t\tAfterEach(func(done Done) {\n\t\t\tExpect(deleteTestAPISpec(APIName, randomAPIID)).To(Succeed())\n\n\t\t\tExpect(k8sClient.Delete(\n\t\t\t\tcontext.Background(),\n\t\t\t\t&batch.BatchJob{\n\t\t\t\t\tObjectMeta: kmeta.ObjectMeta{Name: randomJobID, Namespace: BatchJobNamespace},\n\t\t\t\t},\n\t\t\t)).To(Succeed())\n\t\t\tclose(done)\n\t\t})\n\n\t\tIt(\"Should reach a batch job completed status\", func() {\n\t\t\tWhen(\"Every child resource is created or finishes successfully\", func() {\n\t\t\t\tBy(\"Creating a new BatchJob\")\n\t\t\t\tctx := context.Background()\n\t\t\t\tbatchJob := &batch.BatchJob{\n\t\t\t\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\t\t\t\tName:      randomJobID,\n\t\t\t\t\t\tNamespace: BatchJobNamespace,\n\t\t\t\t\t},\n\t\t\t\t\tSpec: batch.BatchJobSpec{\n\t\t\t\t\t\tAPIName: APIName,\n\t\t\t\t\t\tAPIID:   randomAPIID,\n\t\t\t\t\t\tWorkers: 1,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t\tExpect(k8sClient.Create(ctx, batchJob)).To(Succeed())\n\n\t\t\t\tbatchJobLookupKey := ktypes.NamespacedName{Name: batchJob.Name, Namespace: batchJob.Namespace}\n\t\t\t\tcreatedBatchJob := &batch.BatchJob{}\n\n\t\t\t\t// Check that the resource was created correctly (i.e. if the spec matches)\n\t\t\t\tEventually(func() error {\n\t\t\t\t\treturn k8sClient.Get(ctx, batchJobLookupKey, createdBatchJob)\n\t\t\t\t}, timeout, interval).Should(Succeed())\n\t\t\t\tExpect(createdBatchJob.Spec).Should(Equal(batchJob.Spec))\n\n\t\t\t\tBy(\"Creating an SQS queue successfully\")\n\t\t\t\tEventually(func() bool {\n\t\t\t\t\terr := k8sClient.Get(ctx, batchJobLookupKey, createdBatchJob)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t\treturn createdBatchJob.Status.QueueURL != \"\"\n\t\t\t\t}, timeout, interval).Should(BeTrue())\n\n\t\t\t\tBy(\"Reaching a completed enqueuer status\")\n\t\t\t\tenqueuerJobLookupKey := ktypes.NamespacedName{\n\t\t\t\t\tName:      batchJob.Spec.APIName + \"-\" + batchJob.Name + \"-enqueuer\",\n\t\t\t\t\tNamespace: batchJob.Namespace,\n\t\t\t\t}\n\t\t\t\tcreatedEnqueuerJob := &kbatch.Job{}\n\n\t\t\t\t// wait for the enqueuer job to be created\n\t\t\t\tEventually(func() error {\n\t\t\t\t\treturn k8sClient.Get(ctx, enqueuerJobLookupKey, createdEnqueuerJob)\n\t\t\t\t}, timeout, interval).Should(Succeed())\n\n\t\t\t\t// Mock the enqueuer status to match the success condition\n\t\t\t\tcreatedEnqueuerJob.Status.Succeeded = 1\n\t\t\t\tExpect(k8sClient.Status().Update(ctx, createdEnqueuerJob)).To(Succeed())\n\n\t\t\t\tEventually(func() bool {\n\t\t\t\t\terr := k8sClient.Get(ctx, batchJobLookupKey, createdBatchJob)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t\treturn createdBatchJob.Status.Status == status.JobEnqueuing\n\t\t\t\t}, timeout, interval)\n\n\t\t\t\tBy(\"Reaching a successful worker job status\")\n\n\t\t\t\tworkerJobLookupKey := ktypes.NamespacedName{\n\t\t\t\t\tName:      batchJob.Spec.APIName + \"-\" + batchJob.Name,\n\t\t\t\t\tNamespace: BatchJobNamespace,\n\t\t\t\t}\n\t\t\t\tcreatedWorkerJob := &kbatch.Job{}\n\n\t\t\t\t// Wait for worker job to be created\n\t\t\t\tEventually(func() error {\n\t\t\t\t\treturn k8sClient.Get(ctx, workerJobLookupKey, createdWorkerJob)\n\t\t\t\t}, timeout, interval).Should(Succeed())\n\n\t\t\t\t// Mock the worker job status to match the success condition\n\t\t\t\tcreatedWorkerJob.Status.Succeeded = batchJob.Spec.Workers\n\t\t\t\tExpect(k8sClient.Status().Update(ctx, createdWorkerJob)).To(Succeed())\n\n\t\t\t\tEventually(func() bool {\n\t\t\t\t\terr := k8sClient.Get(ctx, batchJobLookupKey, createdBatchJob)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn false\n\t\t\t\t\t}\n\t\t\t\t\treturn createdBatchJob.Status.Status == status.JobSucceeded\n\t\t\t\t}).Should(BeTrue())\n\t\t\t})\n\t\t})\n\t})\n\n\tContext(\"Reconcialiation TTL\", func() {\n\t\tBeforeEach(func() {\n\t\t\t// ensures the tests can be ran in rapid succession by avoiding the time limits of SQS queue creation\n\t\t\trandomJobID = strings.ToLower(random.String(5))\n\t\t\trandomAPIID = random.Digits(5)\n\n\t\t\tExpect(uploadTestAPISpec(APIName, randomAPIID)).To(Succeed())\n\t\t})\n\n\t\tAfterEach(func(done Done) {\n\t\t\tExpect(deleteTestAPISpec(APIName, randomAPIID)).To(Succeed())\n\t\t\tclose(done)\n\t\t})\n\n\t\tIt(\"Should self clean-up when a completed status is reached and the TTL is exceeded\", func() {\n\t\t\tBy(\"Creating a new BatchJob\")\n\t\t\tttl := kmeta.Duration{Duration: time.Second * 10}\n\t\t\tctx := context.Background()\n\t\t\tbatchJob := &batch.BatchJob{\n\t\t\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\t\t\tName:      randomJobID,\n\t\t\t\t\tNamespace: BatchJobNamespace,\n\t\t\t\t},\n\t\t\t\tSpec: batch.BatchJobSpec{\n\t\t\t\t\tAPIName: APIName,\n\t\t\t\t\tAPIID:   randomAPIID,\n\t\t\t\t\tWorkers: 1,\n\t\t\t\t\tTTL:     &ttl,\n\t\t\t\t},\n\t\t\t}\n\t\t\tExpect(k8sClient.Create(ctx, batchJob)).To(Succeed())\n\n\t\t\tBy(\"Reaching a completed enqueuer status\")\n\t\t\tenqueuerJobLookupKey := ktypes.NamespacedName{\n\t\t\t\tName:      batchJob.Spec.APIName + \"-\" + batchJob.Name + \"-enqueuer\",\n\t\t\t\tNamespace: batchJob.Namespace,\n\t\t\t}\n\t\t\tcreatedEnqueuerJob := &kbatch.Job{}\n\n\t\t\t// wait for the enqueuer job to be created\n\t\t\tEventually(func() error {\n\t\t\t\treturn k8sClient.Get(ctx, enqueuerJobLookupKey, createdEnqueuerJob)\n\t\t\t}, timeout, interval).Should(Succeed())\n\n\t\t\t// Mock the enqueuer status to match the success condition\n\t\t\tcreatedEnqueuerJob.Status.Succeeded = 1\n\t\t\tExpect(k8sClient.Status().Update(ctx, createdEnqueuerJob)).To(Succeed())\n\n\t\t\tBy(\"Reaching a successful worker job status\")\n\t\t\tworkerJobLookupKey := ktypes.NamespacedName{\n\t\t\t\tName:      batchJob.Spec.APIName + \"-\" + batchJob.Name,\n\t\t\t\tNamespace: BatchJobNamespace,\n\t\t\t}\n\t\t\tcreatedWorkerJob := &kbatch.Job{}\n\n\t\t\t// Wait for worker job to be created\n\t\t\tEventually(func() error {\n\t\t\t\treturn k8sClient.Get(ctx, workerJobLookupKey, createdWorkerJob)\n\t\t\t}, timeout, interval).Should(Succeed())\n\n\t\t\t// Mock the worker job status to match the success condition\n\t\t\tcompletionTime := time.Now()\n\t\t\tcreatedWorkerJob.Status.Succeeded = batchJob.Spec.Workers\n\t\t\tcreatedWorkerJob.Status.CompletionTime = &kmeta.Time{Time: completionTime}\n\t\t\tExpect(k8sClient.Status().Update(ctx, createdWorkerJob)).To(Succeed())\n\n\t\t\tBy(\"Waiting for the TTL to kick in\")\n\t\t\tvar deletionTime time.Time\n\t\t\tEventually(func() bool {\n\t\t\t\tif err := k8sClient.Get(ctx, client.ObjectKey{\n\t\t\t\t\tName:      randomJobID,\n\t\t\t\t\tNamespace: BatchJobNamespace,\n\t\t\t\t}, &batch.BatchJob{}); err != nil {\n\t\t\t\t\tif kerrors.IsNotFound(err) {\n\t\t\t\t\t\tdeletionTime = time.Now()\n\t\t\t\t\t\treturn true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn false\n\t\t\t}, ttl.Duration.Seconds()*2).Should(BeTrue())\n\n\t\t\tduration := deletionTime.Sub(completionTime)\n\t\t\tExpect(duration > ttl.Duration).Should(BeTrue())\n\t\t})\n\n\t\tIt(\"Should self clean-up when enqueing fails and the TTL is exceeded\", func() {\n\t\t\tBy(\"Creating a new BatchJob\")\n\t\t\tttl := kmeta.Duration{Duration: time.Second * 10}\n\t\t\tctx := context.Background()\n\t\t\tbatchJob := &batch.BatchJob{\n\t\t\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\t\t\tName:      randomJobID,\n\t\t\t\t\tNamespace: BatchJobNamespace,\n\t\t\t\t},\n\t\t\t\tSpec: batch.BatchJobSpec{\n\t\t\t\t\tAPIName: APIName,\n\t\t\t\t\tAPIID:   randomAPIID,\n\t\t\t\t\tWorkers: 1,\n\t\t\t\t\tTTL:     &ttl,\n\t\t\t\t},\n\t\t\t}\n\t\t\tExpect(k8sClient.Create(ctx, batchJob)).To(Succeed())\n\n\t\t\tBy(\"Reaching a failed enqueuer status\")\n\t\t\tenqueuerJobLookupKey := ktypes.NamespacedName{\n\t\t\t\tName:      batchJob.Spec.APIName + \"-\" + batchJob.Name + \"-enqueuer\",\n\t\t\t\tNamespace: batchJob.Namespace,\n\t\t\t}\n\t\t\tcreatedEnqueuerJob := &kbatch.Job{}\n\n\t\t\t// wait for the enqueuer job to be created\n\t\t\tEventually(func() error {\n\t\t\t\treturn k8sClient.Get(ctx, enqueuerJobLookupKey, createdEnqueuerJob)\n\t\t\t}, timeout, interval).Should(Succeed())\n\n\t\t\t// Mock the enqueuer status to match the success condition\n\t\t\tcreatedEnqueuerJob.Status.Failed = 1\n\t\t\tExpect(k8sClient.Status().Update(ctx, createdEnqueuerJob)).To(Succeed())\n\n\t\t\tBy(\"Waiting for the TTL to kick in\")\n\t\t\tEventually(func() bool {\n\t\t\t\terr := k8sClient.Get(ctx, client.ObjectKey{\n\t\t\t\t\tName:      randomJobID,\n\t\t\t\t\tNamespace: BatchJobNamespace,\n\t\t\t\t}, &batch.BatchJob{})\n\t\t\t\treturn kerrors.IsNotFound(err)\n\t\t\t}, ttl.Duration.Seconds()*2, interval).Should(BeTrue())\n\t\t})\n\t})\n})\n"
  },
  {
    "path": "pkg/crds/controllers/batch/suite_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchcontrollers_test\n\nimport (\n\t// \"os\"\n\t\"path/filepath\"\n\t// \"testing\"\n\n\t// \"github.com/cortexlabs/cortex/pkg/config\"\n\t// \"github.com/cortexlabs/cortex/pkg/consts\"\n\t// batch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t// batchcontrollers \"github.com/cortexlabs/cortex/pkg/crds/controllers/batch\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t// \"github.com/cortexlabs/cortex/pkg/lib/hash\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t// \"github.com/cortexlabs/cortex/pkg/types/metrics\"\n\t// . \"github.com/onsi/ginkgo\"\n\t// . \"github.com/onsi/gomega\"\n\t// \"k8s.io/client-go/kubernetes/scheme\"\n\t\"k8s.io/client-go/rest\"\n\t// ctrl \"sigs.k8s.io/controller-runtime\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n\t\"sigs.k8s.io/controller-runtime/pkg/envtest\"\n\t// \"sigs.k8s.io/controller-runtime/pkg/envtest/printer\"\n\t// logf \"sigs.k8s.io/controller-runtime/pkg/log\"\n\t// \"sigs.k8s.io/controller-runtime/pkg/log/zap\"\n\t// +kubebuilder:scaffold:imports\n)\n\n// These tests use Ginkgo (BDD-style Go testing framework). Refer to\n// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.\n\nvar projectRoot = filepath.Join(\"..\", \"..\", \"..\", \"..\")\nvar devClusterConfigPath = filepath.Join(projectRoot, \"dev\", \"config\", \"cluster.yaml\")\n\nvar cfg *rest.Config\nvar k8sClient client.Client\nvar awsClient *awslib.Client\nvar clusterConfig *clusterconfig.Config\nvar testEnv *envtest.Environment\n\n// func TestAPIs(t *testing.T) {\n// \tRegisterFailHandler(Fail)\n\n// \tRunSpecsWithDefaultAndCustomReporters(t,\n// \t\t\"Controller Suite\",\n// \t\t[]Reporter{printer.NewlineReporter{}})\n// }\n\n// var _ = BeforeSuite(func(done Done) {\n// \tlogf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))\n\n// \tcrdDirectoryPath := filepath.Join(\"..\", \"..\", \"config\", \"crd\", \"bases\")\n// \tExpect(crdDirectoryPath).To(BeADirectory())\n\n// \tBy(\"bootstrapping test environment\")\n// \ttestEnv = &envtest.Environment{\n// \t\tCRDDirectoryPaths: []string{crdDirectoryPath},\n// \t}\n\n// \tvar err error\n// \tcfg, err = testEnv.Start()\n// \tExpect(err).ToNot(HaveOccurred())\n// \tExpect(cfg).ToNot(BeNil())\n\n// \terr = batch.AddToScheme(scheme.Scheme)\n// \tExpect(err).NotTo(HaveOccurred())\n\n// \t// +kubebuilder:scaffold:scheme\n\n// \tk8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})\n// \tExpect(err).ToNot(HaveOccurred())\n// \tExpect(k8sClient).ToNot(BeNil())\n\n// \tk8sManager, err := ctrl.NewManager(cfg, ctrl.Options{\n// \t\tScheme: scheme.Scheme,\n// \t})\n// \tExpect(err).ToNot(HaveOccurred())\n\n// \tclusterConfigPath := os.Getenv(\"CORTEX_TEST_CLUSTER_CONFIG\")\n// \tif clusterConfigPath == \"\" {\n// \t\tclusterConfigPath = devClusterConfigPath\n// \t}\n\n// \tclusterConfig, err = clusterconfig.NewForFile(clusterConfigPath)\n// \tExpect(err).ToNot(HaveOccurred(),\n// \t\t\"error during cluster config creation (custom cluster \"+\n// \t\t\t\"config paths can be set with the CORTEX_TEST_CLUSTER_CONFIG env variable)\",\n// \t)\n\n// \tawsClient, err = awslib.NewForRegion(clusterConfig.Region)\n// \tExpect(err).ToNot(HaveOccurred())\n\n// \taccountID, hashedAccountID, err := awsClient.CheckCredentials()\n// \tExpect(err).ToNot(HaveOccurred())\n\n// \tclusterConfig.AccountID = accountID\n// \tclusterConfig.Bucket = clusterconfig.BucketName(accountID, clusterConfig.ClusterName, clusterConfig.Region)\n\n// \toperatorMetadata := &clusterconfig.OperatorMetadata{\n// \t\tAPIVersion:          consts.CortexVersion,\n// \t\tOperatorID:          hashedAccountID,\n// \t\tClusterID:           hash.String(clusterConfig.ClusterName + clusterConfig.Region + hashedAccountID),\n// \t\tIsOperatorInCluster: false,\n// \t}\n\n// \t// initialize some of the global values for the k8s helpers\n// \tconfig.InitConfigs(clusterConfig, operatorMetadata)\n\n// \t// mock certain methods of the reconciler\n// \treconcilerConfig := batchcontrollers.BatchJobReconcilerConfig{\n// \t\tGetTotalBatchCount: func(r *batchcontrollers.BatchJobReconciler, batchJob batch.BatchJob) (int, error) {\n// \t\t\treturn 1, nil\n// \t\t},\n// \t\tGetMetrics: func(r *batchcontrollers.BatchJobReconciler, batchJob batch.BatchJob) (metrics.BatchMetrics, error) {\n// \t\t\treturn metrics.BatchMetrics{Succeeded: 1}, nil\n// \t\t},\n// \t\tSaveJobMetrics: func(r *batchcontrollers.BatchJobReconciler, batchJob batch.BatchJob) error {\n// \t\t\treturn nil\n// \t\t},\n// \t\tSaveJobStatus: func(r *batchcontrollers.BatchJobReconciler, batchJob batch.BatchJob) error {\n// \t\t\treturn nil\n// \t\t},\n// \t}\n\n// \terr = (&batchcontrollers.BatchJobReconciler{\n// \t\tClient:        k8sManager.GetClient(),\n// \t\tConfig:        reconcilerConfig,\n// \t\tLog:           ctrl.Log.WithName(\"controllers\").WithName(\"BatchJob\"),\n// \t\tClusterConfig: clusterConfig,\n// \t\tAWS:           awsClient,\n// \t\tScheme:        k8sManager.GetScheme(),\n// \t}).SetupWithManager(k8sManager)\n// \tExpect(err).ToNot(HaveOccurred())\n\n// \tgo func() {\n// \t\tdefer GinkgoRecover()\n// \t\terr = k8sManager.Start(ctrl.SetupSignalHandler())\n// \t\tExpect(err).ToNot(HaveOccurred())\n// \t}()\n\n// \tk8sClient = k8sManager.GetClient()\n// \tExpect(k8sClient).ToNot(BeNil())\n\n// \tclose(done)\n// }, 60)\n\n// var _ = AfterSuite(func() {\n// \tBy(\"tearing down the test environment\")\n// \terr := testEnv.Stop()\n// \tExpect(err).ToNot(HaveOccurred())\n// })\n"
  },
  {
    "path": "pkg/crds/controllers/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage controllers\n\nimport (\n\t\"strings\"\n\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n)\n\nconst (\n\t_optimisticLockErrorMessage = \"the object has been modified; please apply your changes to the latest version and try again\"\n)\n\n// IsOptimisticLockError checks if an error is an optimistic lock error\nfunc IsOptimisticLockError(err error) bool {\n\treturn kerrors.IsConflict(err) && strings.Contains(err.Error(), _optimisticLockErrorMessage)\n}\n"
  },
  {
    "path": "pkg/crds/hack/boilerplate.go.txt",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n"
  },
  {
    "path": "pkg/crds/hack/run_manager.sh",
    "content": "#!/usr/bin/env bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# This is a subset of lint.sh, and is only meant to be run on master\n\nCLUSTER_CONFIG=$1\n\nport_forward_cmd=\"kubectl port-forward -n prometheus prometheus-prometheus-0 9090\"\nkill $(pgrep -f \"${port_forward_cmd}\") >/dev/null 2>&1 || true\n\necho \"Port-forwarding Prometheus to localhost:9090\"\neval \"${port_forward_cmd}\" >/dev/null 2>&1 &\n\nCORTEX_DISABLE_JSON_LOGGING=\"true\" \\\nCORTEX_PROMETHEUS_URL=\"http://localhost:9090\" \\\ngo run ./main.go -config \"${CLUSTER_CONFIG}\"\n"
  },
  {
    "path": "pkg/crds/main.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage main\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/hash\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\tpromapi \"github.com/prometheus/client_golang/api\"\n\tpromv1 \"github.com/prometheus/client_golang/api/prometheus/v1\"\n\n\t// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)\n\t// to ensure that exec-entrypoint and run can make use of them.\n\t_ \"k8s.io/client-go/plugin/pkg/client/auth\"\n\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\tutilruntime \"k8s.io/apimachinery/pkg/util/runtime\"\n\tclientgoscheme \"k8s.io/client-go/kubernetes/scheme\"\n\tctrl \"sigs.k8s.io/controller-runtime\"\n\t\"sigs.k8s.io/controller-runtime/pkg/healthz\"\n\t\"sigs.k8s.io/controller-runtime/pkg/log/zap\"\n\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\tbatchcontrollers \"github.com/cortexlabs/cortex/pkg/crds/controllers/batch\"\n\t//+kubebuilder:scaffold:imports\n)\n\nvar (\n\tscheme   = runtime.NewScheme()\n\tsetupLog = ctrl.Log.WithName(\"setup\")\n)\n\nfunc init() {\n\tutilruntime.Must(clientgoscheme.AddToScheme(scheme))\n\n\tutilruntime.Must(batch.AddToScheme(scheme))\n\t//+kubebuilder:scaffold:scheme\n}\n\nfunc main() {\n\tvar (\n\t\tmetricsAddr          string\n\t\tenableLeaderElection bool\n\t\tprobeAddr            string\n\t\tclusterConfigPath    string\n\t\tprometheusURL        string // defaults to http://prometheus.<namespace>:9090\n\t\tinCluster            = strings.ToLower(os.Getenv(\"CORTEX_OPERATOR_IN_CLUSTER\")) == \"true\"\n\t\tuseDevMode           = strings.ToLower(os.Getenv(\"CORTEX_DISABLE_JSON_LOGGING\")) == \"true\"\n\t)\n\tflag.StringVar(&metricsAddr, \"metrics-bind-address\", \":8080\", \"The address the metric endpoint binds to.\")\n\tflag.StringVar(&probeAddr, \"health-probe-bind-address\", \":8081\", \"The address the probe endpoint binds to.\")\n\tflag.BoolVar(&enableLeaderElection, \"leader-elect\", false,\n\t\t\"Enable leader election for controller manager. \"+\n\t\t\t\"Enabling this will ensure there is only one active controller manager.\")\n\tflag.StringVar(&clusterConfigPath, \"config\", os.Getenv(\"CORTEX_CLUSTER_CONFIG_PATH\"),\n\t\t\"The path to the cluster config yaml file. \"+\n\t\t\t\"Can be set with the CORTEX_CLUSTER_CONFIG_PATH env variable. [Required]\",\n\t)\n\tflag.StringVar(&prometheusURL, \"prometheus-url\", os.Getenv(\"CORTEX_PROMETHEUS_URL\"),\n\t\t\"Prometheus server URL\",\n\t)\n\n\topts := zap.Options{\n\t\tDevelopment: useDevMode,\n\t}\n\topts.BindFlags(flag.CommandLine)\n\tflag.Parse()\n\n\tctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))\n\n\tswitch {\n\tcase clusterConfigPath == \"\":\n\t\tsetupLog.Error(nil, \"-config is a required flag\")\n\t\tos.Exit(1)\n\t}\n\n\tclusterConfig, err := clusterconfig.NewForFile(clusterConfigPath)\n\tif err != nil {\n\t\tsetupLog.Error(err, \"failed to initialize cluster config\")\n\t\tos.Exit(1)\n\t}\n\n\tif prometheusURL == \"\" {\n\t\tprometheusURL = fmt.Sprintf(\"http://prometheus.%s:9090\", consts.PrometheusNamespace)\n\t}\n\n\tmgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{\n\t\tScheme:                 scheme,\n\t\tMetricsBindAddress:     metricsAddr,\n\t\tPort:                   9443,\n\t\tHealthProbeBindAddress: probeAddr,\n\t\tLeaderElection:         enableLeaderElection,\n\t\tLeaderElectionID:       \"7cc92962.cortex.dev\",\n\t})\n\tif err != nil {\n\t\tsetupLog.Error(err, \"unable to start manager\")\n\t\tos.Exit(1)\n\t}\n\n\tawsClient, err := awslib.NewForRegion(clusterConfig.Region)\n\tif err != nil {\n\t\tsetupLog.Error(err, \"failed to create AWS client\")\n\t\tos.Exit(1)\n\t}\n\n\taccountID, hashedAccountID, err := awsClient.CheckCredentials()\n\tif err != nil {\n\t\tsetupLog.Error(err, \"failed to check AWS credentials\")\n\t\tos.Exit(1)\n\t}\n\n\tclusterConfig.AccountID = accountID\n\n\toperatorMetadata := &clusterconfig.OperatorMetadata{\n\t\tAPIVersion:          consts.CortexVersion,\n\t\tOperatorID:          hashedAccountID,\n\t\tClusterID:           hash.String(clusterConfig.ClusterName + clusterConfig.Region + hashedAccountID),\n\t\tIsOperatorInCluster: inCluster,\n\t}\n\n\tpromClient, err := promapi.NewClient(promapi.Config{Address: prometheusURL})\n\tif err != nil {\n\t\tsetupLog.Error(err, \"failed to initialize prometheus client\")\n\t\tos.Exit(1)\n\t}\n\n\t// initialize some of the global values for the k8s helpers\n\tconfig.InitConfigs(clusterConfig, operatorMetadata)\n\n\tif err = (&batchcontrollers.BatchJobReconciler{\n\t\tClient:        mgr.GetClient(),\n\t\tConfig:        batchcontrollers.BatchJobReconcilerConfig{}.ApplyDefaults(),\n\t\tLog:           ctrl.Log.WithName(\"controllers\").WithName(\"BatchJob\"),\n\t\tClusterConfig: clusterConfig,\n\t\tAWS:           awsClient,\n\t\tPrometheus:    promv1.NewAPI(promClient),\n\t\tScheme:        mgr.GetScheme(),\n\t}).SetupWithManager(mgr); err != nil {\n\t\tsetupLog.Error(err, \"unable to create controller\", \"controller\", \"BatchJob\")\n\t\tos.Exit(1)\n\t}\n\t//+kubebuilder:scaffold:builder\n\n\tif err := mgr.AddHealthzCheck(\"healthz\", healthz.Ping); err != nil {\n\t\tsetupLog.Error(err, \"unable to set up health check\")\n\t\tos.Exit(1)\n\t}\n\tif err := mgr.AddReadyzCheck(\"readyz\", healthz.Ping); err != nil {\n\t\tsetupLog.Error(err, \"unable to set up ready check\")\n\t\tos.Exit(1)\n\t}\n\n\tsetupLog.Info(\"starting manager\")\n\tif err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {\n\t\tsetupLog.Error(err, \"problem running manager\")\n\t\tos.Exit(1)\n\t}\n}\n"
  },
  {
    "path": "pkg/dequeuer/async_handler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/async\"\n\t\"go.uber.org/zap\"\n)\n\nconst (\n\t// CortexRequestIDHeader is the header containing the workload request id for the user container\n\tCortexRequestIDHeader = \"X-Cortex-Request-ID\"\n)\n\ntype AsyncMessageHandler struct {\n\tconfig       AsyncMessageHandlerConfig\n\taws          *awslib.Client\n\tlog          *zap.SugaredLogger\n\tstoragePath  string\n\thttpClient   *http.Client\n\teventHandler RequestEventHandler\n}\n\ntype AsyncMessageHandlerConfig struct {\n\tClusterUID string\n\tBucket     string\n\tAPIName    string\n\tTargetURL  string\n}\n\nfunc NewAsyncMessageHandler(config AsyncMessageHandlerConfig, awsClient *awslib.Client, eventHandler RequestEventHandler, logger *zap.SugaredLogger) *AsyncMessageHandler {\n\treturn &AsyncMessageHandler{\n\t\tconfig:       config,\n\t\taws:          awsClient,\n\t\tlog:          logger,\n\t\tstoragePath:  async.StoragePath(config.ClusterUID, config.APIName),\n\t\thttpClient:   &http.Client{},\n\t\teventHandler: eventHandler,\n\t}\n}\n\nfunc (h *AsyncMessageHandler) Handle(message *sqs.Message) error {\n\tif message == nil {\n\t\treturn errors.ErrorUnexpected(\"got unexpected nil SQS message\")\n\t}\n\n\tif message.Body == nil || *message.Body == \"\" {\n\t\treturn errors.ErrorUnexpected(\"got unexpected sqs message with empty or nil body\")\n\t}\n\n\trequestID := *message.Body\n\terr := h.handleMessage(requestID)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (h *AsyncMessageHandler) handleMessage(requestID string) error {\n\th.log.Infow(\"processing workload\", \"id\", requestID)\n\n\terr := h.updateStatus(requestID, async.StatusInProgress)\n\tif err != nil {\n\t\treturn errors.Wrap(err, fmt.Sprintf(\"failed to update status to %s\", async.StatusInProgress))\n\t}\n\n\tpayload, err := h.getPayload(requestID)\n\tif err != nil {\n\t\tupdateStatusErr := h.updateStatus(requestID, async.StatusFailed)\n\t\tif updateStatusErr != nil {\n\t\t\th.log.Errorw(\"failed to update status after failure to get payload\", \"id\", requestID, \"error\", updateStatusErr)\n\t\t}\n\t\treturn errors.Wrap(err, \"failed to get payload\")\n\t}\n\tdefer func() {\n\t\th.deletePayload(requestID)\n\t\t_ = payload.Close()\n\t}()\n\n\theaders, err := h.getHeaders(requestID)\n\tif err != nil {\n\t\tupdateStatusErr := h.updateStatus(requestID, async.StatusFailed)\n\t\tif updateStatusErr != nil {\n\t\t\th.log.Errorw(\"failed to update status after failure to get headers\", \"id\", requestID, \"error\", updateStatusErr)\n\t\t}\n\t\treturn errors.Wrap(err, \"failed to get payload\")\n\t}\n\n\tresult, err := h.submitRequest(payload, headers, requestID)\n\tif err != nil {\n\t\th.log.Errorw(\"failed to submit request to user container\", \"id\", requestID, \"error\", err)\n\t\tupdateStatusErr := h.updateStatus(requestID, async.StatusFailed)\n\t\tif updateStatusErr != nil {\n\t\t\treturn errors.Wrap(updateStatusErr, fmt.Sprintf(\"failed to update status to %s\", async.StatusFailed))\n\t\t}\n\t\treturn nil\n\t}\n\n\tif err = h.uploadResult(requestID, result); err != nil {\n\t\tupdateStatusErr := h.updateStatus(requestID, async.StatusFailed)\n\t\tif updateStatusErr != nil {\n\t\t\th.log.Errorw(\"failed to update status after failure to upload result\", \"id\", requestID, \"error\", updateStatusErr)\n\t\t}\n\t\treturn errors.Wrap(err, \"failed to upload result to storage\")\n\t}\n\n\tif err = h.updateStatus(requestID, async.StatusCompleted); err != nil {\n\t\treturn errors.Wrap(err, fmt.Sprintf(\"failed to update status to %s\", async.StatusCompleted))\n\t}\n\n\th.log.Infow(\"workload processing complete\", \"id\", requestID)\n\n\treturn nil\n}\n\nfunc (h *AsyncMessageHandler) updateStatus(requestID string, status async.Status) error {\n\tkey := async.StatusPath(h.storagePath, requestID, status)\n\treturn h.aws.UploadStringToS3(\"\", h.config.Bucket, key)\n}\n\nfunc (h *AsyncMessageHandler) getPayload(requestID string) (io.ReadCloser, error) {\n\tkey := async.PayloadPath(h.storagePath, requestID)\n\toutput, err := h.aws.S3().GetObject(\n\t\t&s3.GetObjectInput{\n\t\t\tKey:    aws.String(key),\n\t\t\tBucket: aws.String(h.config.Bucket),\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn output.Body, nil\n}\n\nfunc (h *AsyncMessageHandler) deletePayload(requestID string) {\n\tkey := async.PayloadPath(h.storagePath, requestID)\n\terr := h.aws.DeleteS3File(h.config.Bucket, key)\n\tif err != nil {\n\t\th.log.Errorw(\"failed to delete user payload\", \"error\", err)\n\t\ttelemetry.Error(errors.Wrap(err, \"failed to delete user payload\"))\n\t}\n}\n\nfunc (h *AsyncMessageHandler) submitRequest(payload io.Reader, headers http.Header, requestID string) (interface{}, error) {\n\treq, err := http.NewRequest(http.MethodPost, h.config.TargetURL, payload)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treq.Header = headers\n\treq.Header.Set(CortexRequestIDHeader, requestID)\n\n\tstartTime := time.Now()\n\tresponse, err := h.httpClient.Do(req)\n\tif err != nil {\n\t\treturn nil, ErrorUserContainerNotReachable(err)\n\t}\n\n\tdefer func() {\n\t\t_ = response.Body.Close()\n\t}()\n\n\th.eventHandler.HandleEvent(\n\t\tRequestEvent{\n\t\t\tStatusCode: response.StatusCode,\n\t\t\tDuration:   time.Since(startTime),\n\t\t},\n\t)\n\n\tif response.StatusCode != http.StatusOK {\n\t\treturn nil, ErrorUserContainerResponseStatusCode(response.StatusCode)\n\t}\n\n\tif !strings.HasPrefix(response.Header.Get(\"Content-Type\"), \"application/json\") {\n\t\treturn nil, ErrorUserContainerResponseMissingJSONHeader()\n\t}\n\n\tvar result interface{}\n\tif err = json.NewDecoder(response.Body).Decode(&result); err != nil {\n\t\treturn nil, ErrorUserContainerResponseNotJSONDecodable()\n\t}\n\n\treturn result, nil\n}\n\nfunc (h *AsyncMessageHandler) uploadResult(requestID string, result interface{}) error {\n\tkey := async.ResultPath(h.storagePath, requestID)\n\treturn h.aws.UploadJSONToS3(result, h.config.Bucket, key)\n}\n\nfunc (h *AsyncMessageHandler) getHeaders(requestID string) (http.Header, error) {\n\tkey := async.HeadersPath(h.storagePath, requestID)\n\n\tvar headers http.Header\n\tif err := h.aws.ReadJSONFromS3(&headers, h.config.Bucket, key); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn headers, nil\n}\n"
  },
  {
    "path": "pkg/dequeuer/async_handler_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/random\"\n\t\"github.com/cortexlabs/cortex/pkg/types/async\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nconst (\n\t_testBucket = \"test\"\n)\n\nfunc TestAsyncMessageHandler_Handle(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\tawsClient := testAWSClient(t)\n\n\trequestID := random.String(8)\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\trequire.Equal(t, requestID, r.Header.Get(CortexRequestIDHeader))\n\t\tw.Header().Set(\"Content-Type\", \"application/json\")\n\t\tw.WriteHeader(http.StatusOK)\n\t\t_, _ = w.Write([]byte(\"{}\"))\n\t}))\n\n\tvar requestEventsCount int\n\teventHandler := NewRequestEventHandlerFunc(func(event RequestEvent) {\n\t\trequestEventsCount++\n\t})\n\n\tasyncHandler := NewAsyncMessageHandler(AsyncMessageHandlerConfig{\n\t\tClusterUID: \"cortex-test\",\n\t\tBucket:     _testBucket,\n\t\tAPIName:    \"async-test\",\n\t\tTargetURL:  server.URL,\n\t}, awsClient, eventHandler, log)\n\n\t_, err := awsClient.S3().CreateBucket(&s3.CreateBucketInput{\n\t\tBucket: aws.String(_testBucket),\n\t})\n\trequire.NoError(t, err)\n\n\terr = awsClient.UploadStringToS3(\"{}\", asyncHandler.config.Bucket, async.PayloadPath(asyncHandler.storagePath, requestID))\n\trequire.NoError(t, err)\n\n\terr = awsClient.UploadStringToS3(\"{}\", asyncHandler.config.Bucket, async.HeadersPath(asyncHandler.storagePath, requestID))\n\trequire.NoError(t, err)\n\n\terr = asyncHandler.Handle(&sqs.Message{\n\t\tBody:      aws.String(requestID),\n\t\tMessageId: aws.String(requestID),\n\t})\n\trequire.NoError(t, err)\n\n\t_, err = awsClient.ReadStringFromS3(\n\t\t_testBucket,\n\t\tfmt.Sprintf(\"%s/%s/status/%s\", asyncHandler.storagePath, requestID, async.StatusCompleted),\n\t)\n\trequire.NoError(t, err)\n\trequire.Equal(t, 1, requestEventsCount)\n}\n\nfunc TestAsyncMessageHandler_Handle_Errors(t *testing.T) {\n\tt.Parallel()\n\n\tcases := []struct {\n\t\tname          string\n\t\tmessage       *sqs.Message\n\t\texpectedError error\n\t}{\n\t\t{\n\t\t\tname:          \"nil\",\n\t\t\tmessage:       nil,\n\t\t\texpectedError: errors.ErrorUnexpected(\"got unexpected nil SQS message\"),\n\t\t},\n\t\t{\n\t\t\tname:          \"nil body\",\n\t\t\tmessage:       &sqs.Message{},\n\t\t\texpectedError: errors.ErrorUnexpected(\"got unexpected sqs message with empty or nil body\"),\n\t\t},\n\t\t{\n\t\t\tname:          \"empty body\",\n\t\t\tmessage:       &sqs.Message{Body: aws.String(\"\")},\n\t\t\texpectedError: errors.ErrorUnexpected(\"got unexpected sqs message with empty or nil body\"),\n\t\t},\n\t}\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\tawsClient := testAWSClient(t)\n\n\teventHandler := NewRequestEventHandlerFunc(func(event RequestEvent) {})\n\n\tasyncHandler := NewAsyncMessageHandler(AsyncMessageHandlerConfig{\n\t\tClusterUID: \"cortex-test\",\n\t\tBucket:     _testBucket,\n\t\tAPIName:    \"async-test\",\n\t\tTargetURL:  \"http://fake.cortex.dev\",\n\t}, awsClient, eventHandler, log)\n\n\tfor _, tt := range cases {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\terr := asyncHandler.Handle(tt.message)\n\t\t\trequire.EqualError(t, err, tt.expectedError.Error())\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/dequeuer/async_stats.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"net/http\"\n\t\"strconv\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promauto\"\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n)\n\ntype AsyncStatsReporter struct {\n\thandler      http.Handler\n\tlatencies    *prometheus.HistogramVec\n\trequestCount *prometheus.CounterVec\n}\n\nfunc NewAsyncPrometheusStatsReporter() *AsyncStatsReporter {\n\tlatenciesHist := promauto.NewHistogramVec(prometheus.HistogramOpts{\n\t\tName: \"cortex_async_latency\",\n\t\tHelp: \"Histogram of the latencies for an AsyncAPI kind in seconds\",\n\t}, []string{\"status_code\"})\n\n\trequestCounter := promauto.NewCounterVec(prometheus.CounterOpts{\n\t\tName: \"cortex_async_request_count\",\n\t\tHelp: \"Request count for an AsyncAPI\",\n\t}, []string{\"status_code\"})\n\n\thandler := promhttp.Handler()\n\n\treturn &AsyncStatsReporter{\n\t\thandler:      handler,\n\t\tlatencies:    latenciesHist,\n\t\trequestCount: requestCounter,\n\t}\n}\n\nfunc (r *AsyncStatsReporter) HandleEvent(event RequestEvent) {\n\tlabels := map[string]string{\n\t\t\"status_code\": strconv.Itoa(event.StatusCode),\n\t}\n\n\tr.latencies.With(labels).Observe(event.Duration.Seconds())\n\tr.requestCount.With(labels).Add(1)\n}\n\nfunc (r *AsyncStatsReporter) ServeHTTP(w http.ResponseWriter, req *http.Request) {\n\tr.handler.ServeHTTP(w, req)\n}\n"
  },
  {
    "path": "pkg/dequeuer/async_stats_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"net/http/httptest\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promauto\"\n\t\"github.com/prometheus/client_golang/prometheus/testutil\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNewAsyncPrometheusStatsReporter(t *testing.T) {\n\tt.Parallel()\n\n\tstatsReporter := NewAsyncPrometheusStatsReporter()\n\n\tstatsReporter.HandleEvent(\n\t\tRequestEvent{\n\t\t\tStatusCode: 200,\n\t\t\tDuration:   100 * time.Millisecond,\n\t\t},\n\t)\n\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/metrics\", nil)\n\tstatsReporter.ServeHTTP(w, r)\n\n\tresult := w.Body.String()\n\trequire.Contains(t, result, \"cortex_async_latency\")\n\trequire.Contains(t, result, \"cortex_async_request_count\")\n}\n\nfunc TestAsyncStatsReporter_HandleEvent(t *testing.T) {\n\tt.Parallel()\n\n\treg := prometheus.NewRegistry()\n\n\tlatenciesHist := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{\n\t\tName: \"cortex_async_latency\",\n\t\tHelp: \"Histogram of the latencies for an AsyncAPI kind in seconds\",\n\t}, []string{\"status_code\"})\n\n\trequestCounter := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{\n\t\tName: \"cortex_async_request_count\",\n\t\tHelp: \"Request count for an AsyncAPI\",\n\t}, []string{\"status_code\"})\n\n\tstatsReporter := AsyncStatsReporter{\n\t\tlatencies:    latenciesHist,\n\t\trequestCount: requestCounter,\n\t}\n\n\tstatsReporter.HandleEvent(\n\t\tRequestEvent{\n\t\t\tStatusCode: 200,\n\t\t\tDuration:   100 * time.Millisecond,\n\t\t},\n\t)\n\n\texpectedHist := `\n# HELP cortex_async_latency Histogram of the latencies for an AsyncAPI kind in seconds\n# TYPE cortex_async_latency histogram\ncortex_async_latency_bucket{status_code=\"200\",le=\"0.005\"} 0\ncortex_async_latency_bucket{status_code=\"200\",le=\"0.01\"} 0\ncortex_async_latency_bucket{status_code=\"200\",le=\"0.025\"} 0\ncortex_async_latency_bucket{status_code=\"200\",le=\"0.05\"} 0\ncortex_async_latency_bucket{status_code=\"200\",le=\"0.1\"} 1\ncortex_async_latency_bucket{status_code=\"200\",le=\"0.25\"} 1\ncortex_async_latency_bucket{status_code=\"200\",le=\"0.5\"} 1\ncortex_async_latency_bucket{status_code=\"200\",le=\"1\"} 1\ncortex_async_latency_bucket{status_code=\"200\",le=\"2.5\"} 1\ncortex_async_latency_bucket{status_code=\"200\",le=\"5\"} 1\ncortex_async_latency_bucket{status_code=\"200\",le=\"10\"} 1\ncortex_async_latency_bucket{status_code=\"200\",le=\"+Inf\"} 1\ncortex_async_latency_sum{status_code=\"200\"} 0.1\ncortex_async_latency_count{status_code=\"200\"} 1\n`\n\n\trequire.Equal(t, float64(1), testutil.ToFloat64(statsReporter.requestCount))\n\trequire.NoError(t, testutil.CollectAndCompare(statsReporter.latencies, strings.NewReader(expectedHist)))\n}\n"
  },
  {
    "path": "pkg/dequeuer/batch_handler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"bytes\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/DataDog/datadog-go/statsd\"\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n\t\"github.com/xtgo/uuid\"\n\t\"go.uber.org/zap\"\n)\n\nconst (\n\t// CortexJobIDHeader is the header containing the job id for the user container\n\tCortexJobIDHeader        = \"X-Cortex-Job-ID\"\n\t_jobCompleteMessageDelay = 10 * time.Second\n)\n\ntype BatchMessageHandler struct {\n\tconfig                  BatchMessageHandlerConfig\n\tjobCompleteMessageDelay time.Duration\n\ttags                    []string\n\taws                     *awslib.Client\n\tmetrics                 statsd.ClientInterface\n\tlog                     *zap.SugaredLogger\n\thttpClient              *http.Client\n}\n\ntype BatchMessageHandlerConfig struct {\n\tAPIName   string\n\tJobID     string\n\tQueueURL  string\n\tRegion    string\n\tTargetURL string\n}\n\nfunc NewBatchMessageHandler(config BatchMessageHandlerConfig, awsClient *awslib.Client, statsdClient statsd.ClientInterface, log *zap.SugaredLogger) *BatchMessageHandler {\n\ttags := []string{\n\t\t\"api_name:\" + config.APIName,\n\t\t\"job_id:\" + config.JobID,\n\t}\n\n\treturn &BatchMessageHandler{\n\t\tconfig:                  config,\n\t\tjobCompleteMessageDelay: _jobCompleteMessageDelay,\n\t\ttags:                    tags,\n\t\taws:                     awsClient,\n\t\tmetrics:                 statsdClient,\n\t\tlog:                     log,\n\t\thttpClient:              &http.Client{},\n\t}\n}\n\nfunc (h *BatchMessageHandler) Handle(message *sqs.Message) error {\n\tif isOnJobCompleteMessage(message) {\n\t\terr := h.onJobComplete(message)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to handle 'onJobComplete' message\")\n\t\t}\n\t\treturn nil\n\t}\n\terr := h.handleBatch(message)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (h *BatchMessageHandler) recordSuccess() error {\n\terr := h.metrics.Incr(\"cortex_batch_succeeded\", h.tags, 1.0)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\treturn nil\n}\n\nfunc (h *BatchMessageHandler) recordFailure() error {\n\terr := h.metrics.Incr(\"cortex_batch_failed\", h.tags, 1.0)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\treturn nil\n}\n\nfunc (h *BatchMessageHandler) recordTimePerBatch(elapsedTime time.Duration) error {\n\terr := h.metrics.Histogram(\"cortex_time_per_batch\", elapsedTime.Seconds(), h.tags, 1.0)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\treturn nil\n}\n\nfunc (h *BatchMessageHandler) submitRequest(messageBody string, isOnJobComplete bool) error {\n\ttargetURL := h.config.TargetURL\n\tif isOnJobComplete {\n\t\ttargetURL = urls.Join(targetURL, \"/on-job-complete\")\n\t}\n\n\treq, err := http.NewRequest(http.MethodPost, targetURL, bytes.NewBuffer([]byte(messageBody)))\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\treq.Header.Set(\"Content-Type\", \"application/json\")\n\treq.Header.Set(CortexJobIDHeader, h.config.JobID)\n\tresponse, err := h.httpClient.Do(req)\n\tif err != nil {\n\t\treturn ErrorUserContainerNotReachable(err)\n\t}\n\tdefer func() {\n\t\t_ = response.Body.Close()\n\t}()\n\n\tif response.StatusCode == http.StatusNotFound && isOnJobComplete {\n\t\treturn nil\n\t}\n\n\tif response.StatusCode != http.StatusOK {\n\t\treturn ErrorUserContainerResponseStatusCode(response.StatusCode)\n\t}\n\n\treturn nil\n}\n\nfunc (h *BatchMessageHandler) handleBatch(message *sqs.Message) error {\n\th.log.Infow(\"processing batch\", \"id\", *message.MessageId)\n\n\tstartTime := time.Now()\n\n\terr := h.submitRequest(*message.Body, false)\n\tif err != nil {\n\t\th.log.Errorw(\"failed to process batch\", \"id\", *message.MessageId, \"error\", err)\n\t\trecordFailureErr := h.recordFailure()\n\t\tif recordFailureErr != nil {\n\t\t\treturn errors.Wrap(recordFailureErr, \"failed to record failure metric\")\n\t\t}\n\t\treturn nil\n\t}\n\n\tendTime := time.Since(startTime)\n\n\terr = h.recordSuccess()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to record success metric\")\n\t}\n\n\terr = h.recordTimePerBatch(endTime)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to record time per batch\")\n\t}\n\treturn nil\n}\n\nfunc (h *BatchMessageHandler) onJobComplete(message *sqs.Message) error {\n\tshouldRunOnJobComplete := false\n\th.log.Info(\"received job_complete message\")\n\tfor {\n\t\tqueueAttributes, err := GetQueueAttributes(h.aws, h.config.QueueURL)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\ttotalMessages := queueAttributes.TotalMessages()\n\n\t\tif totalMessages > 1 {\n\t\t\ttime.Sleep(h.jobCompleteMessageDelay)\n\t\t\th.log.Infow(\"found other messages in queue, requeuing job_complete message\", \"id\", *message.MessageId)\n\t\t\tnewMessageID := uuid.NewRandom().String()\n\t\t\tif _, err = h.aws.SQS().SendMessage(\n\t\t\t\t&sqs.SendMessageInput{\n\t\t\t\t\tQueueUrl:    &h.config.QueueURL,\n\t\t\t\t\tMessageBody: aws.String(\"job_complete\"),\n\t\t\t\t\tMessageAttributes: map[string]*sqs.MessageAttributeValue{\n\t\t\t\t\t\t\"job_complete\": {\n\t\t\t\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\t\t\t\tStringValue: aws.String(\"true\"),\n\t\t\t\t\t\t},\n\t\t\t\t\t\t\"api_name\": {\n\t\t\t\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\t\t\t\tStringValue: aws.String(h.config.APIName),\n\t\t\t\t\t\t},\n\t\t\t\t\t\t\"job_id\": {\n\t\t\t\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\t\t\t\tStringValue: aws.String(h.config.JobID),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tMessageDeduplicationId: aws.String(newMessageID),\n\t\t\t\t\tMessageGroupId:         aws.String(newMessageID),\n\t\t\t\t},\n\t\t\t); err != nil {\n\t\t\t\treturn errors.WithStack(err)\n\t\t\t}\n\n\t\t\treturn nil\n\t\t}\n\n\t\tif shouldRunOnJobComplete {\n\t\t\th.log.Infow(\"processing job_complete message\", \"id\", *message.MessageId)\n\t\t\treturn h.submitRequest(*message.Body, true)\n\t\t}\n\t\tshouldRunOnJobComplete = true\n\n\t\ttime.Sleep(h.jobCompleteMessageDelay)\n\t}\n}\n\nfunc isOnJobCompleteMessage(message *sqs.Message) bool {\n\t_, found := message.MessageAttributes[\"job_complete\"]\n\treturn found\n}\n"
  },
  {
    "path": "pkg/dequeuer/batch_handler_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/DataDog/datadog-go/statsd\"\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestBatchMessageHandler_Handle(t *testing.T) {\n\tt.Parallel()\n\tawsClient := testAWSClient(t)\n\n\tvar callCount int\n\tserver := httptest.NewServer(\n\t\thttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\tcallCount++\n\t\t\tw.WriteHeader(http.StatusOK)\n\t\t}),\n\t)\n\n\tlogger := newLogger(t)\n\tdefer func() { _ = logger.Sync() }()\n\n\tbatchHandler := NewBatchMessageHandler(BatchMessageHandlerConfig{\n\t\tAPIName:   \"test\",\n\t\tJobID:     \"12345\",\n\t\tRegion:    _localStackDefaultRegion,\n\t\tTargetURL: server.URL,\n\t}, awsClient, &statsd.NoOpClient{}, logger)\n\n\terr := batchHandler.Handle(&sqs.Message{\n\t\tBody:      aws.String(\"\"),\n\t\tMessageId: aws.String(\"1\"),\n\t})\n\n\trequire.Equal(t, callCount, 1)\n\trequire.NoError(t, err)\n}\n\nfunc TestBatchMessageHandler_Handle_OnJobComplete(t *testing.T) {\n\tt.Parallel()\n\tawsClient := testAWSClient(t)\n\tqueueURL := createQueue(t, awsClient)\n\n\tvar callCount int\n\tmux := http.NewServeMux()\n\tmux.HandleFunc(\"/on-job-complete\", func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.Method != http.MethodPost {\n\t\t\tw.WriteHeader(http.StatusMethodNotAllowed)\n\t\t\treturn\n\t\t}\n\t\tcallCount++\n\t\tw.WriteHeader(http.StatusOK)\n\t})\n\tserver := httptest.NewServer(mux)\n\n\tlogger := newLogger(t)\n\tdefer func() { _ = logger.Sync() }()\n\n\tbatchHandler := NewBatchMessageHandler(BatchMessageHandlerConfig{\n\t\tAPIName:   \"test\",\n\t\tJobID:     \"12345\",\n\t\tRegion:    _localStackDefaultRegion,\n\t\tTargetURL: server.URL,\n\t\tQueueURL:  queueURL,\n\t}, awsClient, &statsd.NoOpClient{}, logger)\n\n\tbatchHandler.jobCompleteMessageDelay = 0\n\n\terr := batchHandler.Handle(&sqs.Message{\n\t\tBody: aws.String(\"job_complete\"),\n\t\tMessageAttributes: map[string]*sqs.MessageAttributeValue{\n\t\t\t\"job_complete\": {\n\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\tStringValue: aws.String(\"true\"),\n\t\t\t},\n\t\t\t\"api_name\": {\n\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\tStringValue: aws.String(\"test\"),\n\t\t\t},\n\t\t\t\"job_id\": {\n\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\tStringValue: aws.String(\"12345\"),\n\t\t\t},\n\t\t},\n\t\tMessageId: aws.String(\"00000\"),\n\t})\n\n\trequire.NoError(t, err)\n\trequire.Equal(t, callCount, 1)\n}\n"
  },
  {
    "path": "pkg/dequeuer/dequeuer.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/math\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"go.uber.org/zap\"\n)\n\nvar (\n\t_messageAttributes  = []string{\"All\"}\n\t_waitTime           = 10 * time.Second\n\t_visibilityTimeout  = 30 * time.Second\n\t_notFoundSleepTime  = 10 * time.Second\n\t_renewalPeriod      = 10 * time.Second\n\t_probeRefreshPeriod = 1 * time.Second\n)\n\ntype SQSDequeuerConfig struct {\n\tRegion           string\n\tQueueURL         string\n\tStopIfNoMessages bool\n\tWorkers          int\n}\n\ntype SQSDequeuer struct {\n\taws                *awslib.Client\n\tconfig             SQSDequeuerConfig\n\thasDeadLetterQueue bool\n\twaitTimeSeconds    *int64\n\tvisibilityTimeout  *int64\n\tnotFoundSleepTime  time.Duration\n\trenewalPeriod      time.Duration\n\tprobeRefreshPeriod time.Duration\n\tlog                *zap.SugaredLogger\n\tdone               chan struct{}\n}\n\nfunc NewSQSDequeuer(config SQSDequeuerConfig, awsClient *awslib.Client, logger *zap.SugaredLogger) (*SQSDequeuer, error) {\n\tattr, err := GetQueueAttributes(awsClient, config.QueueURL)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &SQSDequeuer{\n\t\taws:                awsClient,\n\t\tconfig:             config,\n\t\thasDeadLetterQueue: attr.HasRedrivePolicy,\n\t\twaitTimeSeconds:    aws.Int64(int64(_waitTime.Seconds())),\n\t\tvisibilityTimeout:  aws.Int64(int64(_visibilityTimeout.Seconds())),\n\t\tnotFoundSleepTime:  _notFoundSleepTime,\n\t\trenewalPeriod:      _renewalPeriod,\n\t\tprobeRefreshPeriod: _probeRefreshPeriod,\n\t\tlog:                logger,\n\t\tdone:               make(chan struct{}),\n\t}, nil\n}\n\nfunc (d *SQSDequeuer) ReceiveMessage() (*sqs.Message, error) {\n\toutput, err := d.aws.SQS().ReceiveMessage(&sqs.ReceiveMessageInput{\n\t\tQueueUrl:              aws.String(d.config.QueueURL),\n\t\tMaxNumberOfMessages:   aws.Int64(1),\n\t\tMessageAttributeNames: aws.StringSlice(_messageAttributes),\n\t\tVisibilityTimeout:     d.visibilityTimeout,\n\t\tWaitTimeSeconds:       d.waitTimeSeconds,\n\t})\n\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif len(output.Messages) == 0 {\n\t\treturn nil, nil\n\t}\n\n\treturn output.Messages[0], nil\n}\n\nfunc (d *SQSDequeuer) Start(messageHandler MessageHandler, readinessProbeFunc func() bool) error {\n\tnumWorkers := math.MaxInt(d.config.Workers, 1)\n\n\td.log.Infof(\"Starting %d workers\", numWorkers)\n\terrCh := make(chan error)\n\tdoneChs := make([]chan struct{}, d.config.Workers)\n\tfor i := 0; i < numWorkers; i++ {\n\t\tdoneChs[i] = make(chan struct{})\n\t\tgo func(i int) {\n\t\t\terrCh <- d.worker(messageHandler, readinessProbeFunc, doneChs[i])\n\t\t}(i)\n\t}\n\n\tselect {\n\tcase err := <-errCh:\n\t\treturn err\n\tcase <-d.done:\n\t\tfor _, doneCh := range doneChs {\n\t\t\tdoneCh <- struct{}{}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (d SQSDequeuer) worker(messageHandler MessageHandler, readinessProbeFunc func() bool, workerDone chan struct{}) error {\n\tnoMessagesInPreviousIteration := false\n\nloop:\n\tfor {\n\t\tselect {\n\t\tcase <-workerDone:\n\t\t\tbreak loop\n\t\tdefault:\n\t\t\tif !readinessProbeFunc() {\n\t\t\t\ttime.Sleep(d.probeRefreshPeriod)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tmessage, err := d.ReceiveMessage()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tif message == nil { // no message received\n\t\t\t\tqueueAttributes, err := GetQueueAttributes(d.aws, d.config.QueueURL)\n\t\t\t\tif err != nil {\n\t\t\t\t\ttelemetry.Error(err)\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\tif queueAttributes.TotalMessages() == 0 {\n\t\t\t\t\tif noMessagesInPreviousIteration && d.config.StopIfNoMessages {\n\t\t\t\t\t\td.log.Info(\"no messages found in queue, exiting ...\")\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t}\n\t\t\t\t\tnoMessagesInPreviousIteration = true\n\t\t\t\t}\n\t\t\t\ttime.Sleep(d.notFoundSleepTime)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tnoMessagesInPreviousIteration = false\n\t\t\treceiptHandle := *message.ReceiptHandle\n\t\t\trenewerDone := d.StartMessageRenewer(receiptHandle)\n\t\t\terr = d.handleMessage(message, messageHandler, renewerDone)\n\t\t\tif err != nil {\n\t\t\t\td.log.Error(err)\n\t\t\t\ttelemetry.Error(err)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (d *SQSDequeuer) Shutdown() {\n\td.done <- struct{}{}\n}\n\nfunc (d *SQSDequeuer) handleMessage(message *sqs.Message, messageHandler MessageHandler, done chan struct{}) error {\n\tmessageErr := messageHandler.Handle(message) // handle error later\n\n\tdone <- struct{}{}\n\tisOnJobComplete := isOnJobCompleteMessage(message)\n\n\tif messageErr != nil && d.hasDeadLetterQueue && !isOnJobComplete {\n\t\t// expire messages when dead letter queue is configured to facilitate redrive policy.\n\t\t// always delete onJobComplete messages regardless of redrive policy because a new one will\n\t\t// be added if an onJobComplete message has been consumed prematurely\n\t\t_, err := d.aws.SQS().ChangeMessageVisibility(\n\t\t\t&sqs.ChangeMessageVisibilityInput{\n\t\t\t\tQueueUrl:          &d.config.QueueURL,\n\t\t\t\tReceiptHandle:     message.ReceiptHandle,\n\t\t\t\tVisibilityTimeout: aws.Int64(0),\n\t\t\t},\n\t\t)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to change sqs message visibility\")\n\t\t}\n\t\treturn nil\n\t}\n\n\t_, err := d.aws.SQS().DeleteMessage(\n\t\t&sqs.DeleteMessageInput{\n\t\t\tQueueUrl:      &d.config.QueueURL,\n\t\t\tReceiptHandle: message.ReceiptHandle,\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to delete sqs message\")\n\t}\n\n\tif messageErr != nil {\n\t\treturn messageErr\n\t}\n\n\treturn nil\n}\n\nfunc (d *SQSDequeuer) StartMessageRenewer(receiptHandle string) chan struct{} {\n\tdone := make(chan struct{})\n\tticker := time.NewTicker(d.renewalPeriod)\n\tstartTime := time.Now()\n\tgo func() {\n\t\tdefer ticker.Stop()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tcase tickerTime := <-ticker.C:\n\t\t\t\tnewVisibilityTimeout := tickerTime.Sub(startTime) + d.renewalPeriod\n\t\t\t\t_, err := d.aws.SQS().ChangeMessageVisibility(\n\t\t\t\t\t&sqs.ChangeMessageVisibilityInput{\n\t\t\t\t\t\tQueueUrl:          &d.config.QueueURL,\n\t\t\t\t\t\tReceiptHandle:     &receiptHandle,\n\t\t\t\t\t\tVisibilityTimeout: aws.Int64(int64(newVisibilityTimeout.Seconds())),\n\t\t\t\t\t},\n\t\t\t\t)\n\t\t\t\tif err != nil {\n\t\t\t\t\td.log.Errorw(\"failed to renew message visibility timeout\", \"error\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\treturn done\n}\n"
  },
  {
    "path": "pkg/dequeuer/dequeuer_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"log\"\n\t\"net/http\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/credentials\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/random\"\n\t\"github.com/ory/dockertest/v3\"\n\tdc \"github.com/ory/dockertest/v3/docker\"\n\t\"github.com/stretchr/testify/require\"\n\t\"go.uber.org/zap\"\n)\n\nvar localStackEndpoint string\n\nconst (\n\t_localStackDefaultRegion = \"us-east-1\"\n)\n\nfunc TestMain(m *testing.M) {\n\t// uses a sensible default on windows (tcp/http) and linux/osx (socket)\n\tlog.Println(\"Starting AWS localstack docker...\")\n\tpool, err := dockertest.NewPool(\"\")\n\tif err != nil {\n\t\tlog.Fatalf(\"Could not connect to docker: %s\", err)\n\t}\n\n\toptions := &dockertest.RunOptions{\n\t\tRepository: \"localstack/localstack\",\n\t\tTag:        \"latest\",\n\t\tPortBindings: map[dc.Port][]dc.PortBinding{\n\t\t\t\"4566/tcp\": {\n\t\t\t\t{HostPort: \"4566\"},\n\t\t\t},\n\t\t},\n\t\tEnv: []string{\"SERVICES=sqs,s3\"},\n\t}\n\n\tresource, err := pool.RunWithOptions(options)\n\tif err != nil {\n\t\tlog.Fatalf(\"Could not start resource: %s\", err)\n\t}\n\n\terr = resource.Expire(90)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tlocalStackEndpoint = fmt.Sprintf(\"localhost:%s\", resource.GetPort(\"4566/tcp\"))\n\n\t// exponential backoff-retry, because the application in the container might not be ready to accept connections yet\n\t// the minio client does not do service discovery for you (i.e. it does not check if connection can be established), so we have to use the health check\n\tif err := pool.Retry(func() error {\n\t\turl := fmt.Sprintf(\"http://%s/health\", localStackEndpoint)\n\t\tresp, err := http.Get(url)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif resp.StatusCode != http.StatusOK {\n\t\t\treturn fmt.Errorf(\"status code not OK\")\n\t\t}\n\t\treturn nil\n\t}); err != nil {\n\t\tlog.Fatalf(\"Could not connect to docker: %s\", err)\n\t}\n\n\tcode := m.Run()\n\n\t// You can't defer this because os.Exit doesn't care for defer\n\tif err := pool.Purge(resource); err != nil {\n\t\tlog.Fatalf(\"Could not purge resource: %s\", err)\n\t}\n\n\tos.Exit(code)\n}\n\nfunc testAWSClient(t *testing.T) *awslib.Client {\n\tt.Helper()\n\n\tsess, err := session.NewSessionWithOptions(session.Options{\n\t\tConfig: aws.Config{\n\t\t\tCredentials:      credentials.NewStaticCredentials(\"test\", \"test\", \"\"),\n\t\t\tEndpoint:         aws.String(localStackEndpoint),\n\t\t\tRegion:           aws.String(_localStackDefaultRegion), // localstack default region\n\t\t\tDisableSSL:       aws.Bool(true),\n\t\t\tS3ForcePathStyle: aws.Bool(true),\n\t\t},\n\t})\n\trequire.NoError(t, err)\n\n\tclient, err := awslib.NewForSession(sess)\n\trequire.NoError(t, err)\n\n\treturn client\n}\n\nfunc newLogger(t *testing.T) *zap.SugaredLogger {\n\tt.Helper()\n\n\tconfig := zap.NewDevelopmentConfig()\n\tconfig.Level = zap.NewAtomicLevelAt(zap.ErrorLevel)\n\tlogger, err := config.Build()\n\trequire.NoError(t, err)\n\n\tlogr := logger.Sugar()\n\n\treturn logr\n}\n\nfunc createQueue(t *testing.T, awsClient *awslib.Client) string {\n\tt.Helper()\n\n\tcreateQueueOutput, err := awsClient.SQS().CreateQueue(\n\t\t&sqs.CreateQueueInput{\n\t\t\tQueueName: aws.String(fmt.Sprintf(\"test-%s.fifo\", random.Digits(5))),\n\t\t\tAttributes: aws.StringMap(\n\t\t\t\tmap[string]string{\n\t\t\t\t\tsqs.QueueAttributeNameFifoQueue:         \"true\",\n\t\t\t\t\tsqs.QueueAttributeNameVisibilityTimeout: \"60\",\n\t\t\t\t},\n\t\t\t),\n\t\t},\n\t)\n\trequire.NoError(t, err)\n\trequire.NotNil(t, createQueueOutput.QueueUrl)\n\trequire.NotEmpty(t, *createQueueOutput.QueueUrl)\n\n\tqueueURL := *createQueueOutput.QueueUrl\n\treturn queueURL\n}\n\nfunc TestSQSDequeuer_ReceiveMessage(t *testing.T) {\n\tt.Parallel()\n\n\tawsClient := testAWSClient(t)\n\tqueueURL := createQueue(t, awsClient)\n\n\tmessageID := \"12345\"\n\tmessageBody := \"blah\"\n\tsentMessage, err := awsClient.SQS().SendMessage(&sqs.SendMessageInput{\n\t\tMessageBody:            aws.String(messageBody),\n\t\tMessageDeduplicationId: aws.String(messageID),\n\t\tMessageGroupId:         aws.String(messageID),\n\t\tQueueUrl:               aws.String(queueURL),\n\t})\n\trequire.NoError(t, err)\n\n\tlogger := newLogger(t)\n\tdefer func() { _ = logger.Sync() }()\n\n\tdq, err := NewSQSDequeuer(\n\t\tSQSDequeuerConfig{\n\t\t\tRegion:           _localStackDefaultRegion,\n\t\t\tQueueURL:         queueURL,\n\t\t\tStopIfNoMessages: true,\n\t\t\tWorkers:          1,\n\t\t}, awsClient, logger,\n\t)\n\trequire.NoError(t, err)\n\n\tgotMessage, err := dq.ReceiveMessage()\n\trequire.NoError(t, err)\n\n\trequire.NotNil(t, gotMessage)\n\trequire.Equal(t, messageBody, *gotMessage.Body)\n\trequire.Equal(t, *sentMessage.MessageId, *gotMessage.MessageId)\n}\n\nfunc TestSQSDequeuer_StartMessageRenewer(t *testing.T) {\n\tt.Parallel()\n\n\tawsClient := testAWSClient(t)\n\tqueueURL := createQueue(t, awsClient)\n\n\tlogger := newLogger(t)\n\tdefer func() { _ = logger.Sync() }()\n\n\tdq, err := NewSQSDequeuer(\n\t\tSQSDequeuerConfig{\n\t\t\tRegion:           _localStackDefaultRegion,\n\t\t\tQueueURL:         queueURL,\n\t\t\tStopIfNoMessages: true,\n\t\t\tWorkers:          1,\n\t\t}, awsClient, logger,\n\t)\n\trequire.NoError(t, err)\n\n\tdq.renewalPeriod = time.Second\n\tdq.visibilityTimeout = aws.Int64(2)\n\n\tmessageID := \"12345\"\n\tmessageBody := \"blah\"\n\t_, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{\n\t\tMessageBody:            aws.String(messageBody),\n\t\tMessageDeduplicationId: aws.String(messageID),\n\t\tMessageGroupId:         aws.String(messageID),\n\t\tQueueUrl:               aws.String(queueURL),\n\t})\n\trequire.NoError(t, err)\n\n\tmessage, err := dq.ReceiveMessage()\n\trequire.NoError(t, err)\n\trequire.NotNil(t, message)\n\n\tdone := dq.StartMessageRenewer(*message.ReceiptHandle)\n\tdefer func() {\n\t\tdone <- struct{}{}\n\t}()\n\n\trequire.Never(t, func() bool {\n\t\tmsg, err := dq.ReceiveMessage()\n\t\trequire.NoError(t, err)\n\n\t\treturn msg != nil\n\t}, time.Second, 10*time.Second)\n}\n\nfunc TestSQSDequeuerTerminationOnEmptyQueue(t *testing.T) {\n\tt.Parallel()\n\n\tawsClient := testAWSClient(t)\n\tqueueURL := createQueue(t, awsClient)\n\n\tlogger := newLogger(t)\n\tdefer func() { _ = logger.Sync() }()\n\n\tdq, err := NewSQSDequeuer(\n\t\tSQSDequeuerConfig{\n\t\t\tRegion:           _localStackDefaultRegion,\n\t\t\tQueueURL:         queueURL,\n\t\t\tStopIfNoMessages: true,\n\t\t\tWorkers:          1,\n\t\t}, awsClient, logger,\n\t)\n\trequire.NoError(t, err)\n\n\tdq.notFoundSleepTime = 0\n\tdq.waitTimeSeconds = aws.Int64(0)\n\n\tmessageID := \"12345\"\n\tmessageBody := \"blah\"\n\t_, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{\n\t\tMessageBody:            aws.String(messageBody),\n\t\tMessageDeduplicationId: aws.String(messageID),\n\t\tMessageGroupId:         aws.String(messageID),\n\t\tQueueUrl:               aws.String(queueURL),\n\t})\n\trequire.NoError(t, err)\n\n\tmsgHandler := &messageHandlerFunc{\n\t\tHandleFunc: func(msg *sqs.Message) error {\n\t\t\treturn nil\n\t\t},\n\t}\n\n\terrCh := make(chan error, 1)\n\tgo func() {\n\t\terrCh <- dq.Start(msgHandler, func() bool {\n\t\t\treturn true\n\t\t})\n\t}()\n\n\ttime.AfterFunc(10*time.Second, func() { errCh <- errors.New(\"timeout: dequeuer did not finish\") })\n\n\terr = <-errCh\n\trequire.NoError(t, err)\n}\n\nfunc TestSQSDequeuer_Shutdown(t *testing.T) {\n\tt.Parallel()\n\n\tawsClient := testAWSClient(t)\n\tqueueURL := createQueue(t, awsClient)\n\n\tlogger := newLogger(t)\n\tdefer func() { _ = logger.Sync() }()\n\n\tdq, err := NewSQSDequeuer(\n\t\tSQSDequeuerConfig{\n\t\t\tRegion:           _localStackDefaultRegion,\n\t\t\tQueueURL:         queueURL,\n\t\t\tStopIfNoMessages: true,\n\t\t\tWorkers:          1,\n\t\t}, awsClient, logger,\n\t)\n\trequire.NoError(t, err)\n\n\tdq.notFoundSleepTime = 0\n\tdq.waitTimeSeconds = aws.Int64(0)\n\n\tmsgHandler := NewMessageHandlerFunc(\n\t\tfunc(message *sqs.Message) error {\n\t\t\treturn nil\n\t\t},\n\t)\n\n\terrCh := make(chan error, 1)\n\tgo func() {\n\t\terrCh <- dq.Start(msgHandler, func() bool {\n\t\t\treturn true\n\t\t})\n\t}()\n\n\ttime.AfterFunc(5*time.Second, func() { errCh <- errors.New(\"timeout: dequeuer did not exit\") })\n\n\tdq.Shutdown()\n\n\terr = <-errCh\n\trequire.NoError(t, err)\n}\n\nfunc TestSQSDequeuer_Start_HandlerError(t *testing.T) {\n\tt.Parallel()\n\n\tawsClient := testAWSClient(t)\n\tqueueURL := createQueue(t, awsClient)\n\n\tlogger := newLogger(t)\n\tdefer func() { _ = logger.Sync() }()\n\n\tdq, err := NewSQSDequeuer(\n\t\tSQSDequeuerConfig{\n\t\t\tRegion:           _localStackDefaultRegion,\n\t\t\tQueueURL:         queueURL,\n\t\t\tStopIfNoMessages: true,\n\t\t\tWorkers:          1,\n\t\t}, awsClient, logger,\n\t)\n\trequire.NoError(t, err)\n\n\tdq.waitTimeSeconds = aws.Int64(0)\n\tdq.notFoundSleepTime = 0\n\tdq.renewalPeriod = 500 * time.Millisecond\n\tdq.visibilityTimeout = aws.Int64(1)\n\n\tmsgHandler := NewMessageHandlerFunc(\n\t\tfunc(message *sqs.Message) error {\n\t\t\treturn fmt.Errorf(\"an error occurred\")\n\t\t},\n\t)\n\n\tmessageID := \"12345\"\n\tmessageBody := \"blah\"\n\t_, err = awsClient.SQS().SendMessage(&sqs.SendMessageInput{\n\t\tMessageBody:            aws.String(messageBody),\n\t\tMessageDeduplicationId: aws.String(messageID),\n\t\tMessageGroupId:         aws.String(messageID),\n\t\tQueueUrl:               aws.String(queueURL),\n\t})\n\trequire.NoError(t, err)\n\n\tgo func() {\n\t\terr := dq.Start(msgHandler, func() bool {\n\t\t\treturn true\n\t\t})\n\t\trequire.NoError(t, err)\n\t}()\n\n\trequire.Never(t, func() bool {\n\t\tmsg, err := dq.ReceiveMessage()\n\t\trequire.NoError(t, err)\n\t\treturn msg != nil\n\t}, 5*time.Second, time.Second)\n}\n\n// this test seems to be non-deterministically timing out\n// it seems to be an issue with the test, not the deqeueur\n// func TestSQSDequeuer_MultipleWorkers(t *testing.T) {\n// \tt.Parallel()\n\n// \tawsClient := testAWSClient(t)\n// \tqueueURL := createQueue(t, awsClient)\n\n// \tnumMessages := 3\n// \texpectedMsgs := make([]string, numMessages)\n// \tfor i := 0; i < numMessages; i++ {\n// \t\tmessage := fmt.Sprintf(\"%d\", i)\n// \t\texpectedMsgs[i] = message\n// \t\t_, err := awsClient.SQS().SendMessage(&sqs.SendMessageInput{\n// \t\t\tMessageBody:            aws.String(message),\n// \t\t\tMessageDeduplicationId: aws.String(message),\n// \t\t\tMessageGroupId:         aws.String(message),\n// \t\t\tQueueUrl:               aws.String(queueURL),\n// \t\t})\n// \t\trequire.NoError(t, err)\n// \t}\n\n// \tlogger := newLogger(t)\n// \tdefer func() { _ = logger.Sync() }()\n\n// \tdq, err := NewSQSDequeuer(\n// \t\tSQSDequeuerConfig{\n// \t\t\tRegion:           _localStackDefaultRegion,\n// \t\t\tQueueURL:         queueURL,\n// \t\t\tStopIfNoMessages: true,\n// \t\t\tWorkers:          numMessages,\n// \t\t}, awsClient, logger,\n// \t)\n// \trequire.NoError(t, err)\n\n// \tdq.waitTimeSeconds = aws.Int64(0)\n// \tdq.notFoundSleepTime = 0\n\n// \tmsgCh := make(chan string, numMessages)\n// \thandler := NewMessageHandlerFunc(\n// \t\tfunc(message *sqs.Message) error {\n// \t\t\tmsgCh <- *message.Body\n// \t\t\treturn nil\n// \t\t},\n// \t)\n\n// \terrCh := make(chan error)\n// \tgo func() {\n// \t\terrCh <- dq.Start(handler, func() bool { return true })\n// \t}()\n\n// \treceivedMessages := make([]string, numMessages)\n// \tfor i := 0; i < numMessages; i++ {\n// \t\treceivedMessages[i] = <-msgCh\n// \t}\n// \tdq.Shutdown()\n\n// \t// timeout test after 30 seconds\n// \ttime.AfterFunc(30*time.Second, func() {\n// \t\tclose(msgCh)\n// \t\terrCh <- errors.New(\"test timed out\")\n// \t})\n\n// \trequire.Len(t, receivedMessages, numMessages)\n\n// \tset := strset.FromSlice(receivedMessages)\n// \trequire.True(t, set.Has(expectedMsgs...))\n\n// \trequire.NoError(t, <-errCh)\n// }\n"
  },
  {
    "path": "pkg/dequeuer/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrUserContainerResponseStatusCode        = \"dequeuer.user_container_response_status_code\"\n\tErrUserContainerResponseMissingJSONHeader = \"dequeuer.user_container_response_missing_json_header\"\n\tErrUserContainerResponseNotJSONDecodable  = \"dequeuer.user_container_response_not_json_decodable\"\n\tErrUserContainerNotReachable              = \"dequeuer.user_container_not_reachable\"\n)\n\nfunc ErrorUserContainerResponseStatusCode(statusCode int) error {\n\treturn &errors.Error{\n\t\tKind:        ErrUserContainerResponseStatusCode,\n\t\tMessage:     fmt.Sprintf(\"invalid response from user container; got status code %d, expected status code 200\", statusCode),\n\t\tNoTelemetry: true,\n\t}\n}\n\nfunc ErrorUserContainerResponseMissingJSONHeader() error {\n\treturn &errors.Error{\n\t\tKind:        ErrUserContainerResponseMissingJSONHeader,\n\t\tMessage:     \"invalid response from user container; response content type header is not 'application/json'\",\n\t\tNoTelemetry: true,\n\t}\n}\n\nfunc ErrorUserContainerResponseNotJSONDecodable() error {\n\treturn &errors.Error{\n\t\tKind:        ErrUserContainerResponseNotJSONDecodable,\n\t\tMessage:     \"invalid response from user container; response is not json decodable\",\n\t\tNoTelemetry: true,\n\t}\n}\n\nfunc ErrorUserContainerNotReachable(err error) error {\n\treturn &errors.Error{\n\t\tKind:        ErrUserContainerNotReachable,\n\t\tMessage:     fmt.Sprintf(\"user container not reachable: %v\", err),\n\t\tNoTelemetry: true,\n\t}\n}\n"
  },
  {
    "path": "pkg/dequeuer/http_handler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport \"net/http\"\n\nfunc HealthcheckHandler(isHealthy func() bool) http.HandlerFunc {\n\treturn func(w http.ResponseWriter, r *http.Request) {\n\t\tif !isHealthy() {\n\t\t\tw.WriteHeader(http.StatusInternalServerError)\n\t\t\t_, _ = w.Write([]byte(\"unhealthy\"))\n\t\t\treturn\n\t\t}\n\n\t\tw.WriteHeader(http.StatusOK)\n\t\t_, _ = w.Write([]byte(\"healthy\"))\n\t}\n}\n"
  },
  {
    "path": "pkg/dequeuer/message_handler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport \"github.com/aws/aws-sdk-go/service/sqs\"\n\ntype MessageHandler interface {\n\tHandle(*sqs.Message) error\n}\n\nfunc NewMessageHandlerFunc(handleFunc func(*sqs.Message) error) MessageHandler {\n\treturn &messageHandlerFunc{HandleFunc: handleFunc}\n}\n\ntype messageHandlerFunc struct {\n\tHandleFunc func(message *sqs.Message) error\n}\n\nfunc (h *messageHandlerFunc) Handle(msg *sqs.Message) error {\n\treturn h.HandleFunc(msg)\n}\n"
  },
  {
    "path": "pkg/dequeuer/probes.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/probe\"\n\t\"go.uber.org/zap\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nfunc ProbesFromFile(probesPath string, logger *zap.SugaredLogger) ([]*probe.Probe, error) {\n\tfileBytes, err := files.ReadFileBytes(probesPath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tprobesMap := map[string]kcore.Probe{}\n\tif err := libjson.Unmarshal(fileBytes, &probesMap); err != nil {\n\t\treturn nil, err\n\t}\n\n\tprobesSlice := make([]*probe.Probe, len(probesMap))\n\tvar i int\n\tfor _, p := range probesMap {\n\t\tauxProbe := p\n\t\tprobesSlice[i] = probe.NewProbe(&auxProbe, logger)\n\t\ti++\n\t}\n\treturn probesSlice, nil\n}\n\nfunc HasTCPProbeTargetingUserPod(probes []*probe.Probe, userPort int) bool {\n\tfor _, pb := range probes {\n\t\tif pb == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif pb.Handler.TCPSocket != nil && pb.Handler.TCPSocket.Port.IntValue() == userPort {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n"
  },
  {
    "path": "pkg/dequeuer/probes_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/probe\"\n\t\"github.com/stretchr/testify/require\"\n\tkcore \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nfunc TestDefaultTCPProbeNotPresent(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\tuserPodPort := 8080\n\n\tprobes := []*probe.Probe{\n\t\tprobe.NewProbe(&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tExec: &kcore.ExecAction{\n\t\t\t\t\tCommand: []string{\"/bin/bash\", \"python\", \"test.py\"},\n\t\t\t\t},\n\t\t\t},\n\t\t}, log),\n\t\tprobe.NewProbe(&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tHTTPGet: &kcore.HTTPGetAction{\n\t\t\t\t\tPath: \"some-path\",\n\t\t\t\t\tPort: intstr.FromInt(12345),\n\t\t\t\t\tHost: \"localhost\",\n\t\t\t\t},\n\t\t\t},\n\t\t}, log),\n\t\tprobe.NewProbe(&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tTCPSocket: &kcore.TCPSocketAction{\n\t\t\t\t\tPort: intstr.FromInt(8447),\n\t\t\t\t\tHost: \"localhost\",\n\t\t\t\t},\n\t\t\t},\n\t\t}, log),\n\t}\n\n\trequire.False(t, HasTCPProbeTargetingUserPod(probes, userPodPort))\n}\n\nfunc TestDefaultTCPProbePresent(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\tuserPodPort := intstr.FromInt(8080)\n\n\tprobes := []*probe.Probe{\n\t\tprobe.NewProbe(&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tExec: &kcore.ExecAction{\n\t\t\t\t\tCommand: []string{\"/bin/bash\", \"python\", \"test.py\"},\n\t\t\t\t},\n\t\t\t},\n\t\t}, log),\n\t\tprobe.NewProbe(&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tHTTPGet: &kcore.HTTPGetAction{\n\t\t\t\t\tPath: \"some-path\",\n\t\t\t\t\tPort: intstr.FromInt(12345),\n\t\t\t\t\tHost: \"localhost\",\n\t\t\t\t},\n\t\t\t},\n\t\t}, log),\n\t\tprobe.NewProbe(&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tTCPSocket: &kcore.TCPSocketAction{\n\t\t\t\t\tPort: userPodPort,\n\t\t\t\t\tHost: \"localhost\",\n\t\t\t\t},\n\t\t\t},\n\t\t}, log),\n\t}\n\n\trequire.True(t, HasTCPProbeTargetingUserPod(probes, userPodPort.IntValue()))\n}\n"
  },
  {
    "path": "pkg/dequeuer/queue_attributes.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype QueueAttributes struct {\n\tVisibleMessages   int\n\tInvisibleMessages int\n\tHasRedrivePolicy  bool\n}\n\nfunc (attr QueueAttributes) TotalMessages() int {\n\treturn attr.VisibleMessages + attr.InvisibleMessages\n}\n\nfunc GetQueueAttributes(client *awslib.Client, queueURL string) (QueueAttributes, error) {\n\tresult, err := client.SQS().GetQueueAttributes(\n\t\t&sqs.GetQueueAttributesInput{\n\t\t\tQueueUrl:       aws.String(queueURL),\n\t\t\tAttributeNames: aws.StringSlice([]string{\"All\"}),\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn QueueAttributes{}, errors.WithStack(err)\n\t}\n\n\tattributes := aws.StringValueMap(result.Attributes)\n\n\tvar visibleCount int\n\tvar notVisibleCount int\n\tvar hasRedrivePolicy bool\n\tif val, found := attributes[\"ApproximateNumberOfMessages\"]; found {\n\t\tcount, ok := s.ParseInt(val)\n\t\tif ok {\n\t\t\tvisibleCount = count\n\t\t}\n\t}\n\n\tif val, found := attributes[\"ApproximateNumberOfMessagesNotVisible\"]; found {\n\t\tcount, ok := s.ParseInt(val)\n\t\tif ok {\n\t\t\tnotVisibleCount = count\n\t\t}\n\t}\n\n\t_, hasRedrivePolicy = attributes[\"RedrivePolicy\"]\n\n\treturn QueueAttributes{\n\t\tVisibleMessages:   visibleCount,\n\t\tInvisibleMessages: notVisibleCount,\n\t\tHasRedrivePolicy:  hasRedrivePolicy,\n\t}, nil\n}\n"
  },
  {
    "path": "pkg/dequeuer/request_stats.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage dequeuer\n\nimport \"time\"\n\ntype RequestEvent struct {\n\tStatusCode int\n\tDuration   time.Duration\n}\n\ntype RequestEventHandler interface {\n\tHandleEvent(event RequestEvent)\n}\n\ntype requestEventHandlerFunc struct {\n\tHandleFunc func(event RequestEvent)\n}\n\nfunc (h *requestEventHandlerFunc) HandleEvent(event RequestEvent) {\n\th.HandleFunc(event)\n}\n\nfunc NewRequestEventHandlerFunc(handleFunc func(event RequestEvent)) RequestEventHandler {\n\treturn &requestEventHandlerFunc{\n\t\tHandleFunc: handleFunc,\n\t}\n}\n"
  },
  {
    "path": "pkg/enqueuer/enqueuer.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage enqueuer\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/random\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"go.uber.org/zap\"\n)\n\nconst (\n\t_s3DownloadChunkSize = 32 * 1024 * 1024\n)\n\ntype EnvConfig struct {\n\tClusterUID string\n\tRegion     string\n\tVersion    string\n\tBucket     string\n\tAPIName    string\n\tJobID      string\n}\n\n// FIXME: all these types should be shared with the cortex web server (from where the payload is submitted)\n\ntype ItemList struct {\n\tItems     []json.RawMessage `json:\"items\"`\n\tBatchSize int               `json:\"batch_size\"`\n}\n\ntype S3Lister struct {\n\tS3Paths    []string `json:\"s3_paths\"` // s3://<bucket_name>/key\n\tIncludes   []string `json:\"includes\"`\n\tExcludes   []string `json:\"excludes\"`\n\tMaxResults *int64   `json:\"-\"` // this is not currently exposed to the user (it's used for validations)\n}\n\ntype FilePathLister struct {\n\tS3Lister\n\tBatchSize int `json:\"batch_size\"`\n}\n\ntype DelimitedFiles struct {\n\tS3Lister\n\tBatchSize int `json:\"batch_size\"`\n}\n\ntype JobSubmission struct {\n\tItemList       *ItemList       `json:\"item_list\"`\n\tFilePathLister *FilePathLister `json:\"file_path_lister\"`\n\tDelimitedFiles *DelimitedFiles `json:\"delimited_files\"`\n}\n\ntype onJobCompleteRequestBody struct {\n\tMessage string `json:\"message\"`\n}\n\nfunc randomMessageID() string {\n\treturn random.String(40) // maximum is 80 (for sqs.SendMessageBatchRequestEntry.Id) but this ID may show up in a user error message\n}\n\ntype Enqueuer struct {\n\taws       *awslib.Client\n\tenvConfig EnvConfig\n\tqueueURL  string\n\tlogger    *zap.Logger\n}\n\nfunc NewEnqueuer(envConfig EnvConfig, queueURL string, logger *zap.Logger) (*Enqueuer, error) {\n\tawsClient, err := awslib.NewForRegion(envConfig.Region)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &Enqueuer{\n\t\taws:       awsClient,\n\t\tenvConfig: envConfig,\n\t\tqueueURL:  queueURL,\n\t\tlogger:    logger,\n\t}, nil\n}\n\nfunc (e *Enqueuer) Enqueue() (int, error) {\n\tsubmission, err := e.getJobPayload()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\ttotalBatches := 0\n\tif submission.ItemList != nil {\n\t\ttotalBatches, err = e.enqueueItems(submission.ItemList)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t} else if submission.FilePathLister != nil {\n\t\ttotalBatches, err = e.enqueueS3Paths(submission.FilePathLister)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t} else if submission.DelimitedFiles != nil {\n\t\ttotalBatches, err = e.enqueueS3FileContents(submission.DelimitedFiles)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t}\n\n\tonJobCompleteBodyBytes, err := json.Marshal(onJobCompleteRequestBody{\n\t\tMessage: \"job_complete\",\n\t})\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\trandomID := randomMessageID()\n\t_, err = e.aws.SQS().SendMessage(&sqs.SendMessageInput{\n\t\tQueueUrl:               aws.String(e.queueURL),\n\t\tMessageBody:            aws.String(string(onJobCompleteBodyBytes)),\n\t\tMessageDeduplicationId: aws.String(randomID), // prevent content based deduping\n\t\tMessageGroupId:         aws.String(randomID), // aws recommends message group id per message to improve chances of exactly-once\n\t\tMessageAttributes: map[string]*sqs.MessageAttributeValue{\n\t\t\t\"job_complete\": {\n\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\tStringValue: aws.String(\"true\"),\n\t\t\t},\n\t\t\t\"api_name\": {\n\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\tStringValue: aws.String(e.envConfig.APIName),\n\t\t\t},\n\t\t\t\"job_id\": {\n\t\t\t\tDataType:    aws.String(\"String\"),\n\t\t\t\tStringValue: aws.String(e.envConfig.JobID),\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, \"failed to enqueue job_complete placeholder\")\n\t}\n\n\tif err = e.deleteJobPayload(); err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn totalBatches, nil\n}\n\nfunc (e *Enqueuer) UploadBatchCount(batchCount int) error {\n\tkey := spec.JobBatchCountKey(e.envConfig.ClusterUID, userconfig.BatchAPIKind, e.envConfig.APIName, e.envConfig.JobID)\n\treturn e.aws.UploadStringToS3(s.Int(batchCount), e.envConfig.Bucket, key)\n}\n\nfunc (e *Enqueuer) getJobPayload() (JobSubmission, error) {\n\t// e.g. <cluster uid>/jobs/<job_api_kind>/<cortex version>/<api_name>/<job_id>\n\tkey := spec.JobPayloadKey(e.envConfig.ClusterUID, userconfig.BatchAPIKind, e.envConfig.APIName, e.envConfig.JobID)\n\n\tsubmissionBytes, err := e.aws.ReadBytesFromS3(e.envConfig.Bucket, key)\n\tif err != nil {\n\t\treturn JobSubmission{}, err\n\t}\n\n\tvar submission JobSubmission\n\tif err = json.Unmarshal(submissionBytes, &submission); err != nil {\n\t\treturn JobSubmission{}, err\n\t}\n\n\treturn submission, nil\n}\n\nfunc (e *Enqueuer) deleteJobPayload() error {\n\tkey := spec.JobPayloadKey(e.envConfig.ClusterUID, userconfig.BatchAPIKind, e.envConfig.APIName, e.envConfig.JobID)\n\tif err := e.aws.DeleteS3File(e.envConfig.Bucket, key); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (e *Enqueuer) enqueueItems(itemList *ItemList) (int, error) {\n\tlog := e.logger\n\n\tbatchCount := len(itemList.Items) / itemList.BatchSize\n\tif len(itemList.Items)%itemList.BatchSize != 0 {\n\t\tbatchCount++\n\t}\n\n\tlog.Info(\n\t\t\"partitioning items found in job submission into batches\",\n\t\tzap.Int(\"numItems\", len(itemList.Items)),\n\t\tzap.Int(\"batchCount\", batchCount),\n\t\tzap.Int(\"batchSize\", itemList.BatchSize),\n\t)\n\n\tuploader := newSQSBatchUploader(e.envConfig.APIName, e.envConfig.JobID, e.queueURL, e.aws.SQS())\n\n\tfor i := 0; i < batchCount; i++ {\n\t\tmin := i * (itemList.BatchSize)\n\t\tmax := (i + 1) * (itemList.BatchSize)\n\t\tif max > len(itemList.Items) {\n\t\t\tmax = len(itemList.Items)\n\t\t}\n\n\t\tjsonBytes, err := json.Marshal(itemList.Items[min:max])\n\t\tif err != nil {\n\t\t\tif itemList.BatchSize == 1 {\n\t\t\t\treturn 0, errors.Wrap(err, fmt.Sprintf(\"item %d\", i))\n\t\t\t}\n\t\t\treturn 0, errors.Wrap(err, fmt.Sprintf(\"items with index between %d to %d\", min, max))\n\t\t}\n\n\t\terr = uploader.AddToBatch(randomMessageID(), pointer.String(string(jsonBytes)))\n\t\tif err != nil {\n\t\t\tif itemList.BatchSize == 1 {\n\t\t\t\treturn 0, errors.Wrap(err, fmt.Sprintf(\"item %d\", i))\n\t\t\t}\n\t\t\treturn 0, errors.Wrap(err, fmt.Sprintf(\"items with index between %d to %d\", min, max))\n\t\t}\n\t\tif uploader.TotalBatches%100 == 0 {\n\t\t\tlog.Info(\"enqueued batches\", zap.Int(\"batchCount\", uploader.TotalBatches))\n\t\t}\n\t}\n\n\terr := uploader.Flush()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn uploader.TotalBatches, nil\n}\n\nfunc (e *Enqueuer) enqueueS3Paths(s3PathsLister *FilePathLister) (int, error) {\n\tlog := e.logger\n\n\tvar s3PathList []string\n\tuploader := newSQSBatchUploader(e.envConfig.APIName, e.envConfig.JobID, e.queueURL, e.aws.SQS())\n\n\t_, err := s3IteratorFromLister(e.aws, s3PathsLister.S3Lister, func(bucket string, s3Obj *s3.Object) (bool, error) {\n\t\ts3Path := awslib.S3Path(bucket, *s3Obj.Key)\n\n\t\ts3PathList = append(s3PathList, s3Path)\n\t\tif len(s3PathList) == s3PathsLister.BatchSize {\n\t\t\terr := addS3PathsToQueue(uploader, s3PathList)\n\t\t\tif err != nil {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\t\ts3PathList = nil\n\n\t\t\tif uploader.TotalBatches%100 == 0 {\n\t\t\t\tlog.Info(\"enqueued batches\", zap.Int(\"numBatches\", uploader.TotalBatches))\n\t\t\t}\n\t\t}\n\n\t\treturn true, nil\n\t})\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif len(s3PathList) > 0 {\n\t\terr := addS3PathsToQueue(uploader, s3PathList)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t}\n\n\terr = uploader.Flush()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn uploader.TotalBatches, nil\n}\n\nfunc (e *Enqueuer) enqueueS3FileContents(delimitedFiles *DelimitedFiles) (int, error) {\n\tlog := e.logger\n\n\tjsonMessageList := newJSONBuffer(delimitedFiles.BatchSize)\n\tuploader := newSQSBatchUploader(e.envConfig.APIName, e.envConfig.JobID, e.queueURL, e.aws.SQS())\n\n\tbytesBuffer := bytes.NewBuffer([]byte{})\n\t_, err := s3IteratorFromLister(e.aws, delimitedFiles.S3Lister, func(bucket string, s3Obj *s3.Object) (bool, error) {\n\t\ts3Path := awslib.S3Path(bucket, *s3Obj.Key)\n\t\tlog.Info(\"enqueuing contents from file\", zap.String(\"path\", s3Path))\n\n\t\tawsClientForBucket, err := awslib.NewFromClientS3Path(s3Path, e.aws)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\titemIndex := 0\n\t\terr = awsClientForBucket.S3FileIterator(bucket, s3Obj, _s3DownloadChunkSize, func(readCloser io.ReadCloser, isLastChunk bool) (bool, error) {\n\t\t\t_, err := bytesBuffer.ReadFrom(readCloser)\n\t\t\tif err != nil {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\t\terr = e.streamJSONToQueue(uploader, bytesBuffer, jsonMessageList, &itemIndex)\n\t\t\tif err != nil {\n\t\t\t\tif errors.CauseOrSelf(err) != io.ErrUnexpectedEOF || (errors.CauseOrSelf(err) == io.ErrUnexpectedEOF && isLastChunk) {\n\t\t\t\t\treturn false, err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true, nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\treturn true, nil\n\t})\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif jsonMessageList.Length() != 0 {\n\t\terr := addJSONObjectsToQueue(uploader, jsonMessageList)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t}\n\terr = uploader.Flush()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn uploader.TotalBatches, nil\n}\n\nfunc (e *Enqueuer) streamJSONToQueue(uploader *sqsBatchUploader, bytesBuffer *bytes.Buffer, jsonMessageList *jsonBuffer, itemIndex *int) error {\n\tlog := e.logger\n\n\tdec := json.NewDecoder(bytesBuffer)\n\tfor {\n\t\tvar doc json.RawMessage\n\n\t\terr := dec.Decode(&doc)\n\t\tif err == io.EOF {\n\t\t\tbreak\n\t\t} else if err == io.ErrUnexpectedEOF {\n\t\t\tbytesBuffer.Reset()\n\t\t\t_, _ = bytesBuffer.ReadFrom(dec.Buffered())\n\t\t\treturn io.ErrUnexpectedEOF\n\t\t} else if err != nil {\n\t\t\treturn errors.Wrap(err, fmt.Sprintf(\"item %d\", *itemIndex))\n\t\t}\n\n\t\tif len(doc) > _messageSizeLimit {\n\t\t\treturn errors.Wrap(ErrorMessageExceedsMaxSize(len(doc), _messageSizeLimit), fmt.Sprintf(\"item %d\", *itemIndex))\n\t\t}\n\t\t*itemIndex++\n\t\tjsonMessageList.Add(doc)\n\t\tif jsonMessageList.Length() == jsonMessageList.BatchSize {\n\t\t\terr := addJSONObjectsToQueue(uploader, jsonMessageList)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tjsonMessageList.Clear()\n\n\t\t\tif uploader.TotalBatches%100 == 0 {\n\t\t\t\tlog.Info(\"enqueued batches\", zap.Int(\"numBatches\", uploader.TotalBatches))\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc addS3PathsToQueue(uploader *sqsBatchUploader, s3PathList []string) error {\n\tjsonBytes, err := json.Marshal(s3PathList)\n\tif err != nil {\n\t\treturn errors.Wrap(err, fmt.Sprintf(\"batch %d\", uploader.TotalBatches))\n\t}\n\n\terr = uploader.AddToBatch(randomMessageID(), pointer.String(string(jsonBytes)))\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/enqueuer/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage enqueuer\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrFailedToEnqueueMessages = \"batchapi.failed_to_enqueue_messages\"\n\tErrMessageExceedsMaxSize   = \"batchapi.message_exceeds_max_size\"\n)\n\nfunc ErrorFailedToEnqueueMessages(message string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFailedToEnqueueMessages,\n\t\tMessage: message,\n\t})\n}\n\nfunc ErrorMessageExceedsMaxSize(messageSize int, messageLimit int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMessageExceedsMaxSize,\n\t\tMessage: fmt.Sprintf(\"cannot enqueue message because its size of %d bytes exceeds the %d bytes limit; use a smaller batch size or reduce the size of each of item in the batch\", messageSize, messageLimit),\n\t})\n}\n"
  },
  {
    "path": "pkg/enqueuer/helpers.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage enqueuer\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/gobwas/glob\"\n)\n\ntype jsonBuffer struct {\n\tBatchSize   int\n\tmessageList []json.RawMessage\n}\n\nfunc newJSONBuffer(batchSize int) *jsonBuffer {\n\treturn &jsonBuffer{\n\t\tBatchSize:   batchSize,\n\t\tmessageList: make([]json.RawMessage, 0, batchSize),\n\t}\n}\n\nfunc (j *jsonBuffer) Add(jsonMessage json.RawMessage) {\n\tj.messageList = append(j.messageList, jsonMessage)\n}\n\nfunc (j *jsonBuffer) Clear() {\n\tj.messageList = make([]json.RawMessage, 0, j.BatchSize)\n}\n\nfunc (j *jsonBuffer) Length() int {\n\treturn len(j.messageList)\n}\n\nfunc addJSONObjectsToQueue(uploader *sqsBatchUploader, jsonMessageList *jsonBuffer) error {\n\tjsonBytes, err := json.Marshal(jsonMessageList.messageList)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = uploader.AddToBatch(randomMessageID(), pointer.String(string(jsonBytes)))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc s3IteratorFromLister(awsClient *awslib.Client, s3Lister S3Lister, fn func(string, *s3.Object) (bool, error)) (int64, error) {\n\tincludeGlobPatterns := make([]glob.Glob, 0, len(s3Lister.Includes))\n\n\tfor _, includePattern := range s3Lister.Includes {\n\t\tglobExpression, err := glob.Compile(includePattern, '/')\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, \"failed to interpret glob pattern\", includePattern)\n\t\t}\n\t\tincludeGlobPatterns = append(includeGlobPatterns, globExpression)\n\t}\n\n\texcludeGlobPatterns := make([]glob.Glob, 0, len(s3Lister.Excludes))\n\tfor _, excludePattern := range s3Lister.Excludes {\n\t\tglobExpression, err := glob.Compile(excludePattern, '/')\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, \"failed to interpret glob pattern\", excludePattern)\n\t\t}\n\t\texcludeGlobPatterns = append(excludeGlobPatterns, globExpression)\n\t}\n\n\tvar numResults int64\n\n\tfor _, s3Path := range s3Lister.S3Paths {\n\t\tbucket, key, err := awslib.SplitS3Path(s3Path)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\tawsClientForBucket, err := awslib.NewFromClientS3Path(s3Path, awsClient)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\terr = awsClientForBucket.S3Iterator(bucket, key, false, nil, nil, func(s3Obj *s3.Object) (bool, error) {\n\t\t\ts3FilePath := awslib.S3Path(bucket, *s3Obj.Key)\n\n\t\t\tshouldSkip := false\n\t\t\tif len(includeGlobPatterns) > 0 {\n\t\t\t\tshouldSkip = true\n\t\t\t\tfor _, includeGlobPattern := range includeGlobPatterns {\n\t\t\t\t\tif includeGlobPattern.Match(s3FilePath) {\n\t\t\t\t\t\tshouldSkip = false\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor _, excludeGlobPattern := range excludeGlobPatterns {\n\t\t\t\tif excludeGlobPattern.Match(s3FilePath) {\n\t\t\t\t\tshouldSkip = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif !shouldSkip {\n\t\t\t\tshouldContinue, err := fn(bucket, s3Obj)\n\t\t\t\tnumResults++\n\t\t\t\tif s3Lister.MaxResults != nil && numResults >= *s3Lister.MaxResults {\n\t\t\t\t\tshouldContinue = false\n\t\t\t\t}\n\t\t\t\treturn shouldContinue, err\n\t\t\t}\n\n\t\t\treturn true, nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\tif s3Lister.MaxResults != nil && numResults >= *s3Lister.MaxResults {\n\t\t\treturn numResults, nil\n\t\t}\n\t}\n\n\treturn numResults, nil\n}\n"
  },
  {
    "path": "pkg/enqueuer/uploader.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage enqueuer\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\t_messageSizeLimit    = 250 * 1024 // normally its 256 * 1024 but reserve 6k for message attributes\n\t_maxMessagesPerBatch = 10\n)\n\ntype sqsBatchUploader struct {\n\tclient               *sqs.SQS\n\tmessageAttributes    map[string]*sqs.MessageAttributeValue\n\tqueueURL             string\n\tretries              int // default 3 times\n\tmessageList          []*sqs.SendMessageBatchRequestEntry\n\tmessageIDToListIndex map[string]int\n\ttotalBytes           int\n\tTotalBatches         int\n}\n\nfunc newSQSBatchUploader(apiName, jobID, queueURL string, client *sqs.SQS) *sqsBatchUploader {\n\tmessageAttributes := map[string]*sqs.MessageAttributeValue{\n\t\t\"api_name\": {\n\t\t\tDataType:    aws.String(\"String\"),\n\t\t\tStringValue: aws.String(apiName),\n\t\t},\n\t\t\"job_id\": {\n\t\t\tDataType:    aws.String(\"String\"),\n\t\t\tStringValue: aws.String(jobID),\n\t\t},\n\t}\n\n\treturn &sqsBatchUploader{\n\t\tclient:               client,\n\t\tmessageAttributes:    messageAttributes,\n\t\tqueueURL:             queueURL,\n\t\tretries:              3,\n\t\tmessageIDToListIndex: map[string]int{},\n\t}\n}\n\nfunc (uploader *sqsBatchUploader) AddToBatch(id string, body *string) error {\n\tif len(*body) > _messageSizeLimit {\n\t\treturn ErrorMessageExceedsMaxSize(len(*body), _messageSizeLimit)\n\t}\n\n\tmessage := &sqs.SendMessageBatchRequestEntry{\n\t\tMessageAttributes:      uploader.messageAttributes,\n\t\tId:                     aws.String(id),\n\t\tMessageBody:            body,\n\t\tMessageDeduplicationId: aws.String(id), // prevent content based deduping\n\t\tMessageGroupId:         aws.String(id), // aws recommends message group id per message to improve chances of exactly-once\n\t}\n\n\tif len(*message.MessageBody)+uploader.totalBytes > _messageSizeLimit || len(uploader.messageList) == _maxMessagesPerBatch {\n\t\terr := uploader.Flush()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tuploader.messageList = append(uploader.messageList, message)\n\tuploader.messageIDToListIndex[id] = uploader.TotalBatches\n\tuploader.totalBytes += len(*message.MessageBody)\n\tuploader.TotalBatches++\n\treturn nil\n}\n\nfunc (uploader *sqsBatchUploader) Flush() error {\n\tif len(uploader.messageList) == 0 {\n\t\treturn nil\n\t}\n\n\tvar err error\n\n\tfor attempt := 0; attempt < uploader.retries; attempt++ {\n\t\terr = uploader.enqueueToSQS()\n\t\tif err == nil {\n\t\t\tuploader.messageList = nil\n\t\t\tuploader.messageIDToListIndex = map[string]int{}\n\t\t\tuploader.totalBytes = 0\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn errors.Wrap(err, fmt.Sprintf(\"failed after retrying %d times\", uploader.retries))\n}\n\nfunc (uploader *sqsBatchUploader) enqueueToSQS() error {\n\toutput, err := uploader.client.SendMessageBatch(&sqs.SendMessageBatchInput{\n\t\tQueueUrl: aws.String(uploader.queueURL),\n\t\tEntries:  uploader.messageList,\n\t})\n\tif err != nil {\n\t\tif output == nil {\n\t\t\treturn errors.ErrorUnexpected(\"got unexpected nil pointer\")\n\t\t}\n\n\t\tif len(output.Failed) == 0 {\n\t\t\treturn errors.WithStack(err)\n\t\t}\n\n\t\treturn errors.Wrap(ErrorFailedToEnqueueMessages(*output.Failed[0].Message), fmt.Sprintf(\"batch %d\", uploader.messageIDToListIndex[*output.Failed[0].Id]))\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/health/health.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage health\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tkmetrics \"k8s.io/metrics/pkg/client/clientset/versioned\"\n\tctrlclient \"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\n// ClusterHealth represents the healthiness of each component of a cluster\ntype ClusterHealth struct {\n\tOperator             bool `json:\"operator\"`\n\tControllerManager    bool `json:\"controller_manager\"`\n\tPrometheus           bool `json:\"prometheus\"`\n\tAutoscaler           bool `json:\"autoscaler\"`\n\tActivator            bool `json:\"activator\"`\n\tAsyncGateway         bool `json:\"async_gateway\"`\n\tGrafana              bool `json:\"grafana\"`\n\tOperatorGateway      bool `json:\"operator_gateway\"`\n\tAPIsGateway          bool `json:\"apis_gateway\"`\n\tClusterAutoscaler    bool `json:\"cluster_autoscaler\"`\n\tOperatorLoadBalancer bool `json:\"operator_load_balancer\"`\n\tAPIsLoadBalancer     bool `json:\"apis_load_balancer\"`\n\tFluentBit            bool `json:\"fluent_bit\"`\n\tNodeExporter         bool `json:\"node_exporter\"`\n\tDCGMExporter         bool `json:\"dcgm_exporter\"`\n\tStatsDExporter       bool `json:\"statsd_exporter\"`\n\tEventExporter        bool `json:\"event_exporter\"`\n\tKubeStateMetrics     bool `json:\"kube_state_metrics\"`\n}\n\nfunc (c ClusterHealth) String() string {\n\tbytes, err := json.Marshal(c)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\treturn string(bytes)\n}\n\ntype ClusterWarnings struct {\n\tPrometheus string\n}\n\n// HasWarnings checks if ClusterWarnings has any warnings in its' fields\nfunc (w ClusterWarnings) HasWarnings() bool {\n\tv := reflect.ValueOf(w)\n\tfor i := 0; i < v.NumField(); i++ {\n\t\tif v.Field(i).String() != \"\" {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Check checks for the health of the different components of a cluster\nfunc Check(awsClient *awslib.Client, k8sClient *k8s.Client, clusterName string) (ClusterHealth, error) {\n\tvar (\n\t\toperatorHealth             bool\n\t\tcontrollerManagerHealth    bool\n\t\tprometheusHealth           bool\n\t\tautoscalerHealth           bool\n\t\tactivatorHealth            bool\n\t\tasyncGatewayHealth         bool\n\t\tgrafanaHealth              bool\n\t\toperatorGatewayHealth      bool\n\t\tapisGatewayHealth          bool\n\t\tclusterAutoscalerHealth    bool\n\t\toperatorLoadBalancerHealth bool\n\t\tapisLoadBalancerHealth     bool\n\t\tfluentBitHealth            bool\n\t\tnodeExporterHealth         bool\n\t\tdcgmExporterHealth         bool\n\t\tstatsdExporterHealth       bool\n\t\teventExporterHealth        bool\n\t\tkubeStateMetricsHealth     bool\n\t)\n\n\tif err := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\toperatorHealth, err = getDeploymentReadiness(k8sClient, \"operator\", consts.DefaultNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tcontrollerManagerHealth, err = getDeploymentReadiness(k8sClient, \"operator-controller-manager\", consts.DefaultNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tprometheusHealth, err = getStatefulSetReadiness(k8sClient, \"prometheus-prometheus\", consts.PrometheusNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tautoscalerHealth, err = getDeploymentReadiness(k8sClient, \"autoscaler\", consts.DefaultNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tactivatorHealth, err = getDeploymentReadiness(k8sClient, \"activator\", consts.DefaultNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tasyncGatewayHealth, err = getDeploymentReadiness(k8sClient, \"async-gateway\", consts.DefaultNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tgrafanaHealth, err = getStatefulSetReadiness(k8sClient, \"grafana\", consts.DefaultNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\toperatorGatewayHealth, err = getDeploymentReadiness(k8sClient, \"ingressgateway-operator\", consts.IstioNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tapisGatewayHealth, err = getDeploymentReadiness(k8sClient, \"ingressgateway-apis\", consts.IstioNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tclusterAutoscalerHealth, err = getDeploymentReadiness(k8sClient, \"cluster-autoscaler\", consts.KubeSystemNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\toperatorLoadBalancerHealth, err = getLoadBalancerHealth(awsClient, clusterName, \"operator\", false)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tapisLoadBalancerHealth, err = getLoadBalancerHealth(awsClient, clusterName, \"api\", true)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tfluentBitHealth, err = getDaemonSetReadiness(k8sClient, \"fluent-bit\", consts.LoggingNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tdcgmExporterHealth, err = getDaemonSetReadiness(k8sClient, \"dcgm-exporter\", consts.PrometheusNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tnodeExporterHealth, err = getDaemonSetReadiness(k8sClient, \"node-exporter\", consts.PrometheusNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tstatsdExporterHealth, err = getDeploymentReadiness(k8sClient, \"prometheus-statsd-exporter\", consts.PrometheusNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\teventExporterHealth, err = getDeploymentReadiness(k8sClient, \"event-exporter\", consts.LoggingNamespace)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tkubeStateMetricsHealth, err = getDeploymentReadiness(k8sClient, \"kube-state-metrics\", consts.PrometheusNamespace)\n\t\t\treturn err\n\t\t},\n\t); err != nil {\n\t\treturn ClusterHealth{}, err\n\t}\n\n\treturn ClusterHealth{\n\t\tOperator:             operatorHealth,\n\t\tControllerManager:    controllerManagerHealth,\n\t\tPrometheus:           prometheusHealth,\n\t\tAutoscaler:           autoscalerHealth,\n\t\tActivator:            activatorHealth,\n\t\tAsyncGateway:         asyncGatewayHealth,\n\t\tGrafana:              grafanaHealth,\n\t\tOperatorGateway:      operatorGatewayHealth,\n\t\tAPIsGateway:          apisGatewayHealth,\n\t\tClusterAutoscaler:    clusterAutoscalerHealth,\n\t\tOperatorLoadBalancer: operatorLoadBalancerHealth,\n\t\tAPIsLoadBalancer:     apisLoadBalancerHealth,\n\t\tFluentBit:            fluentBitHealth,\n\t\tNodeExporter:         nodeExporterHealth,\n\t\tDCGMExporter:         dcgmExporterHealth,\n\t\tStatsDExporter:       statsdExporterHealth,\n\t\tEventExporter:        eventExporterHealth,\n\t\tKubeStateMetrics:     kubeStateMetricsHealth,\n\t}, nil\n}\n\nfunc GetWarnings(k8sClient *k8s.Client) (ClusterWarnings, error) {\n\tvar prometheusMemorySaturationWarn string\n\n\tsaturation, err := getPodMemorySaturation(k8sClient, \"prometheus-prometheus-0\", consts.PrometheusNamespace)\n\tif err != nil {\n\t\treturn ClusterWarnings{}, err\n\t}\n\n\tif saturation >= 0.7 {\n\t\tprometheusMemorySaturationWarn = fmt.Sprintf(\"memory usage is critically high (%.1f%%)\", saturation*100)\n\t}\n\n\treturn ClusterWarnings{\n\t\tPrometheus: prometheusMemorySaturationWarn,\n\t}, nil\n}\n\nfunc getDeploymentReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) {\n\tctx := context.Background()\n\tvar deployment kapps.Deployment\n\tif err := k8sClient.Get(ctx, ctrlclient.ObjectKey{\n\t\tNamespace: namespace,\n\t\tName:      name,\n\t}, &deployment); err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, err\n\t}\n\n\treturn deployment.Status.ReadyReplicas > 0, nil\n}\n\nfunc getStatefulSetReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) {\n\tctx := context.Background()\n\tvar statefulSet kapps.StatefulSet\n\tif err := k8sClient.Get(ctx, ctrlclient.ObjectKey{\n\t\tNamespace: namespace,\n\t\tName:      name,\n\t}, &statefulSet); err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, err\n\t}\n\treturn statefulSet.Status.ReadyReplicas > 0, nil\n}\n\nfunc getDaemonSetReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) {\n\tctx := context.Background()\n\tvar daemonSet kapps.DaemonSet\n\tif err := k8sClient.Get(ctx, ctrlclient.ObjectKey{\n\t\tNamespace: namespace,\n\t\tName:      name,\n\t}, &daemonSet); err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, err\n\t}\n\treturn daemonSet.Status.NumberReady == daemonSet.Status.CurrentNumberScheduled, nil\n}\n\nfunc getLoadBalancerHealth(awsClient *awslib.Client, clusterName string, loadBalancerName string, testClassicLB bool) (bool, error) {\n\tloadBalancerV2, err := awsClient.FindLoadBalancerV2(map[string]string{\n\t\tclusterconfig.ClusterNameTag: clusterName,\n\t\t\"cortex.dev/load-balancer\":   loadBalancerName,\n\t})\n\tloadBalancerV2Exists := err == nil && loadBalancerV2 != nil\n\n\tif loadBalancerV2Exists {\n\t\treturn aws.IsLoadBalancerV2Healthy(*loadBalancerV2), nil\n\t}\n\n\tif !testClassicLB {\n\t\tif err == nil {\n\t\t\treturn false, errors.ErrorUnexpected(fmt.Sprintf(\"unable to locate %s nlb load balancer\", loadBalancerName))\n\t\t}\n\t\treturn false, errors.Wrap(err, fmt.Sprintf(\"unable to locate %s nlb load balancer\", loadBalancerName))\n\t}\n\n\tloadBalancer, err := awsClient.FindLoadBalancer(map[string]string{\n\t\tclusterconfig.ClusterNameTag: clusterName,\n\t\t\"cortex.dev/load-balancer\":   loadBalancerName,\n\t})\n\tloadBalancerExists := err == nil && loadBalancer != nil\n\tif !loadBalancerExists {\n\t\tif err == nil {\n\t\t\treturn false, errors.ErrorUnexpected(fmt.Sprintf(\"unable to locate %s elb load balancer\", loadBalancerName))\n\t\t}\n\t\treturn false, errors.Wrap(err, fmt.Sprintf(\"unable to locate %s elb load balancer\", loadBalancerName))\n\t}\n\thealthy, err := awsClient.IsLoadBalancerHealthy(*loadBalancer.LoadBalancerName)\n\tif err != nil {\n\t\treturn false, errors.Wrap(err, fmt.Sprintf(\"unable to check %s elb load balancer\", loadBalancerName))\n\t}\n\treturn healthy, nil\n}\n\nfunc getPodMemorySaturation(k8sClient *k8s.Client, podName, namespace string) (float64, error) {\n\tctx := context.Background()\n\tvar pod kcore.Pod\n\tif err := k8sClient.Get(ctx, ctrlclient.ObjectKey{\n\t\tNamespace: namespace,\n\t\tName:      podName,\n\t}, &pod); err != nil {\n\t\treturn 0, err\n\t}\n\n\tmetricsClient, err := kmetrics.NewForConfig(k8sClient.RestConfig)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tpodMetrics, err := metricsClient.MetricsV1beta1().PodMetricses(namespace).Get(ctx, podName, kmeta.GetOptions{})\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tvar totalMememoryUsage kresource.Quantity\n\tfor _, container := range podMetrics.Containers {\n\t\tmemory := container.Usage.Memory()\n\t\tif memory != nil {\n\t\t\ttotalMememoryUsage.Add(*container.Usage.Memory())\n\t\t}\n\t}\n\n\tnode, err := k8sClient.ClientSet().CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, kmeta.GetOptions{})\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tnodeMemory := node.Status.Allocatable.Memory()\n\n\tmemRatio := totalMememoryUsage.AsApproximateFloat64() / nodeMemory.AsApproximateFloat64()\n\n\treturn memRatio, nil\n}\n"
  },
  {
    "path": "pkg/lib/archive/archive_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage archive\n\nimport (\n\t\"io/ioutil\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/maps\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestArchive(t *testing.T) {\n\ttmpDir, err := files.TmpDir()\n\tdefer os.RemoveAll(tmpDir)\n\trequire.NoError(t, err)\n\n\tfilesList := []string{\n\t\tfilepath.Join(tmpDir, \"1.txt\"),\n\t\tfilepath.Join(tmpDir, \"2.py\"),\n\t\tfilepath.Join(tmpDir, \"3/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/2.txt\"),\n\t\tfilepath.Join(tmpDir, \"3/2/3/.tmp\"),\n\t\tfilepath.Join(tmpDir, \"4/1.yaml\"),\n\t\tfilepath.Join(tmpDir, \"4/2.pyc\"),\n\t\tfilepath.Join(tmpDir, \"5/4/3/2/1.txt\"),\n\t\tfilepath.Join(tmpDir, \"5/4/3/2/1.py\"),\n\t\tfilepath.Join(tmpDir, \"5/4/3/2/2/1.py\"),\n\t}\n\n\terr = files.MakeEmptyFiles(filesList[0], filesList[1:]...)\n\trequire.NoError(t, err)\n\n\tvar input *Input\n\tvar expected []string\n\n\tinput = &Input{\n\t\tBytes: []BytesInput{\n\t\t\t{\n\t\t\t\tContent: []byte(\"\"),\n\t\t\t\tDest:    \"text.txt\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tContent: []byte(\"\"),\n\t\t\t\tDest:    \"test2/text2.txt\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tContent: []byte(\"\"),\n\t\t\t\tDest:    \"/test3/text3.txt\",\n\t\t\t},\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"text.txt\",\n\t\t\"test2/text2.txt\",\n\t\t\"test3/text3.txt\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tFiles: []FileInput{\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"1.txt\"),\n\t\t\t\tDest:   \"1.txt\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"3/1.py\"),\n\t\t\t\tDest:   \"3/1.py\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"3/2/1.py\"),\n\t\t\t\tDest:   \"3/2/1.py\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"3/2/3/.tmp\"),\n\t\t\t\tDest:   \"3/2/3/.tmp\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"4/2.pyc\"),\n\t\t\t\tDest:   \"4/4/2.pyc\",\n\t\t\t},\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"1.txt\",\n\t\t\"3/1.py\",\n\t\t\"3/2/1.py\",\n\t\t\"3/2/3/.tmp\",\n\t\t\"4/4/2.pyc\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tBytes: []BytesInput{\n\t\t\t{\n\t\t\t\tContent: []byte(\"\"),\n\t\t\t\tDest:    \"text.txt\",\n\t\t\t},\n\t\t},\n\t\tFiles: []FileInput{\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"1.txt\"),\n\t\t\t\tDest:   \"1/2/3.txt\",\n\t\t\t},\n\t\t},\n\t\tAddPrefix: \"test\",\n\t}\n\texpected = []string{\n\t\t\"test/text.txt\",\n\t\t\"test/1/2/3.txt\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tBytes: []BytesInput{\n\t\t\t{\n\t\t\t\tContent: []byte(\"\"),\n\t\t\t\tDest:    \"text.txt\",\n\t\t\t},\n\t\t},\n\t\tFiles: []FileInput{\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"1.txt\"),\n\t\t\t\tDest:   \"1/2/3.txt\",\n\t\t\t},\n\t\t},\n\t\tAddPrefix: \"/test\",\n\t}\n\texpected = []string{\n\t\t\"test/text.txt\",\n\t\t\"test/1/2/3.txt\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tEmptyFiles: []string{\n\t\t\t\"text.txt\",\n\t\t\t\"1/2/3.txt\",\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"text.txt\",\n\t\t\"1/2/3.txt\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource: tmpDir,\n\t\t\t},\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"1.txt\",\n\t\t\"2.py\",\n\t\t\"3/1.py\",\n\t\t\"3/2/1.py\",\n\t\t\"3/2/2.txt\",\n\t\t\"3/2/3/.tmp\",\n\t\t\"4/1.yaml\",\n\t\t\"4/2.pyc\",\n\t\t\"5/4/3/2/1.txt\",\n\t\t\"5/4/3/2/1.py\",\n\t\t\"5/4/3/2/2/1.py\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"3\"),\n\t\t\t\tDest:   \".\",\n\t\t\t},\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"1.py\",\n\t\t\"2/1.py\",\n\t\t\"2/2.txt\",\n\t\t\"2/3/.tmp\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource:    tmpDir,\n\t\t\t\tDest:      \"/\",\n\t\t\t\tIgnoreFns: []files.IgnoreFn{files.IgnoreHiddenFiles},\n\t\t\t},\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"1.txt\",\n\t\t\"2.py\",\n\t\t\"3/1.py\",\n\t\t\"3/2/1.py\",\n\t\t\"3/2/2.txt\",\n\t\t\"4/1.yaml\",\n\t\t\"4/2.pyc\",\n\t\t\"5/4/3/2/1.txt\",\n\t\t\"5/4/3/2/1.py\",\n\t\t\"5/4/3/2/2/1.py\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource:    filepath.Join(tmpDir, \"3\"),\n\t\t\t\tIgnoreFns: []files.IgnoreFn{files.IgnoreHiddenFiles},\n\t\t\t\tDest:      \"test3\",\n\t\t\t\tFlatten:   true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tSource: filepath.Join(tmpDir, \"4\"),\n\t\t\t\tDest:   \"test4/\",\n\t\t\t},\n\t\t},\n\t\tAllowOverwrite: true,\n\t}\n\texpected = []string{\n\t\t\"test3/1.py\",\n\t\t\"test3/2.txt\",\n\t\t\"test4/1.yaml\",\n\t\t\"test4/2.pyc\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource:       filepath.Join(tmpDir, \"5\"),\n\t\t\t\tRemovePrefix: \"4/3\",\n\t\t\t},\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"2/1.txt\",\n\t\t\"2/1.py\",\n\t\t\"2/2/1.py\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource:       filepath.Join(tmpDir, \"5\"),\n\t\t\t\tRemovePrefix: \"/4/3\",\n\t\t\t},\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"2/1.txt\",\n\t\t\"2/1.py\",\n\t\t\"2/2/1.py\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource:  filepath.Join(tmpDir, \"5\"),\n\t\t\t\tFlatten: true,\n\t\t\t},\n\t\t},\n\t\tAllowOverwrite: true,\n\t}\n\texpected = []string{\n\t\t\"1.txt\",\n\t\t\"1.py\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource:             filepath.Join(tmpDir, \"5\"),\n\t\t\t\tRemoveCommonPrefix: true,\n\t\t\t},\n\t\t},\n\t}\n\texpected = []string{\n\t\t\"1.txt\",\n\t\t\"1.py\",\n\t\t\"2/1.py\",\n\t}\n\tCheckArchive(input, expected, false, t)\n\n\tinput = &Input{\n\t\tBytes: []BytesInput{\n\t\t\t{\n\t\t\t\tContent: []byte(\"\"),\n\t\t\t\tDest:    \"1/text.txt\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tContent: []byte(\"\"),\n\t\t\t\tDest:    \"1/text.txt\",\n\t\t\t},\n\t\t},\n\t}\n\tCheckArchive(input, nil, true, t)\n\n\tinput = &Input{\n\t\tBytes: []BytesInput{\n\t\t\t{\n\t\t\t\tContent: []byte(\"\"),\n\t\t\t\tDest:    \"1/text.txt\",\n\t\t\t},\n\t\t},\n\t\tEmptyFiles: []string{\"1/text.txt\"},\n\t}\n\tCheckArchive(input, nil, true, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource:  filepath.Join(tmpDir, \"3\"),\n\t\t\t\tFlatten: true,\n\t\t\t},\n\t\t},\n\t}\n\tCheckArchive(input, nil, true, t)\n\n\tinput = &Input{\n\t\tDirs: []DirInput{\n\t\t\t{\n\t\t\t\tSource:  filepath.Join(tmpDir, \"5\"),\n\t\t\t\tFlatten: true,\n\t\t\t},\n\t\t},\n\t}\n\tCheckArchive(input, nil, true, t)\n}\n\nfunc CheckArchive(input *Input, expected []string, shouldErr bool, t *testing.T) {\n\tCheckZip(input, expected, shouldErr, t)\n\tCheckTar(input, expected, shouldErr, t)\n\tCheckTgz(input, expected, shouldErr, t)\n}\n\nfunc CheckZip(input *Input, expected []string, shouldErr bool, t *testing.T) {\n\ttmpDir, err := files.TmpDir()\n\tdefer os.RemoveAll(tmpDir)\n\trequire.NoError(t, err)\n\n\t_, err = ZipToFile(input, filepath.Join(tmpDir, \"archive.zip\"))\n\tif shouldErr {\n\t\trequire.Error(t, err)\n\t\treturn\n\t}\n\trequire.NoError(t, err)\n\n\t_, err = UnzipFileToDir(filepath.Join(tmpDir, \"archive.zip\"), filepath.Join(tmpDir, \"archive\"))\n\trequire.NoError(t, err)\n\n\tunzippedFiles, err := files.ListDirRecursive(filepath.Join(tmpDir, \"archive\"), true)\n\trequire.NoError(t, err)\n\n\trequire.ElementsMatch(t, expected, unzippedFiles)\n\n\tcontents, err := UnzipFileToMem(filepath.Join(tmpDir, \"archive.zip\"))\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents))\n\n\tzipBytes, err := ioutil.ReadFile(filepath.Join(tmpDir, \"archive.zip\"))\n\trequire.NoError(t, err)\n\tcontents, err = UnzipMemToMem(zipBytes)\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents))\n}\n\nfunc CheckTar(input *Input, expected []string, shouldErr bool, t *testing.T) {\n\ttmpDir, err := files.TmpDir()\n\tdefer os.RemoveAll(tmpDir)\n\trequire.NoError(t, err)\n\n\t_, err = TarToFile(input, filepath.Join(tmpDir, \"archive.tar\"))\n\tif shouldErr {\n\t\trequire.Error(t, err)\n\t\treturn\n\t}\n\trequire.NoError(t, err)\n\n\t_, err = UntarFileToDir(filepath.Join(tmpDir, \"archive.tar\"), filepath.Join(tmpDir, \"archive\"))\n\trequire.NoError(t, err)\n\n\tuntaredFiles, err := files.ListDirRecursive(filepath.Join(tmpDir, \"archive\"), true)\n\trequire.NoError(t, err)\n\n\trequire.ElementsMatch(t, expected, untaredFiles)\n\n\tcontents, err := UntarFileToMem(filepath.Join(tmpDir, \"archive.tar\"))\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents))\n\n\ttarBytes, err := ioutil.ReadFile(filepath.Join(tmpDir, \"archive.tar\"))\n\trequire.NoError(t, err)\n\tcontents, err = UntarMemToMem(tarBytes)\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents))\n}\n\nfunc CheckTgz(input *Input, expected []string, shouldErr bool, t *testing.T) {\n\ttmpDir, err := files.TmpDir()\n\tdefer os.RemoveAll(tmpDir)\n\trequire.NoError(t, err)\n\n\t_, err = TgzToFile(input, filepath.Join(tmpDir, \"archive.tgz\"))\n\tif shouldErr {\n\t\trequire.Error(t, err)\n\t\treturn\n\t}\n\trequire.NoError(t, err)\n\n\t_, err = UntgzFileToDir(filepath.Join(tmpDir, \"archive.tgz\"), filepath.Join(tmpDir, \"archive\"))\n\trequire.NoError(t, err)\n\n\tuntgzedFiles, err := files.ListDirRecursive(filepath.Join(tmpDir, \"archive\"), true)\n\trequire.NoError(t, err)\n\n\trequire.ElementsMatch(t, expected, untgzedFiles)\n\n\tcontents, err := UntgzFileToMem(filepath.Join(tmpDir, \"archive.tgz\"))\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents))\n\n\ttgzBytes, err := ioutil.ReadFile(filepath.Join(tmpDir, \"archive.tgz\"))\n\trequire.NoError(t, err)\n\tcontents, err = UntgzMemToMem(tgzBytes)\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, maps.InterfaceMapKeysUnsafe(contents))\n}\n"
  },
  {
    "path": "pkg/lib/archive/archiver.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage archive\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype archiveType int\n\nconst (\n\tunknownArchiveType archiveType = iota\n\tzipArchiveType\n\ttarArchiveType\n\ttgzArchiveType\n)\n\ntype archiver interface {\n\tadd(reader io.Reader, dest string, size int64) error\n\tclose() error\n}\n\nfunc archive(input *Input, arc archiver) (strset.Set, error) {\n\taddedPaths := strset.New()\n\tvar err error\n\n\tfor i := range input.Bytes {\n\t\terr = addBytesToArchive(&input.Bytes[i], input, arc, addedPaths)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tfor i := range input.Files {\n\t\terr = addFileToArchive(&input.Files[i], input, arc, addedPaths)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tfor i := range input.Dirs {\n\t\terr = addDirToArchive(&input.Dirs[i], input, arc, addedPaths)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tfor i := range input.FileLists {\n\t\terr = addFileListToArchive(&input.FileLists[i], input, arc, addedPaths)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tfor _, emptyFilePath := range input.EmptyFiles {\n\t\terr = addEmptyFileToArchive(emptyFilePath, input, arc, addedPaths)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn addedPaths, nil\n}\n\nfunc addBytesToArchive(byteInput *BytesInput, input *Input, arc archiver, addedPaths strset.Set) error {\n\tpath := filepath.Join(input.AddPrefix, byteInput.Dest)\n\tpath = strings.TrimPrefix(path, \"/\")\n\n\tif !input.AllowOverwrite {\n\t\tif addedPaths.Has(path) {\n\t\t\treturn ErrorDuplicatePath(path)\n\t\t}\n\t\taddedPaths.Add(path)\n\t}\n\n\treader := bytes.NewReader(byteInput.Content)\n\treturn arc.add(reader, path, reader.Size())\n}\n\nfunc addFileToArchive(fileInput *FileInput, input *Input, arc archiver, addedPaths strset.Set) error {\n\tcontent, err := files.ReadFileBytes(fileInput.Source)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbyteInput := &BytesInput{\n\t\tContent: content,\n\t\tDest:    fileInput.Dest,\n\t}\n\n\treturn addBytesToArchive(byteInput, input, arc, addedPaths)\n}\n\nfunc addDirToArchive(dirInput *DirInput, input *Input, arc archiver, addedPaths strset.Set) error {\n\tpaths, err := files.ListDirRecursive(dirInput.Source, true, dirInput.IgnoreFns...)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tcommonPrefix := \"\"\n\tif dirInput.RemoveCommonPrefix {\n\t\tcommonPrefix = s.LongestCommonPrefix(paths...)\n\t}\n\n\tremovePrefix := strings.TrimPrefix(dirInput.RemovePrefix, \"/\")\n\n\tfor _, path := range paths {\n\t\tfile := filepath.Join(dirInput.Source, path)\n\n\t\tif dirInput.Flatten {\n\t\t\tpath = filepath.Base(path)\n\t\t} else {\n\t\t\tpath = strings.TrimPrefix(path, removePrefix)\n\t\t\tpath = strings.TrimPrefix(path, commonPrefix)\n\t\t}\n\n\t\tfileInput := &FileInput{\n\t\t\tSource: file,\n\t\t\tDest:   filepath.Join(dirInput.Dest, path),\n\t\t}\n\n\t\terr = addFileToArchive(fileInput, input, arc, addedPaths)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc addFileListToArchive(fileListInput *FileListInput, input *Input, arc archiver, addedPaths strset.Set) error {\n\tcommonPrefix := \"\"\n\tif fileListInput.RemoveCommonPrefix {\n\t\tcommonPrefix = s.LongestCommonPrefix(fileListInput.Sources...)\n\t}\n\n\tfor _, path := range fileListInput.Sources {\n\t\tfullPath := path\n\n\t\tif fileListInput.Flatten {\n\t\t\tpath = filepath.Base(path)\n\t\t} else {\n\t\t\tpath = strings.TrimPrefix(path, fileListInput.RemovePrefix)\n\t\t\tpath = strings.TrimPrefix(path, commonPrefix)\n\t\t}\n\n\t\tfileInput := &FileInput{\n\t\t\tSource: fullPath,\n\t\t\tDest:   filepath.Join(fileListInput.Dest, path),\n\t\t}\n\n\t\terr := addFileToArchive(fileInput, input, arc, addedPaths)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc addEmptyFileToArchive(path string, input *Input, arc archiver, addedPaths strset.Set) error {\n\tbyteInput := &BytesInput{\n\t\tContent: []byte{},\n\t\tDest:    path,\n\t}\n\treturn addBytesToArchive(byteInput, input, arc, addedPaths)\n}\n\nfunc archiveToWriter(input *Input, writer io.Writer, arcType archiveType) (strset.Set, error) {\n\tvar arc archiver\n\tswitch arcType {\n\tcase zipArchiveType:\n\t\tarc = newZipArchiver(writer)\n\tcase tarArchiveType:\n\t\tarc = newTarArchiver(writer)\n\tcase tgzArchiveType:\n\t\tarc = newTgzArchiver(writer)\n\tdefault:\n\t\treturn nil, errors.ErrorUnexpected(\"unknown archive type:\", arcType)\n\t}\n\n\tpaths, err := archive(input, arc)\n\tif err != nil {\n\t\tarc.close()\n\t\treturn nil, err\n\t}\n\n\terr = arc.close()\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrCreateArchive)\n\t}\n\n\treturn paths, nil\n}\n\nfunc archiveToMem(input *Input, arcType archiveType) ([]byte, strset.Set, error) {\n\tbuf := new(bytes.Buffer)\n\n\tpaths, err := archiveToWriter(input, buf, arcType)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\treturn buf.Bytes(), paths, nil\n}\n\nfunc archiveToFile(input *Input, destDir string, arcType archiveType) (strset.Set, error) {\n\tcleanDestDir, err := files.EscapeTilde(destDir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tarchiveFile, err := files.Create(cleanDestDir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tpaths, err := archiveToWriter(input, archiveFile, arcType)\n\tif err != nil {\n\t\tarchiveFile.Close()\n\t\treturn nil, err\n\t}\n\n\terr = archiveFile.Close()\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, destDir, _errStrCreateArchive)\n\t}\n\n\treturn paths, nil\n}\n"
  },
  {
    "path": "pkg/lib/archive/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage archive\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\t_errStrCreateArchive = \"unable to create archive\"\n\t_errStrCreateZip     = \"unable to create zip file\"\n\t_errStrCreateTar     = \"unable to create tar file\"\n\t_errStrUnzip         = \"unable to unzip file\"\n\t_errStrUntar         = \"unable to extract tar file\"\n)\n\nconst (\n\tErrDuplicatePath = \"archive.duplicate_path\"\n)\n\nfunc ErrorDuplicatePath(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDuplicatePath,\n\t\tMessage: fmt.Sprintf(\"duplicate path was provided (%s)\", s.UserStr(path)),\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/archive/input.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage archive\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n)\n\ntype Input struct {\n\tFiles          []FileInput\n\tBytes          []BytesInput\n\tDirs           []DirInput\n\tFileLists      []FileListInput\n\tAddPrefix      string   // Gets added to every item\n\tEmptyFiles     []string // Empty files to be created\n\tAllowOverwrite bool     // Don't error if a file in the zip is overwritten\n}\n\ntype FileInput struct {\n\tSource string\n\tDest   string\n}\n\ntype BytesInput struct {\n\tContent []byte\n\tDest    string\n}\n\ntype DirInput struct {\n\tSource             string\n\tDest               string\n\tIgnoreFns          []files.IgnoreFn\n\tFlatten            bool\n\tRemovePrefix       string\n\tRemoveCommonPrefix bool\n}\n\ntype FileListInput struct {\n\tSources            []string\n\tDest               string\n\tFlatten            bool\n\tRemovePrefix       string\n\tRemoveCommonPrefix bool\n}\n"
  },
  {
    "path": "pkg/lib/archive/tar.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage archive\n\nimport (\n\t\"archive/tar\"\n\t\"bytes\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\ntype tarArchiver struct {\n\twriter *tar.Writer\n}\n\nfunc newTarArchiver(writer io.Writer) *tarArchiver {\n\treturn &tarArchiver{\n\t\twriter: tar.NewWriter(writer),\n\t}\n}\n\nfunc (arc *tarArchiver) add(reader io.Reader, dest string, size int64) error {\n\theader := &tar.Header{\n\t\tName: dest,\n\t\tSize: size,\n\t}\n\n\terr := arc.writer.WriteHeader(header)\n\tif err != nil {\n\t\treturn errors.Wrap(err, _errStrCreateTar)\n\t}\n\n\t_, err = io.Copy(arc.writer, reader)\n\tif err != nil {\n\t\treturn errors.Wrap(err, _errStrCreateTar)\n\t}\n\n\treturn nil\n}\n\nfunc (arc *tarArchiver) close() error {\n\treturn arc.writer.Close()\n}\n\nfunc TarToWriter(input *Input, writer io.Writer) (strset.Set, error) {\n\treturn archiveToWriter(input, writer, tarArchiveType)\n}\n\nfunc TarToFile(input *Input, destDir string) (strset.Set, error) {\n\treturn archiveToFile(input, destDir, tarArchiveType)\n}\n\nfunc TarToMem(input *Input) ([]byte, strset.Set, error) {\n\treturn archiveToMem(input, tarArchiveType)\n}\n\n// Will create destDir if missing\nfunc UntarReaderToDir(reader io.Reader, destDir string) (strset.Set, error) {\n\tdestDir, err := files.Clean(destDir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\ttarReader := tar.NewReader(reader)\n\n\tfilenames := strset.New()\n\n\tfor {\n\t\theader, err := tarReader.Next()\n\n\t\tswitch {\n\t\tcase err == io.EOF:\n\t\t\treturn filenames, nil\n\n\t\tcase err != nil:\n\t\t\treturn nil, errors.WithStack(err)\n\n\t\tcase header == nil:\n\t\t\tcontinue\n\t\t}\n\n\t\tname := strings.TrimPrefix(header.Name, \"/\")\n\t\ttarget := filepath.Join(destDir, name)\n\n\t\tswitch header.Typeflag {\n\t\tcase tar.TypeDir:\n\t\t\terr := files.CreateDir(target)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\tcase tar.TypeReg:\n\t\t\tfilenames.Add(target)\n\n\t\t\terr := files.CreateDir(filepath.Dir(target))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\toutFile, err := files.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\t_, err = io.Copy(outFile, tarReader)\n\t\t\tif err != nil {\n\t\t\t\toutFile.Close()\n\t\t\t\treturn nil, errors.Wrap(err, _errStrUntar)\n\t\t\t}\n\n\t\t\toutFile.Close()\n\t\t}\n\t}\n}\n\n// Will create destDir if missing\nfunc UntarFileToDir(src string, destDir string) (strset.Set, error) {\n\tfile, err := files.Open(src)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer file.Close()\n\n\treturn UntarReaderToDir(file, destDir)\n}\n\nfunc UntarReaderToMem(reader io.Reader) (map[string][]byte, error) {\n\tfileMap := map[string][]byte{}\n\n\ttarReader := tar.NewReader(reader)\n\n\tfor {\n\t\theader, err := tarReader.Next()\n\n\t\tswitch {\n\t\tcase err == io.EOF:\n\t\t\treturn fileMap, nil\n\n\t\tcase err != nil:\n\t\t\treturn nil, err\n\n\t\tcase header == nil:\n\t\t\tcontinue\n\t\t}\n\n\t\tif header.Typeflag == tar.TypeReg {\n\t\t\tcontents, err := ioutil.ReadAll(tarReader)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrap(err, _errStrUntar)\n\t\t\t}\n\n\t\t\tpath := strings.TrimPrefix(header.Name, \"/\")\n\t\t\tfileMap[path] = contents\n\t\t}\n\t}\n}\n\nfunc UntarMemToMem(tarBytes []byte) (map[string][]byte, error) {\n\treturn UntarReaderToMem(bytes.NewReader(tarBytes))\n}\n\nfunc UntarFileToMem(src string) (map[string][]byte, error) {\n\tfile, err := files.Open(src)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrUntar)\n\t}\n\tdefer file.Close()\n\n\treturn UntarReaderToMem(file)\n}\n"
  },
  {
    "path": "pkg/lib/archive/tgz.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage archive\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"io\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\ntype tgzArchiver struct {\n\ttarArc     *tarArchiver\n\tgzipWriter *gzip.Writer\n}\n\nfunc newTgzArchiver(writer io.Writer) *tgzArchiver {\n\tgzipWriter := gzip.NewWriter(writer)\n\treturn &tgzArchiver{\n\t\ttarArc:     newTarArchiver(gzipWriter),\n\t\tgzipWriter: gzipWriter,\n\t}\n}\n\nfunc (arc *tgzArchiver) add(reader io.Reader, dest string, size int64) error {\n\treturn arc.tarArc.add(reader, dest, size)\n}\n\nfunc (arc *tgzArchiver) close() error {\n\terr1 := arc.tarArc.close()\n\terr2 := arc.gzipWriter.Close()\n\treturn errors.FirstError(err1, err2)\n}\n\nfunc TgzToWriter(input *Input, writer io.Writer) (strset.Set, error) {\n\treturn archiveToWriter(input, writer, tgzArchiveType)\n}\n\nfunc TgzToFile(input *Input, destDir string) (strset.Set, error) {\n\treturn archiveToFile(input, destDir, tgzArchiveType)\n}\n\nfunc TgzToMem(input *Input) ([]byte, strset.Set, error) {\n\treturn archiveToMem(input, tgzArchiveType)\n}\n\n// Will create destDir if missing\nfunc UntgzReaderToDir(reader io.Reader, destDir string) (strset.Set, error) {\n\tgzipReader, err := gzip.NewReader(reader)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer gzipReader.Close()\n\n\treturn UntarReaderToDir(gzipReader, destDir)\n}\n\n// Will create destDir if missing\nfunc UntgzFileToDir(src string, destDir string) (strset.Set, error) {\n\tfile, err := files.Open(src)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer file.Close()\n\n\treturn UntgzReaderToDir(file, destDir)\n}\n\nfunc UntgzReaderToMem(reader io.Reader) (map[string][]byte, error) {\n\tgzipReader, err := gzip.NewReader(reader)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer gzipReader.Close()\n\n\treturn UntarReaderToMem(gzipReader)\n}\n\nfunc UntgzMemToMem(tgzBytes []byte) (map[string][]byte, error) {\n\treturn UntgzReaderToMem(bytes.NewReader(tgzBytes))\n}\n\nfunc UntgzFileToMem(src string) (map[string][]byte, error) {\n\tfile, err := files.Open(src)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrUntar)\n\t}\n\tdefer file.Close()\n\n\treturn UntgzReaderToMem(file)\n}\n"
  },
  {
    "path": "pkg/lib/archive/zip.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage archive\n\nimport (\n\t\"archive/zip\"\n\t\"bytes\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\ntype zipArchiver struct {\n\twriter *zip.Writer\n}\n\nfunc newZipArchiver(writer io.Writer) *zipArchiver {\n\treturn &zipArchiver{\n\t\twriter: zip.NewWriter(writer),\n\t}\n}\n\nfunc (arc *zipArchiver) add(reader io.Reader, dest string, size int64) error {\n\twriter, err := arc.writer.Create(dest)\n\tif err != nil {\n\t\treturn errors.Wrap(err, _errStrCreateZip)\n\t}\n\n\t_, err = io.Copy(writer, reader)\n\tif err != nil {\n\t\treturn errors.Wrap(err, _errStrCreateZip)\n\t}\n\n\treturn nil\n}\n\nfunc (arc *zipArchiver) close() error {\n\treturn arc.writer.Close()\n}\n\nfunc ZipToWriter(input *Input, writer io.Writer) (strset.Set, error) {\n\treturn archiveToWriter(input, writer, zipArchiveType)\n}\n\nfunc ZipToFile(input *Input, destDir string) (strset.Set, error) {\n\treturn archiveToFile(input, destDir, zipArchiveType)\n}\n\nfunc ZipToMem(input *Input) ([]byte, strset.Set, error) {\n\treturn archiveToMem(input, zipArchiveType)\n}\n\n// Will create destDir if missing\nfunc UnzipFileToDir(src string, destDir string) (strset.Set, error) {\n\tdestDir, err := files.Clean(destDir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tcleanSrc, err := files.EscapeTilde(src)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfilenames := strset.New()\n\n\tzipReader, err := zip.OpenReader(cleanSrc)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrUnzip)\n\t}\n\tdefer zipReader.Close()\n\n\tfor _, zipReaderFile := range zipReader.File {\n\t\tzipFileReader, err := zipReaderFile.Open()\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, _errStrUnzip)\n\t\t}\n\t\tdefer zipFileReader.Close()\n\n\t\tname := strings.TrimPrefix(zipReaderFile.Name, \"/\")\n\t\ttarget := filepath.Join(destDir, name)\n\n\t\tif zipReaderFile.FileInfo().IsDir() {\n\t\t\terr := files.CreateDir(target)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t} else {\n\t\t\tfilenames.Add(target)\n\n\t\t\terr := files.CreateDir(filepath.Dir(target))\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\toutFile, err := files.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\t_, err = io.Copy(outFile, zipFileReader)\n\t\t\tif err != nil {\n\t\t\t\toutFile.Close()\n\t\t\t\treturn nil, errors.Wrap(err, _errStrUnzip)\n\t\t\t}\n\n\t\t\toutFile.Close()\n\t\t}\n\t}\n\treturn filenames, nil\n}\n\nfunc UnzipMemToMem(zipBytes []byte) (map[string][]byte, error) {\n\tzipReader, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrUnzip)\n\t}\n\n\treturn unzipZipReaderToMem(zipReader)\n}\n\nfunc UnzipFileToMem(src string) (map[string][]byte, error) {\n\tcleanSrc, err := files.Clean(src)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tzipReader, err := zip.OpenReader(cleanSrc)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, _errStrUnzip)\n\t}\n\tdefer zipReader.Close()\n\n\treturn unzipZipReaderToMem(&zipReader.Reader)\n}\n\nfunc unzipZipReaderToMem(zipReader *zip.Reader) (map[string][]byte, error) {\n\tfileMap := map[string][]byte{}\n\n\tfor _, zipReaderFile := range zipReader.File {\n\t\tif !zipReaderFile.FileInfo().IsDir() {\n\t\t\tzipFileReader, err := zipReaderFile.Open()\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrap(err, _errStrUnzip)\n\t\t\t}\n\t\t\tdefer zipFileReader.Close()\n\n\t\t\tcontents, err := ioutil.ReadAll(zipFileReader)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrap(err, _errStrUnzip)\n\t\t\t}\n\n\t\t\tpath := strings.TrimPrefix(zipReaderFile.Name, \"/\")\n\t\t\tfileMap[path] = contents\n\t\t}\n\t}\n\n\treturn fileMap, nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/acm.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/acm\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nfunc (c *Client) DoesCertificateExist(sslCertificateARN string) (bool, error) {\n\t_, err := c.ACM().DescribeCertificate(&acm.DescribeCertificateInput{\n\t\tCertificateArn: aws.String(sslCertificateARN),\n\t})\n\n\tif err != nil {\n\t\tif IsErrCode(err, \"ResourceNotFoundException\") {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.Wrap(err, sslCertificateARN)\n\t}\n\n\treturn true, nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/apigateway.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/apigatewayv2\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\n// CreateAPIGateway Creates a new API Gateway with the default stage\nfunc (c *Client) CreateAPIGateway(name string, tags map[string]string) (string, error) {\n\tcreateAPIResponse, err := c.APIGatewayV2().CreateApi(&apigatewayv2.CreateApiInput{\n\t\tName:         aws.String(name),\n\t\tProtocolType: aws.String(apigatewayv2.ProtocolTypeHttp),\n\t\tTags:         aws.StringMap(tags),\n\t})\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to create api gateway\")\n\t}\n\tif createAPIResponse.ApiId == nil {\n\t\treturn \"\", errors.ErrorUnexpected(\"failed to create api gateway\")\n\t}\n\n\t_, err = c.APIGatewayV2().CreateStage(&apigatewayv2.CreateStageInput{\n\t\tApiId:      createAPIResponse.ApiId,\n\t\tAutoDeploy: aws.Bool(true),\n\t\tStageName:  aws.String(\"$default\"),\n\t\tTags:       aws.StringMap(tags),\n\t})\n\tif err != nil {\n\t\tc.DeleteAPIGateway(*createAPIResponse.ApiId) // best effort cleanup\n\t\treturn \"\", errors.Wrap(err, \"failed to create $default api gateway stage\")\n\t}\n\n\treturn *createAPIResponse.ApiId, nil\n}\n\n// GetVPCLinkByTag Gets a VPC Link by tag (returns nil if there are no matches)\nfunc (c *Client) GetVPCLinkByTag(tagName string, tagValue string) (*apigatewayv2.VpcLink, error) {\n\tvar nextToken *string\n\n\tfor {\n\t\tvpcLinks, err := c.APIGatewayV2().GetVpcLinks(&apigatewayv2.GetVpcLinksInput{\n\t\t\tNextToken: nextToken,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"failed to get vpc links\")\n\t\t}\n\n\t\tfor _, vpcLink := range vpcLinks.Items {\n\t\t\tfor tag, value := range vpcLink.Tags {\n\t\t\t\tif tag == tagName && *value == tagValue {\n\t\t\t\t\treturn vpcLink, nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tnextToken = vpcLinks.NextToken\n\t\tif nextToken == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\n// GetAPIGatewayByTag Gets an API Gateway by tag (returns nil if there are no matches)\nfunc (c *Client) GetAPIGatewayByTag(tagName string, tagValue string) (*apigatewayv2.Api, error) {\n\tvar nextToken *string\n\n\tfor {\n\t\tapis, err := c.APIGatewayV2().GetApis(&apigatewayv2.GetApisInput{\n\t\t\tNextToken: nextToken,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"failed to get api gateways\")\n\t\t}\n\n\t\tfor _, api := range apis.Items {\n\t\t\tfor tag, value := range api.Tags {\n\t\t\t\tif tag == tagName && *value == tagValue {\n\t\t\t\t\treturn api, nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tnextToken = apis.NextToken\n\t\tif nextToken == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\n// DeleteVPCLinkByTag Deletes a VPC Link by tag (returns the deleted VPC Link, or nil if it was not found)\nfunc (c *Client) DeleteVPCLinkByTag(tagName string, tagValue string) (*apigatewayv2.VpcLink, error) {\n\tvpcLink, err := c.GetVPCLinkByTag(tagName, tagValue)\n\tif err != nil {\n\t\treturn nil, err\n\t} else if vpcLink == nil {\n\t\treturn nil, nil\n\t}\n\n\t_, err = c.APIGatewayV2().DeleteVpcLink(&apigatewayv2.DeleteVpcLinkInput{\n\t\tVpcLinkId: vpcLink.VpcLinkId,\n\t})\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"failed to delete vpc link \"+*vpcLink.VpcLinkId)\n\t}\n\n\treturn vpcLink, nil\n}\n\n// DeleteAPIGatewayByTag Deletes an API Gateway by tag (returns the deleted API Gateway, or nil if it was not found)\nfunc (c *Client) DeleteAPIGatewayByTag(tagName string, tagValue string) (*apigatewayv2.Api, error) {\n\tapiGateway, err := c.GetAPIGatewayByTag(tagName, tagValue)\n\tif err != nil {\n\t\treturn nil, err\n\t} else if apiGateway == nil {\n\t\treturn nil, nil\n\t}\n\n\terr = c.DeleteAPIGateway(*apiGateway.ApiId)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn apiGateway, nil\n}\n\n// DeleteAPIGateway Deletes an API Gateway by ID (returns an error if the API Gateway does not exist)\nfunc (c *Client) DeleteAPIGateway(apiGatewayID string) error {\n\t// Delete mappings in case user added a custom domain name (otherwise this will block API Gateway deletion)\n\terr := c.DeleteAPIGatewayMappings(apiGatewayID)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = c.APIGatewayV2().DeleteApi(&apigatewayv2.DeleteApiInput{\n\t\tApiId: aws.String(apiGatewayID),\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to delete api gateway \"+apiGatewayID)\n\t}\n\n\treturn nil\n}\n\n// DeleteAPIGatewayMappingsForDomainName deletes all API mappings that point to the provided api gateway from the provided domain name\nfunc (c *Client) DeleteAPIGatewayMappingsForDomainName(apiGatewayID string, domainName string) error {\n\tvar nextToken *string\n\n\tfor {\n\t\tapiMappings, err := c.APIGatewayV2().GetApiMappings(&apigatewayv2.GetApiMappingsInput{\n\t\t\tDomainName: aws.String(domainName),\n\t\t\tNextToken:  nextToken,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to get api mappings\")\n\t\t}\n\n\t\tfor _, apiMapping := range apiMappings.Items {\n\t\t\tif *apiMapping.ApiId != apiGatewayID {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t_, err := c.APIGatewayV2().DeleteApiMapping(&apigatewayv2.DeleteApiMappingInput{\n\t\t\t\tDomainName:   aws.String(domainName),\n\t\t\t\tApiMappingId: apiMapping.ApiMappingId,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, fmt.Sprintf(\"failed to delete api mapping %s in domain %s\", *apiMapping.ApiMappingId, domainName))\n\t\t\t}\n\t\t}\n\n\t\tnextToken = apiMappings.NextToken\n\t\tif nextToken == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// DeleteAPIGatewayMappings deletes all API mappings that point to the provided api gateway\nfunc (c *Client) DeleteAPIGatewayMappings(apiGatewayID string) error {\n\tvar nextToken *string\n\n\tfor {\n\t\tdomainNames, err := c.APIGatewayV2().GetDomainNames(&apigatewayv2.GetDomainNamesInput{\n\t\t\tNextToken: nextToken,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to get domain names\")\n\t\t}\n\n\t\tfor _, domainName := range domainNames.Items {\n\t\t\terr := c.DeleteAPIGatewayMappingsForDomainName(apiGatewayID, *domainName.DomainName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tnextToken = domainNames.NextToken\n\t\tif nextToken == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// GetVPCLinkIntegration gets the VPC Link integration in an API Gateway, or nil if unable to find it\nfunc (c *Client) GetVPCLinkIntegration(apiGatewayID string, vpcLinkID string) (*apigatewayv2.Integration, error) {\n\tvar nextToken *string\n\n\tfor {\n\t\tintegrations, err := c.APIGatewayV2().GetIntegrations(&apigatewayv2.GetIntegrationsInput{\n\t\t\tApiId:     &apiGatewayID,\n\t\t\tNextToken: nextToken,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"failed to get api gateway integrations for api gateway \"+apiGatewayID)\n\t\t}\n\n\t\t// find integration which is connected to the VPC link\n\t\tfor _, integration := range integrations.Items {\n\t\t\tif *integration.ConnectionId == vpcLinkID {\n\t\t\t\treturn integration, nil\n\t\t\t}\n\t\t}\n\n\t\tnextToken = integrations.NextToken\n\t\tif nextToken == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\n// GetRouteIntegrationID returns the integration which is attached to a endpoint route, or empty string if unable to find it\nfunc (c *Client) GetRouteIntegrationID(apiGatewayID string, endpoint string) (string, error) {\n\troute, err := c.GetRoute(apiGatewayID, endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif route == nil {\n\t\treturn \"\", nil\n\t}\n\n\treturn ExtractRouteIntegrationID(route), nil\n}\n\n// ExtractRouteIntegrationID extracts the integration ID which is attached to a route, or \"\" if no route is attached\nfunc ExtractRouteIntegrationID(route *apigatewayv2.Route) string {\n\tif route == nil || route.Target == nil {\n\t\treturn \"\"\n\t}\n\n\t// trim of prefix of integrationID.\n\t// Note: Integrations get attached to routes via a target of the format integrations/<integrationID>\n\tintegrationID := strings.TrimPrefix(*route.Target, \"integrations/\")\n\treturn integrationID\n}\n\n// GetRoute retrieves the route matching an endpoint, or nil if unable to find it\nfunc (c *Client) GetRoute(apiGatewayID string, endpoint string) (*apigatewayv2.Route, error) {\n\tvar nextToken *string\n\n\tfor {\n\t\troutes, err := c.APIGatewayV2().GetRoutes(&apigatewayv2.GetRoutesInput{\n\t\t\tApiId:     &apiGatewayID,\n\t\t\tNextToken: nextToken,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"failed to get api gateway routes for api gateway \"+apiGatewayID)\n\t\t}\n\n\t\t// find route which matches the endpoint\n\t\tfor _, route := range routes.Items {\n\t\t\tif *route.RouteKey == \"ANY \"+endpoint {\n\t\t\t\treturn route, nil\n\t\t\t}\n\t\t}\n\n\t\tnextToken = routes.NextToken\n\t\tif nextToken == nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\n// CreateRoute creates a new route and attaches the route to the integration\nfunc (c *Client) CreateRoute(apiGatewayID string, integrationID string, endpoint string) error {\n\t_, err := c.APIGatewayV2().CreateRoute(&apigatewayv2.CreateRouteInput{\n\t\tApiId:    &apiGatewayID,\n\t\tRouteKey: aws.String(\"ANY \" + endpoint),\n\t\tTarget:   aws.String(\"integrations/\" + integrationID),\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, fmt.Sprintf(\"failed to create %s route for api gateway %s with integration %s\", endpoint, apiGatewayID, integrationID))\n\t}\n\treturn nil\n}\n\n// CreateHTTPIntegration creates new HTTP integration for API Gateway, returns integration ID\nfunc (c *Client) CreateHTTPIntegration(apiGatewayID string, targetEndpoint string) (string, error) {\n\tintegrationResponse, err := c.APIGatewayV2().CreateIntegration(&apigatewayv2.CreateIntegrationInput{\n\t\tApiId:                &apiGatewayID,\n\t\tIntegrationType:      aws.String(\"HTTP_PROXY\"),\n\t\tIntegrationUri:       &targetEndpoint,\n\t\tPayloadFormatVersion: aws.String(\"1.0\"),\n\t\tIntegrationMethod:    aws.String(\"ANY\"),\n\t})\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, fmt.Sprintf(\"failed to create api gateway integration for endpoint %s in api gateway %s\", targetEndpoint, apiGatewayID))\n\t}\n\treturn *integrationResponse.IntegrationId, nil\n}\n\n// DeleteIntegration deletes an integration from API Gateway\nfunc (c *Client) DeleteIntegration(apiGatewayID string, integrationID string) error {\n\t_, err := c.APIGatewayV2().DeleteIntegration(&apigatewayv2.DeleteIntegrationInput{\n\t\tApiId:         &apiGatewayID,\n\t\tIntegrationId: &integrationID,\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, fmt.Sprintf(\"failed to delete api gateway integration %s in api gateway %s\", integrationID, apiGatewayID))\n\t}\n\treturn nil\n}\n\n// DeleteRoute deletes a route from API Gateway, and returns the deleted route (or nil if it wasn't found)\nfunc (c *Client) DeleteRoute(apiGatewayID string, endpoint string) (*apigatewayv2.Route, error) {\n\troute, err := c.GetRoute(apiGatewayID, endpoint)\n\tif err != nil {\n\t\treturn nil, err\n\t} else if route == nil {\n\t\treturn nil, nil\n\t}\n\n\t_, err = c.APIGatewayV2().DeleteRoute(&apigatewayv2.DeleteRouteInput{\n\t\tApiId:   &apiGatewayID,\n\t\tRouteId: route.RouteId,\n\t})\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"failed to delete api gateway route %s with endpoint %s in api gateway %s\", *route.RouteId, endpoint, apiGatewayID))\n\t}\n\n\treturn route, nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/autoscaling.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/autoscaling\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\n// if specified, all tags must be present\nfunc (c *Client) AutoscalingGroups(tags map[string]string) ([]*autoscaling.Group, error) {\n\tvar asgs []*autoscaling.Group\n\n\tparams := autoscaling.DescribeAutoScalingGroupsInput{\n\t\tAutoScalingGroupNames: nil,\n\t}\n\terr := c.Autoscaling().DescribeAutoScalingGroupsPages(&params,\n\t\tfunc(page *autoscaling.DescribeAutoScalingGroupsOutput, lastPage bool) bool {\n\t\t\tfor _, asg := range page.AutoScalingGroups {\n\t\t\t\tasgTags := make(map[string]string, len(asg.Tags))\n\t\t\t\tfor _, asgTag := range asg.Tags {\n\t\t\t\t\tif asgTag.Key != nil && asgTag.Value != nil {\n\t\t\t\t\t\tasgTags[*asgTag.Key] = *asgTag.Value\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tmissingTag := false\n\t\t\t\tfor key, value := range tags {\n\t\t\t\t\tif asgTags[key] != value {\n\t\t\t\t\t\tmissingTag = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif missingTag {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tasgs = append(asgs, asg)\n\t\t\t}\n\n\t\t\treturn true\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn asgs, nil\n}\n\n// Returns the most recent activity for the ASG, or nil if there are no activities\nfunc (c *Client) MostRecentASGActivity(asgName string) (*autoscaling.Activity, error) {\n\tresp, err := c.Autoscaling().DescribeScalingActivities(&autoscaling.DescribeScalingActivitiesInput{\n\t\tAutoScalingGroupName: aws.String(asgName),\n\t\tMaxRecords:           aws.Int64(1),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif len(resp.Activities) == 0 {\n\t\treturn nil, nil\n\t}\n\n\treturn resp.Activities[0], nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/aws.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n//go:generate python3 gen_resource_metadata.py\n//go:generate gofmt -s -w resource_metadata.go\n\npackage aws\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/credentials\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype Client struct {\n\tRegion          string\n\tsess            *session.Session\n\tIsAnonymous     bool\n\tclients         clients\n\taccountID       *string\n\thashedAccountID *string\n}\n\nfunc NewForSession(sess *session.Session) (*Client, error) {\n\tif sess.Config.Region == nil {\n\t\treturn nil, errors.ErrorUnexpected(\"session config is missing the Region field\")\n\t}\n\n\treturn &Client{\n\t\tRegion: *sess.Config.Region,\n\t\tsess:   sess,\n\t}, nil\n}\n\nfunc NewFromClientS3Path(s3Path string, awsClient *Client) (*Client, error) {\n\tif !awsClient.IsAnonymous {\n\t\treturn NewFromS3Path(s3Path)\n\t}\n\n\tregion, err := GetBucketRegionFromS3Path(s3Path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn NewAnonymousClientWithRegion(region)\n}\n\nfunc NewFromS3Path(s3Path string) (*Client, error) {\n\tbucket, _, err := SplitS3Path(s3Path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn NewFromS3Bucket(bucket)\n}\n\nfunc NewFromS3Bucket(bucket string) (*Client, error) {\n\tregion, err := GetBucketRegion(bucket)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn NewForRegion(region)\n}\n\nfunc NewForRegion(region string) (*Client, error) {\n\tsess, err := session.NewSessionWithOptions(session.Options{\n\t\tConfig: aws.Config{\n\t\t\tRegion: aws.String(region),\n\t\t},\n\t\tSharedConfigState: session.SharedConfigEnable,\n\t})\n\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif sess.Config.Credentials == nil {\n\t\treturn nil, ErrorUnableToFindCredentials()\n\t}\n\n\tcreds, err := sess.Config.Credentials.Get()\n\tif err != nil {\n\t\treturn nil, ErrorUnableToFindCredentials()\n\t}\n\n\tif creds.AccessKeyID == \"\" || creds.SecretAccessKey == \"\" {\n\t\treturn nil, ErrorUnexpectedMissingCredentials(creds.AccessKeyID, creds.SecretAccessKey)\n\t}\n\n\treturn &Client{\n\t\tsess:   sess,\n\t\tRegion: region,\n\t}, nil\n}\n\nfunc New() (*Client, error) {\n\tsess := session.Must(session.NewSessionWithOptions(session.Options{\n\t\tSharedConfigState: session.SharedConfigEnable,\n\t}))\n\n\tif sess.Config.Region == nil {\n\t\treturn nil, ErrorRegionNotConfigured()\n\t}\n\n\tif sess.Config.Credentials == nil {\n\t\treturn nil, ErrorUnableToFindCredentials()\n\t}\n\n\tcreds, err := sess.Config.Credentials.Get()\n\tif err != nil {\n\t\treturn nil, ErrorUnableToFindCredentials()\n\t}\n\n\t// make sure that credential exists\n\tif creds.AccessKeyID == \"\" || creds.SecretAccessKey == \"\" {\n\t\treturn nil, ErrorUnexpectedMissingCredentials(creds.AccessKeyID, creds.SecretAccessKey)\n\t}\n\n\treturn &Client{\n\t\tsess:   sess,\n\t\tRegion: *sess.Config.Region,\n\t}, nil\n}\n\nfunc NewAnonymousClientWithRegion(region string) (*Client, error) {\n\tsess, err := session.NewSession(&aws.Config{\n\t\tCredentials: credentials.AnonymousCredentials,\n\t\tRegion:      aws.String(region),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Client{\n\t\tsess:        sess,\n\t\tRegion:      region,\n\t\tIsAnonymous: true,\n\t}, nil\n}\n\nfunc (c Client) Session() *session.Session {\n\treturn c.sess\n}\n"
  },
  {
    "path": "pkg/lib/aws/clients.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"github.com/aws/aws-sdk-go/service/acm\"\n\t\"github.com/aws/aws-sdk-go/service/apigatewayv2\"\n\t\"github.com/aws/aws-sdk-go/service/autoscaling\"\n\t\"github.com/aws/aws-sdk-go/service/cloudformation\"\n\t\"github.com/aws/aws-sdk-go/service/cloudwatch\"\n\t\"github.com/aws/aws-sdk-go/service/cloudwatchlogs\"\n\t\"github.com/aws/aws-sdk-go/service/ec2\"\n\t\"github.com/aws/aws-sdk-go/service/ecr\"\n\t\"github.com/aws/aws-sdk-go/service/eks\"\n\t\"github.com/aws/aws-sdk-go/service/elb\"\n\t\"github.com/aws/aws-sdk-go/service/elbv2\"\n\t\"github.com/aws/aws-sdk-go/service/iam\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/aws/aws-sdk-go/service/s3/s3manager\"\n\t\"github.com/aws/aws-sdk-go/service/servicequotas\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/aws/aws-sdk-go/service/sts\"\n)\n\ntype clients struct {\n\ts3             *s3.S3\n\ts3Uploader     *s3manager.Uploader\n\ts3Downloader   *s3manager.Downloader\n\tsts            *sts.STS\n\tsqs            *sqs.SQS\n\tec2            *ec2.EC2\n\telb            *elb.ELB\n\telbv2          *elbv2.ELBV2\n\teks            *eks.EKS\n\tecr            *ecr.ECR\n\tacm            *acm.ACM\n\tautoscaling    *autoscaling.AutoScaling\n\tcloudWatchLogs *cloudwatchlogs.CloudWatchLogs\n\tcloudWatch     *cloudwatch.CloudWatch\n\tapiGatewayV2   *apigatewayv2.ApiGatewayV2\n\tserviceQuotas  *servicequotas.ServiceQuotas\n\tcloudFormation *cloudformation.CloudFormation\n\tiam            *iam.IAM\n}\n\nfunc (c *Client) S3() *s3.S3 {\n\tif c.clients.s3 == nil {\n\t\tc.clients.s3 = s3.New(c.sess)\n\t}\n\treturn c.clients.s3\n}\n\nfunc (c *Client) S3Uploader() *s3manager.Uploader {\n\tif c.clients.s3Uploader == nil {\n\t\tc.clients.s3Uploader = s3manager.NewUploader(c.sess)\n\t}\n\treturn c.clients.s3Uploader\n}\n\nfunc (c *Client) S3Downloader() *s3manager.Downloader {\n\tif c.clients.s3Downloader == nil {\n\t\tc.clients.s3Downloader = s3manager.NewDownloader(c.sess)\n\t}\n\treturn c.clients.s3Downloader\n}\n\nfunc (c *Client) STS() *sts.STS {\n\tif c.clients.sts == nil {\n\t\tc.clients.sts = sts.New(c.sess)\n\t}\n\treturn c.clients.sts\n}\n\nfunc (c *Client) SQS() *sqs.SQS {\n\tif c.clients.sqs == nil {\n\t\tc.clients.sqs = sqs.New(c.sess)\n\t}\n\treturn c.clients.sqs\n}\n\nfunc (c *Client) EC2() *ec2.EC2 {\n\tif c.clients.ec2 == nil {\n\t\tc.clients.ec2 = ec2.New(c.sess)\n\t}\n\treturn c.clients.ec2\n}\n\nfunc (c *Client) ELB() *elb.ELB {\n\tif c.clients.elb == nil {\n\t\tc.clients.elb = elb.New(c.sess)\n\t}\n\treturn c.clients.elb\n}\n\nfunc (c *Client) ELBV2() *elbv2.ELBV2 {\n\tif c.clients.elbv2 == nil {\n\t\tc.clients.elbv2 = elbv2.New(c.sess)\n\t}\n\treturn c.clients.elbv2\n}\n\nfunc (c *Client) EKS() *eks.EKS {\n\tif c.clients.eks == nil {\n\t\tc.clients.eks = eks.New(c.sess)\n\t}\n\treturn c.clients.eks\n}\n\nfunc (c *Client) ECR() *ecr.ECR {\n\tif c.clients.ecr == nil {\n\t\tc.clients.ecr = ecr.New(c.sess)\n\t}\n\treturn c.clients.ecr\n}\n\nfunc (c *Client) CloudFormation() *cloudformation.CloudFormation {\n\tif c.clients.cloudFormation == nil {\n\t\tc.clients.cloudFormation = cloudformation.New(c.sess)\n\t}\n\treturn c.clients.cloudFormation\n}\n\nfunc (c *Client) Autoscaling() *autoscaling.AutoScaling {\n\tif c.clients.autoscaling == nil {\n\t\tc.clients.autoscaling = autoscaling.New(c.sess)\n\t}\n\treturn c.clients.autoscaling\n}\n\nfunc (c *Client) ACM() *acm.ACM {\n\tif c.clients.acm == nil {\n\t\tc.clients.acm = acm.New(c.sess)\n\t}\n\treturn c.clients.acm\n}\n\nfunc (c *Client) CloudWatchLogs() *cloudwatchlogs.CloudWatchLogs {\n\tif c.clients.cloudWatchLogs == nil {\n\t\tc.clients.cloudWatchLogs = cloudwatchlogs.New(c.sess)\n\t}\n\treturn c.clients.cloudWatchLogs\n}\n\nfunc (c *Client) CloudWatch() *cloudwatch.CloudWatch {\n\tif c.clients.cloudWatch == nil {\n\t\tc.clients.cloudWatch = cloudwatch.New(c.sess)\n\t}\n\treturn c.clients.cloudWatch\n}\n\nfunc (c *Client) APIGatewayV2() *apigatewayv2.ApiGatewayV2 {\n\tif c.clients.apiGatewayV2 == nil {\n\t\tc.clients.apiGatewayV2 = apigatewayv2.New(c.sess)\n\t}\n\treturn c.clients.apiGatewayV2\n}\n\nfunc (c *Client) ServiceQuotas() *servicequotas.ServiceQuotas {\n\tif c.clients.serviceQuotas == nil {\n\t\tc.clients.serviceQuotas = servicequotas.New(c.sess)\n\t}\n\treturn c.clients.serviceQuotas\n}\n\nfunc (c *Client) IAM() *iam.IAM {\n\tif c.clients.iam == nil {\n\t\tc.clients.iam = iam.New(c.sess)\n\t}\n\treturn c.clients.iam\n}\n"
  },
  {
    "path": "pkg/lib/aws/cloudformation.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"github.com/aws/aws-sdk-go/service/cloudformation\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\nfunc (c *Client) ListEKSStacks(controlPlaneStackName string, nodeGroupStackNamePrefixes strset.Set) ([]*cloudformation.StackSummary, error) {\n\tmostRecentStackRecordsByName := map[string]*cloudformation.StackSummary{}\n\tstackSet := strset.Union(nodeGroupStackNamePrefixes, strset.New(controlPlaneStackName))\n\n\terr := c.CloudFormation().ListStacksPages(\n\t\t&cloudformation.ListStacksInput{},\n\t\tfunc(listStackOutput *cloudformation.ListStacksOutput, lastPage bool) bool {\n\t\t\tfor _, stackSummary := range listStackOutput.StackSummaries {\n\t\t\t\tif stackSummary == nil || stackSummary.StackName == nil || !stackSet.HasWithPrefix(*stackSummary.StackName) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif _, ok := mostRecentStackRecordsByName[*stackSummary.StackName]; !ok {\n\t\t\t\t\tmostRecentStackRecordsByName[*stackSummary.StackName] = stackSummary\n\t\t\t\t} else {\n\t\t\t\t\tcreated := mostRecentStackRecordsByName[*stackSummary.StackName].CreationTime\n\t\t\t\t\tif created != nil && stackSummary.CreationTime != nil && stackSummary.CreationTime.After(*created) {\n\t\t\t\t\t\tmostRecentStackRecordsByName[*stackSummary.StackName] = stackSummary\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif *stackSummary.StackName == controlPlaneStackName {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn true\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn getStackSummariesFromMap(mostRecentStackRecordsByName), nil\n}\n\nfunc getStackSummariesFromMap(stackSummaries map[string]*cloudformation.StackSummary) []*cloudformation.StackSummary {\n\tvar stackSummariesSlice []*cloudformation.StackSummary\n\tfor _, stack := range stackSummaries {\n\t\tstackSummariesSlice = append(stackSummariesSlice, stack)\n\t}\n\treturn stackSummariesSlice\n}\n"
  },
  {
    "path": "pkg/lib/aws/cloudwatch.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/cloudwatch\"\n\t\"github.com/aws/aws-sdk-go/service/cloudwatchlogs\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nvar (\n\t_dashboardMinWidthUnits  = 1\n\t_dashboardMaxWidthUnits  = 24\n\t_dashboardMinHeightUnits = 1\n\t_dashboardMaxHeightUnits = 1000\n)\n\ntype CloudWatchDashboard struct {\n\tStart          string             `json:\"start\"`\n\tPeriodOverride string             `json:\"periodOverride\"`\n\tWidgets        []CloudWatchWidget `json:\"widgets\"`\n}\n\n// Example:\n// CloudWatchWidget{\n// \t\"type\":\"metric\",\n// \t\"x\":0,\n// \t\"y\":0,\n// \t\"width\":12,\n// \t\"height\":6,\n// \t\"properties\":{\n// \t   \"metrics\":[\n// \t\t  [\n// \t\t\t \"AWS/EC2\",\n// \t\t\t \"CPUUtilization\",\n// \t\t\t \"InstanceId\",\n// \t\t\t \"i-012345\"\n// \t\t  ]\n// \t   ],\n// \t   \"period\":300,\n// \t   \"stat\":\"Average\",\n// \t   \"region\":\"us-east-1\",\n// \t   \"title\":\"EC2 Instance CPU\"\n// \t}\n//  }\ntype CloudWatchWidget struct {\n\tType       string                 `json:\"type\"`\n\tX          int                    `json:\"x\"`\n\tY          int                    `json:\"y\"`\n\tWidth      int                    `json:\"width\"`\n\tHeight     int                    `json:\"height\"`\n\tProperties map[string]interface{} `json:\"properties\"`\n}\n\ntype CloudWatchWidgetGrid struct {\n\tXOrigin      int                `json:\"x_origin\"`\n\tYOrigin      int                `json:\"y_origin\"`\n\tNumColumns   int                `json:\"num_columns\"`\n\tNumRows      int                `json:\"num_rows\"`\n\tWidgetHeight int                `json:\"widget_height\"`\n\tWidgetWidth  int                `json:\"widget_width\"`\n\tWidgets      []CloudWatchWidget `json:\"widgets\"`\n}\n\nfunc (c *Client) DoesLogGroupExist(logGroup string) (bool, error) {\n\t_, err := c.CloudWatchLogs().ListTagsLogGroup(&cloudwatchlogs.ListTagsLogGroupInput{\n\t\tLogGroupName: aws.String(logGroup),\n\t})\n\tif err != nil {\n\t\tif IsErrCode(err, \"ResourceNotFoundException\") {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.Wrap(err, \"log group \"+logGroup)\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) CreateLogGroup(logGroup string, tags map[string]string) error {\n\t_, err := c.CloudWatchLogs().CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{\n\t\tLogGroupName: aws.String(logGroup),\n\t\tTags:         aws.StringMap(tags),\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"creating log group \"+logGroup)\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) DeleteLogGroup(logGroup string) error {\n\t_, err := c.CloudWatchLogs().DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{\n\t\tLogGroupName: aws.String(logGroup),\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"log group \"+logGroup)\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) TagLogGroup(logGroup string, tagMap map[string]string) error {\n\ttags := map[string]*string{}\n\tfor key, value := range tagMap {\n\t\ttags[key] = aws.String(value)\n\t}\n\n\t_, err := c.CloudWatchLogs().TagLogGroup(&cloudwatchlogs.TagLogGroupInput{\n\t\tLogGroupName: aws.String(logGroup),\n\t\tTags:         tags,\n\t})\n\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to add tags to log group\", logGroup)\n\t}\n\n\treturn nil\n}\n\n// NewDashboard creates a new dashboard object with title\nfunc (c *Client) NewDashboard(title string) *CloudWatchDashboard {\n\treturn &CloudWatchDashboard{\n\t\tStart:          \"-PT1H\",\n\t\tPeriodOverride: \"inherit\",\n\t\tWidgets: []CloudWatchWidget{\n\t\t\tTextWidget(7, 0, 10, 1, title),\n\t\t},\n\t}\n}\n\n// GetDashboard gets a dashboard from cloudwatch\nfunc (c *Client) GetDashboard(dashboardName string) (*CloudWatchDashboard, error) {\n\tdashboardOutput, err := c.CloudWatch().GetDashboard(&cloudwatch.GetDashboardInput{\n\t\tDashboardName: aws.String(dashboardName),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"failed to get dashboard\", dashboardName)\n\t}\n\n\tdashboardString := *dashboardOutput.DashboardBody\n\n\tvar dashboard CloudWatchDashboard\n\terr = json.Unmarshal([]byte(dashboardString), &dashboard)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"failed to decode cloudwatch body json\")\n\t}\n\n\treturn &dashboard, nil\n}\n\n// GetDashboardOrEmpty gets a dashboard if it exists, or initializes an empty one if not\nfunc (c *Client) GetDashboardOrEmpty(dashboardName string, title string) (*CloudWatchDashboard, error) {\n\tdashboard, err := c.GetDashboard(dashboardName)\n\tif err != nil {\n\t\tif IsErrCode(err, \"ResourceNotFound\") {\n\t\t\tdashboard = c.NewDashboard(title)\n\t\t} else {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn dashboard, nil\n}\n\n// CreateDashboard creates a new dashboard (or clears an existing one if it already exists)\nfunc (c *Client) CreateDashboard(dashboardName string, title string) error {\n\tdashboard := c.NewDashboard(title)\n\n\terr := c.PutDashboard(dashboard, dashboardName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// PutDashboard updates a dashboard (or creates it if it doesn't exit)\nfunc (c *Client) PutDashboard(dashboard *CloudWatchDashboard, dashboardName string) error {\n\tdashboardJSON, err := json.Marshal(dashboard)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to encode cloudwatch body into json\")\n\t}\n\n\t_, err = c.CloudWatch().PutDashboard(&cloudwatch.PutDashboardInput{\n\t\tDashboardName: aws.String(dashboardName),\n\t\tDashboardBody: aws.String(string(dashboardJSON)),\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to put dashboard\", dashboardName)\n\t}\n\n\treturn nil\n}\n\n// DeleteDashboard deletes a dashboard\nfunc (c *Client) DeleteDashboard(dashboardName string) error {\n\t_, err := c.CloudWatch().DeleteDashboards(&cloudwatch.DeleteDashboardsInput{\n\t\tDashboardNames: []*string{aws.String(dashboardName)},\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to delete dashboard\", dashboardName)\n\t}\n\n\treturn nil\n}\n\n// DoesDashboardExist checks if a dashboard exists\nfunc (c *Client) DoesDashboardExist(dashboardName string) (bool, error) {\n\t_, err := c.CloudWatch().GetDashboard(&cloudwatch.GetDashboardInput{\n\t\tDashboardName: aws.String(dashboardName),\n\t})\n\tif err != nil {\n\t\tif IsErrCode(err, \"ResourceNotFound\") {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.Wrap(err, \"dashboard\", dashboardName)\n\t}\n\n\treturn true, nil\n}\n\n// TextWidget creates new text widget\n// Example:\n// title_widget = {\n//     \"type\": \"text\",\n//     \"x\": x,\n//     \"y\": y,\n//     \"width\": wewidthi,\n//     \"height\": height,\n//     \"properties\": {\"markdown\": markdown},\n// }\nfunc TextWidget(x int, y int, width int, height int, markdown string) CloudWatchWidget {\n\treturn CloudWatchWidget{Type: \"text\", X: x, Y: y, Width: width, Height: height, Properties: map[string]interface{}{\"markdown\": markdown}}\n}\n\n// NewHorizontalGrid sets a CloudWatch Dashboard grid to be filled from left to right, row by row\nfunc NewHorizontalGrid(xOrigin, yOrigin, widgetHeight, widgetWidth, numColumns int) (*CloudWatchWidgetGrid, error) {\n\tif widgetHeight < 1 || widgetHeight > _dashboardMaxHeightUnits {\n\t\treturn &CloudWatchWidgetGrid{}, ErrorDashboardHeightOutOfRange(widgetHeight)\n\t}\n\tif widgetWidth < 1 || widgetWidth > _dashboardMaxWidthUnits {\n\t\treturn &CloudWatchWidgetGrid{}, ErrorDashboardWidthOutOfRange(widgetWidth)\n\t}\n\tif xOrigin+numColumns*widgetWidth > _dashboardMaxWidthUnits {\n\t\treturn &CloudWatchWidgetGrid{}, ErrorDashboardWidthOutOfRange(xOrigin + numColumns*widgetWidth)\n\t}\n\treturn &CloudWatchWidgetGrid{\n\t\tXOrigin:      xOrigin,\n\t\tYOrigin:      yOrigin,\n\t\tWidgetHeight: widgetHeight,\n\t\tWidgetWidth:  widgetWidth,\n\t\tNumColumns:   numColumns,\n\t\tWidgets:      make([]CloudWatchWidget, 0),\n\t}, nil\n}\n\n// NewVerticalGrid sets a CloudWatch Dashboard grid to be filled from top to bottom, column by column\nfunc NewVerticalGrid(xOrigin, yOrigin, widgetHeight, widgetWidth, numRows int) (*CloudWatchWidgetGrid, error) {\n\tif widgetHeight < 1 || widgetHeight > _dashboardMaxHeightUnits {\n\t\treturn &CloudWatchWidgetGrid{}, ErrorDashboardHeightOutOfRange(widgetHeight)\n\t}\n\tif widgetWidth < 1 || widgetWidth > _dashboardMaxWidthUnits {\n\t\treturn &CloudWatchWidgetGrid{}, ErrorDashboardWidthOutOfRange(widgetWidth)\n\t}\n\tif yOrigin+numRows*widgetHeight > _dashboardMaxHeightUnits {\n\t\treturn &CloudWatchWidgetGrid{}, ErrorDashboardHeightOutOfRange(yOrigin + numRows*widgetHeight)\n\t}\n\treturn &CloudWatchWidgetGrid{\n\t\tXOrigin:      xOrigin,\n\t\tYOrigin:      yOrigin,\n\t\tWidgetHeight: widgetHeight,\n\t\tWidgetWidth:  widgetWidth,\n\t\tNumRows:      numRows,\n\t\tWidgets:      make([]CloudWatchWidget, 0),\n\t}, nil\n}\n\n// AddWidget adds a widget to the configured grid\nfunc (grid *CloudWatchWidgetGrid) AddWidget(\n\tmetric []interface{},\n\ttitle string,\n\tstat string,\n\tperiod int,\n\tregion string,\n) error {\n\tvar currentColumn, currentRow int\n\tif grid.NumColumns > 0 {\n\t\tcurrentRow = len(grid.Widgets) / grid.NumColumns\n\t\tcurrentColumn = len(grid.Widgets) - currentRow*grid.NumColumns\n\t}\n\tif grid.NumRows > 0 {\n\t\tcurrentColumn = len(grid.Widgets) / grid.NumRows\n\t\tcurrentRow = len(grid.Widgets) - currentColumn*grid.NumRows\n\t}\n\tx := grid.XOrigin + currentColumn*grid.WidgetWidth\n\ty := grid.YOrigin + currentRow*grid.WidgetHeight\n\n\tif x+grid.WidgetWidth > _dashboardMaxWidthUnits {\n\t\treturn ErrorDashboardWidthOutOfRange(x + grid.WidgetWidth)\n\t}\n\tif y+grid.WidgetHeight > _dashboardMaxHeightUnits {\n\t\treturn ErrorDashboardHeightOutOfRange(y + grid.WidgetHeight)\n\t}\n\n\tgrid.Widgets = append(grid.Widgets,\n\t\tCloudWatchWidget{\n\t\t\tType:   \"metric\",\n\t\t\tX:      x,\n\t\t\tY:      y,\n\t\t\tWidth:  grid.WidgetWidth,\n\t\t\tHeight: grid.WidgetHeight,\n\t\t\tProperties: map[string]interface{}{\n\t\t\t\t\"metrics\": metric,\n\t\t\t\t\"title\":   title,\n\t\t\t\t\"stat\":    stat,\n\t\t\t\t\"period\":  period,\n\t\t\t\t\"region\":  region,\n\t\t\t\t\"view\":    \"timeSeries\",\n\t\t\t},\n\t\t},\n\t)\n\n\treturn nil\n}\n\n// HighestY returns the largest Y coordinate of a widget on the dashboard (i.e. the lowest widget)\nfunc HighestY(dashboard *CloudWatchDashboard) (int, error) {\n\thighestY := 0\n\n\tfor _, wid := range dashboard.Widgets {\n\t\tif highestY < wid.Y {\n\t\t\thighestY = wid.Y\n\t\t}\n\t}\n\n\treturn highestY, nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/credentials.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\n// access key ID may be unavailable depending on how the client was instantiated\nfunc (c *Client) AccessKeyID() *string {\n\tif c.sess.Config.Credentials == nil {\n\t\treturn nil\n\t}\n\n\tsessCreds, err := c.sess.Config.Credentials.Get()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tif sessCreds.AccessKeyID == \"\" {\n\t\treturn nil\n\t}\n\n\treturn &sessCreds.AccessKeyID\n}\n\nfunc (c *Client) SecretAccessKey() *string {\n\tif c.sess.Config.Credentials == nil {\n\t\treturn nil\n\t}\n\n\tsessCreds, err := c.sess.Config.Credentials.Get()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tif sessCreds.SecretAccessKey == \"\" {\n\t\treturn nil\n\t}\n\n\treturn &sessCreds.SecretAccessKey\n}\n\nfunc (c *Client) SessionToken() *string {\n\tif c.sess.Config.Credentials == nil {\n\t\treturn nil\n\t}\n\n\tsessCreds, err := c.sess.Config.Credentials.Get()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tif sessCreds.SessionToken == \"\" {\n\t\treturn nil\n\t}\n\n\treturn &sessCreds.SessionToken\n}\n"
  },
  {
    "path": "pkg/lib/aws/ec2.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"math\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/ec2\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nvar (\n\t_digitsRegex         = regexp.MustCompile(`[0-9]+`)\n\t_gpuInstanceFamilies = strset.New(\"g\", \"p\")\n)\n\ntype ParsedInstanceType struct {\n\tFamily       string\n\tGeneration   int\n\tCapabilities strset.Set\n\tSize         string\n}\n\n// Checks weather the input is an AWS instance type\nfunc IsValidInstanceType(instanceType string) bool {\n\treturn AllInstanceTypes.Has(instanceType)\n}\n\n// Checks whether the input is an AWS instance type\nfunc CheckValidInstanceType(instanceType string) error {\n\tif !IsValidInstanceType(instanceType) {\n\t\treturn ErrorInvalidInstanceType(instanceType)\n\t}\n\treturn nil\n}\n\n// AWS instance types take the form of: [family][generation][capabilities].[size]\n// the first group is the instance family, e.g. \"m\", \"t\", \"g\", \"inf\", ...\n// the second group is a generation number for that series, e.g. 3, 4, ...\n// the third group is optional, and is a set of single-character capabilities\n//   \"g\" represents ARM (graviton), \"a\" for AMD, \"n\" for fast networking, \"d\" for fast storage, etc.\n// the fourth and final group (after the dot) is the instance size, e.g. \"large\"\nfunc ParseInstanceType(instanceType string) (ParsedInstanceType, error) {\n\tif err := CheckValidInstanceType(instanceType); err != nil {\n\t\treturn ParsedInstanceType{}, err\n\t}\n\n\tparts := strings.Split(instanceType, \".\")\n\tif len(parts) != 2 {\n\t\treturn ParsedInstanceType{}, errors.ErrorUnexpected(\"unexpected invalid instance type: \" + instanceType)\n\t}\n\n\tprefix := parts[0]\n\tsize := parts[1]\n\n\tdigitSets := _digitsRegex.FindAllString(prefix, -1)\n\tif len(digitSets) == 0 {\n\t\treturn ParsedInstanceType{}, errors.ErrorUnexpected(\"unexpected invalid instance type: \" + instanceType)\n\t}\n\n\tprefixParts := _digitsRegex.Split(prefix, -1)\n\tcapabilitiesStr := prefixParts[len(prefixParts)-1]\n\tcapabilities := strset.FromSlice(strings.Split(capabilitiesStr, \"\"))\n\n\tgenerationStr := digitSets[len(digitSets)-1]\n\tgeneration, ok := s.ParseInt(generationStr)\n\tif !ok {\n\t\treturn ParsedInstanceType{}, errors.ErrorUnexpected(\"unexpected invalid instance type: \" + instanceType)\n\t}\n\n\tgenerationIndex := strings.LastIndex(prefix, generationStr)\n\tif generationIndex == -1 {\n\t\treturn ParsedInstanceType{}, errors.ErrorUnexpected(\"unexpected invalid instance type: \" + instanceType)\n\t}\n\tfamily := prefix[:generationIndex]\n\n\treturn ParsedInstanceType{\n\t\tFamily:       family,\n\t\tGeneration:   generation,\n\t\tCapabilities: capabilities,\n\t\tSize:         size,\n\t}, nil\n}\n\nfunc IsARMInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif parsedType.Family == \"a\" {\n\t\treturn true, nil\n\t}\n\n\tif parsedType.Capabilities.Has(\"g\") {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc IsAMDGPUInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif !_gpuInstanceFamilies.Has(parsedType.Family) {\n\t\treturn false, nil\n\t}\n\n\tif parsedType.Capabilities.Has(\"a\") {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc IsNvidiaGPUInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif !_gpuInstanceFamilies.Has(parsedType.Family) {\n\t\treturn false, nil\n\t}\n\n\tif !parsedType.Capabilities.Has(\"a\") {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc IsGPUInstance(instanceType string) (bool, error) {\n\tisAMDGPU, err := IsAMDGPUInstance(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tisNvidiaGPU, err := IsNvidiaGPUInstance(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn isAMDGPU || isNvidiaGPU, nil\n}\n\nfunc IsInferentiaInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn parsedType.Family == \"inf\", nil\n}\n\nfunc IsTrainiumInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn parsedType.Family == \"trn\", nil\n}\n\nfunc IsMacInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif parsedType.Family == \"mac\" {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc IsFPGAInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif parsedType.Family == \"f\" {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc IsAlevoInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif parsedType.Family == \"vt\" {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc IsGaudiInstance(instanceType string) (bool, error) {\n\tparsedType, err := ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif parsedType.Family == \"dl\" {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc (c *Client) SpotInstancePrice(instanceType string) (float64, error) {\n\tresult, err := c.EC2().DescribeSpotPriceHistory(&ec2.DescribeSpotPriceHistoryInput{\n\t\tInstanceTypes:       []*string{aws.String(instanceType)},\n\t\tProductDescriptions: []*string{aws.String(\"Linux/UNIX\")},\n\t\tStartTime:           aws.Time(time.Now()),\n\t})\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, \"checking spot instance price\")\n\t}\n\n\tmin := math.MaxFloat64\n\n\tfor _, spotPrice := range result.SpotPriceHistory {\n\t\tif spotPrice == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tprice, ok := s.ParseFloat64(*spotPrice.SpotPrice)\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\n\t\tif price < min {\n\t\t\tmin = price\n\t\t}\n\t}\n\n\tif min == math.MaxFloat64 {\n\t\treturn 0, ErrorNoValidSpotPrices(instanceType, c.Region)\n\t}\n\n\tif min <= 0 {\n\t\treturn 0, ErrorNoValidSpotPrices(instanceType, c.Region)\n\t}\n\n\treturn min, nil\n}\n\nfunc (c *Client) ListAllRegions() (strset.Set, error) {\n\tresult, err := c.EC2().DescribeRegions(&ec2.DescribeRegionsInput{\n\t\tAllRegions: aws.Bool(true),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tregions := strset.New()\n\tfor _, region := range result.Regions {\n\t\tif region.RegionName != nil {\n\t\t\tregions.Add(*region.RegionName)\n\t\t}\n\t}\n\n\treturn regions, nil\n}\n\n// Returns only regions that are enabled for your account\nfunc (c *Client) ListEnabledRegions() (strset.Set, error) {\n\tresult, err := c.EC2().DescribeRegions(&ec2.DescribeRegionsInput{\n\t\tAllRegions: aws.Bool(false),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tregions := strset.New()\n\tfor _, region := range result.Regions {\n\t\tif region.RegionName != nil {\n\t\t\tregions.Add(*region.RegionName)\n\t\t}\n\t}\n\n\treturn regions, nil\n}\n\n// Returns all regions and enabled regions\nfunc (c *Client) ListRegions() (strset.Set, strset.Set, error) {\n\tvar allRegions strset.Set\n\tvar enabledRegions strset.Set\n\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tallRegions, err = c.ListAllRegions()\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tenabledRegions, err = c.ListEnabledRegions()\n\t\t\treturn err\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\treturn allRegions, enabledRegions, nil\n}\n\nfunc (c *Client) ListAvailabilityZonesInRegion() (strset.Set, error) {\n\tinput := &ec2.DescribeAvailabilityZonesInput{\n\t\tFilters: []*ec2.Filter{\n\t\t\t{\n\t\t\t\tName:   aws.String(\"region-name\"),\n\t\t\t\tValues: []*string{aws.String(c.Region)},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName:   aws.String(\"state\"),\n\t\t\t\tValues: []*string{aws.String(ec2.AvailabilityZoneStateAvailable)},\n\t\t\t},\n\t\t},\n\t}\n\n\tresult, err := c.EC2().DescribeAvailabilityZones(input)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tzones := strset.New()\n\tfor _, az := range result.AvailabilityZones {\n\t\tif az.ZoneName != nil {\n\t\t\tzones.Add(*az.ZoneName)\n\t\t}\n\t}\n\n\treturn zones, nil\n}\n\nfunc (c *Client) listSupportedAvailabilityZonesSingle(instanceType string) (strset.Set, error) {\n\tinput := &ec2.DescribeReservedInstancesOfferingsInput{\n\t\tInstanceType:       &instanceType,\n\t\tIncludeMarketplace: aws.Bool(false),\n\t\tFilters: []*ec2.Filter{\n\t\t\t{\n\t\t\t\tName:   aws.String(\"scope\"),\n\t\t\t\tValues: []*string{aws.String(ec2.ScopeAvailabilityZone)},\n\t\t\t},\n\t\t},\n\t}\n\n\tzones := strset.New()\n\terr := c.EC2().DescribeReservedInstancesOfferingsPages(input, func(output *ec2.DescribeReservedInstancesOfferingsOutput, lastPage bool) bool {\n\t\tfor _, offering := range output.ReservedInstancesOfferings {\n\t\t\tif offering.AvailabilityZone != nil {\n\t\t\t\tzones.Add(*offering.AvailabilityZone)\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn zones, nil\n}\n\nfunc (c *Client) ListSupportedAvailabilityZones(instanceType string, instanceTypes ...string) (strset.Set, error) {\n\tallInstanceTypes := append(instanceTypes, instanceType)\n\tzoneSets := make([]strset.Set, len(allInstanceTypes))\n\tfns := make([]func() error, len(allInstanceTypes))\n\n\tfor i := range allInstanceTypes {\n\t\tlocalIdx := i\n\t\tfns[i] = func() error {\n\t\t\tzones, err := c.listSupportedAvailabilityZonesSingle(allInstanceTypes[localIdx])\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tzoneSets[localIdx] = zones\n\t\t\treturn nil\n\t\t}\n\t}\n\n\terr := parallel.RunFirstErr(fns[0], fns[1:]...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn strset.Intersection(zoneSets...), nil\n}\n\nfunc (c *Client) ListElasticIPs() ([]string, error) {\n\taddresses, err := c.EC2().DescribeAddresses(&ec2.DescribeAddressesInput{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\taddressesList := []string{}\n\tif addresses != nil {\n\t\tfor _, address := range addresses.Addresses {\n\t\t\tif address != nil && address.PublicIp != nil {\n\t\t\t\taddressesList = append(addressesList, *address.PublicIp)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn addressesList, nil\n}\n\nfunc (c *Client) ListInternetGateways() ([]string, error) {\n\tgatewaysList := []string{}\n\terr := c.EC2().DescribeInternetGatewaysPages(&ec2.DescribeInternetGatewaysInput{}, func(output *ec2.DescribeInternetGatewaysOutput, lastPage bool) bool {\n\t\tif output == nil {\n\t\t\treturn false\n\t\t}\n\t\tfor _, gateway := range output.InternetGateways {\n\t\t\tif gateway != nil && gateway.InternetGatewayId != nil {\n\t\t\t\tgatewaysList = append(gatewaysList, *gateway.InternetGatewayId)\n\t\t\t}\n\t\t}\n\n\t\treturn true\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn gatewaysList, nil\n}\n\nfunc (c *Client) DescribeNATGateways() ([]ec2.NatGateway, error) {\n\tvar gateways []ec2.NatGateway\n\terr := c.EC2().DescribeNatGatewaysPages(&ec2.DescribeNatGatewaysInput{}, func(output *ec2.DescribeNatGatewaysOutput, lastPage bool) bool {\n\t\tif output == nil {\n\t\t\treturn false\n\t\t}\n\t\tfor _, gateway := range output.NatGateways {\n\t\t\tif gateway == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tgateways = append(gateways, *gateway)\n\t\t}\n\n\t\treturn true\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn gateways, nil\n}\n\nfunc (c *Client) DescribeSubnets() ([]ec2.Subnet, error) {\n\tvar subnets []ec2.Subnet\n\terr := c.EC2().DescribeSubnetsPages(&ec2.DescribeSubnetsInput{}, func(output *ec2.DescribeSubnetsOutput, lastPage bool) bool {\n\t\tif output == nil {\n\t\t\treturn false\n\t\t}\n\t\tfor _, subnet := range output.Subnets {\n\t\t\tif subnet == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsubnets = append(subnets, *subnet)\n\t\t}\n\n\t\treturn true\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn subnets, nil\n}\n\nfunc (c *Client) DescribeVpcs() ([]ec2.Vpc, error) {\n\tvar vpcs []ec2.Vpc\n\terr := c.EC2().DescribeVpcsPages(&ec2.DescribeVpcsInput{}, func(output *ec2.DescribeVpcsOutput, lastPage bool) bool {\n\t\tif output == nil {\n\t\t\treturn false\n\t\t}\n\t\tfor _, vpc := range output.Vpcs {\n\t\t\tif vpc == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tvpcs = append(vpcs, *vpc)\n\t\t}\n\n\t\treturn true\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn vpcs, nil\n}\n\nfunc (c *Client) DescribeSecurityGroups() ([]ec2.SecurityGroup, error) {\n\tvar sgs []ec2.SecurityGroup\n\terr := c.EC2().DescribeSecurityGroupsPages(&ec2.DescribeSecurityGroupsInput{}, func(output *ec2.DescribeSecurityGroupsOutput, lastPage bool) bool {\n\t\tif output == nil {\n\t\t\treturn false\n\t\t}\n\t\tfor _, sg := range output.SecurityGroups {\n\t\t\tif sg == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsgs = append(sgs, *sg)\n\t\t}\n\n\t\treturn true\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn sgs, nil\n}\n\nfunc (c *Client) ListVolumes(tags ...ec2.Tag) ([]ec2.Volume, error) {\n\tvar volumes []ec2.Volume\n\terr := c.EC2().DescribeVolumesPages(&ec2.DescribeVolumesInput{}, func(output *ec2.DescribeVolumesOutput, lastPage bool) bool {\n\t\tif output == nil {\n\t\t\treturn false\n\t\t}\n\t\tfor _, volume := range output.Volumes {\n\t\t\tif volume == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif hasAllEC2Tags(tags, volume.Tags) {\n\t\t\t\tvolumes = append(volumes, *volume)\n\t\t\t}\n\t\t}\n\n\t\treturn true\n\t})\n\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn volumes, nil\n}\n\nfunc (c *Client) DeleteVolume(volumeID string) error {\n\t_, err := c.EC2().DeleteVolume(&ec2.DeleteVolumeInput{\n\t\tVolumeId: aws.String(volumeID),\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err)\n\t}\n\n\treturn nil\n}\n\nfunc hasAllEC2Tags(queryTags []ec2.Tag, allResourceTags []*ec2.Tag) bool {\n\tfor _, queryTag := range queryTags {\n\t\tif !hasEC2Tag(queryTag, allResourceTags) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// if queryTag's value is nil, the tag will match as long as the key is present in the resource's tags\nfunc hasEC2Tag(queryTag ec2.Tag, allResourceTags []*ec2.Tag) bool {\n\tfor _, resourceTag := range allResourceTags {\n\t\tif queryTag.Key != nil && resourceTag.Key != nil && *queryTag.Key == *resourceTag.Key {\n\t\t\tif queryTag.Value == nil {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tif queryTag.Value != nil && resourceTag.Value != nil && *queryTag.Value == *resourceTag.Value {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n"
  },
  {
    "path": "pkg/lib/aws/ec2_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestParseInstanceType(t *testing.T) {\n\tvar testcases = []struct {\n\t\tinstanceType string\n\t\texpected     ParsedInstanceType\n\t}{\n\t\t{\"t3.small\", ParsedInstanceType{\"t\", 3, strset.New(), \"small\"}},\n\t\t{\"g4dn.xlarge\", ParsedInstanceType{\"g\", 4, strset.New(\"d\", \"n\"), \"xlarge\"}},\n\t\t{\"inf1.24xlarge\", ParsedInstanceType{\"inf\", 1, strset.New(), \"24xlarge\"}},\n\t\t{\"u-9tb1.metal\", ParsedInstanceType{\"u-9tb\", 1, strset.New(), \"metal\"}},\n\t}\n\n\tinvalidTypes := []string{\n\t\t\"badtype\",\n\t\t\"badtype.large\",\n\t\t\"badtype1.large\",\n\t\t\"badtype2ad.large\",\n\t}\n\n\tfor _, testcase := range testcases {\n\t\tparsed, err := ParseInstanceType(testcase.instanceType)\n\t\trequire.NoError(t, err)\n\t\trequire.Equal(t, testcase.expected.Family, parsed.Family, fmt.Sprintf(\"unexpected family for input: %s\", testcase.instanceType))\n\t\trequire.Equal(t, testcase.expected.Generation, parsed.Generation, fmt.Sprintf(\"unexpected generation for input: %s\", testcase.instanceType))\n\t\trequire.ElementsMatch(t, testcase.expected.Capabilities.Slice(), parsed.Capabilities.Slice(), fmt.Sprintf(\"unexpected capabilities for input: %s\", testcase.instanceType))\n\t\trequire.Equal(t, testcase.expected.Size, parsed.Size, fmt.Sprintf(\"unexpected size for input: %s\", testcase.instanceType))\n\t}\n\n\tfor _, instanceType := range invalidTypes {\n\t\t_, err := ParseInstanceType(instanceType)\n\t\trequire.Error(t, err)\n\t}\n\n\tfor instanceType := range AllInstanceTypes {\n\t\t_, err := ParseInstanceType(instanceType)\n\t\trequire.NoError(t, err)\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/aws/ecr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"encoding/base64\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/service/ecr\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/regex\"\n)\n\ntype ECRAuthConfig struct {\n\tUsername      string\n\tAccessToken   string\n\tProxyEndpoint string\n}\n\nvar _ecrRegionRegex = regexp.MustCompile(`ecr\\.(\\S+)\\.amazon`)\n\nfunc (c *Client) GetECRAuthToken() (*ecr.GetAuthorizationTokenOutput, error) {\n\tresult, err := c.ECR().GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})\n\tif err != nil {\n\t\treturn result, errors.Wrap(err, \"failed to retrieve ECR auth token\")\n\t}\n\treturn result, nil\n}\n\nfunc (c *Client) GetECRAuthConfig() (ECRAuthConfig, error) {\n\ttokenOutput, err := c.GetECRAuthToken()\n\tif err != nil {\n\t\treturn ECRAuthConfig{}, err\n\t}\n\tif len(tokenOutput.AuthorizationData) == 0 {\n\t\treturn ECRAuthConfig{}, ErrorECRExtractingCredentials()\n\t}\n\tauthData := tokenOutput.AuthorizationData[0]\n\n\tcredentials, err := base64.URLEncoding.DecodeString(*authData.AuthorizationToken)\n\tif err != nil {\n\t\treturn ECRAuthConfig{}, errors.Wrap(err, ErrorECRExtractingCredentials().Error())\n\t}\n\tcredentialsString := string(credentials)\n\tsplitCredentials := strings.Split(credentialsString, \":\")\n\tif len(splitCredentials) != 2 {\n\t\treturn ECRAuthConfig{}, ErrorECRExtractingCredentials()\n\t}\n\n\treturn ECRAuthConfig{\n\t\tUsername:      splitCredentials[0],\n\t\tAccessToken:   splitCredentials[1],\n\t\tProxyEndpoint: *authData.ProxyEndpoint,\n\t}, nil\n}\n\nfunc GetAccountIDFromECRURL(path string) string {\n\tif regex.IsValidECRURL(path) {\n\t\treturn strings.Split(path, \".\")[0]\n\t}\n\treturn \"\"\n}\n\nfunc GetRegionFromECRURL(path string) string {\n\tres := _ecrRegionRegex.FindStringSubmatch(path)\n\tif len(res) != 2 {\n\t\treturn \"\"\n\t}\n\treturn res[1]\n}\n"
  },
  {
    "path": "pkg/lib/aws/eks.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"github.com/aws/aws-sdk-go/service/eks\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\nvar EKSSupportedRegions strset.Set\n\nfunc init() {\n\tEKSSupportedRegions = strset.New()\n\tfor region := range InstanceMetadatas {\n\t\tEKSSupportedRegions.Add(region)\n\t}\n}\n\n// Returns info for the cluster, or nil of no cluster exists with the provided name\nfunc (c *Client) EKSClusterOrNil(clusterName string) (*eks.Cluster, error) {\n\tclusterInfo, err := c.EKS().DescribeCluster(&eks.DescribeClusterInput{Name: &clusterName})\n\tif err != nil {\n\t\tif IsErrCode(err, eks.ErrCodeResourceNotFoundException) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn clusterInfo.Cluster, nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/elb.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/elb\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype ClassicLoadBalancerState string\n\nconst (\n\tLoadBalancerStateInService    ClassicLoadBalancerState = \"InService\"\n\tLoadBalancerStateOutOfService ClassicLoadBalancerState = \"OutOfService\"\n\tLoadBalancerStateUnknown      ClassicLoadBalancerState = \"Unknown\"\n)\n\nfunc (state ClassicLoadBalancerState) String() string {\n\treturn string(state)\n}\n\n// returns the first classic load balancer which has all of the specified tags, or nil if no load balancers match\nfunc (c *Client) FindLoadBalancer(tags map[string]string) (*elb.LoadBalancerDescription, error) {\n\tvar loadBalancer *elb.LoadBalancerDescription\n\tvar fnErr error\n\n\tparams := elb.DescribeLoadBalancersInput{\n\t\tPageSize: aws.Int64(20), // 20 is the limit for DescribeTags()\n\t}\n\terr := c.ELB().DescribeLoadBalancersPages(&params,\n\t\tfunc(page *elb.DescribeLoadBalancersOutput, lastPage bool) bool {\n\t\t\tloadBalancerNames := make([]string, len(page.LoadBalancerDescriptions))\n\t\t\tloadBalancers := make(map[string]*elb.LoadBalancerDescription)\n\n\t\t\tfor i := range page.LoadBalancerDescriptions {\n\t\t\t\tloadBalancerName := *page.LoadBalancerDescriptions[i].LoadBalancerName\n\t\t\t\tloadBalancerNames[i] = loadBalancerName\n\t\t\t\tloadBalancers[loadBalancerName] = page.LoadBalancerDescriptions[i]\n\t\t\t}\n\n\t\t\ttagsOutput, err := c.ELB().DescribeTags(&elb.DescribeTagsInput{\n\t\t\t\tLoadBalancerNames: aws.StringSlice(loadBalancerNames),\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tfnErr = errors.WithStack(err)\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tfor _, tagDescription := range tagsOutput.TagDescriptions {\n\t\t\t\tlbTags := make(map[string]string, len(tagDescription.Tags))\n\t\t\t\tfor _, lbTag := range tagDescription.Tags {\n\t\t\t\t\tif lbTag.Key != nil && lbTag.Value != nil {\n\t\t\t\t\t\tlbTags[*lbTag.Key] = *lbTag.Value\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tmissingTag := false\n\t\t\t\tfor key, value := range tags {\n\t\t\t\t\tif lbTags[key] != value {\n\t\t\t\t\t\tmissingTag = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif !missingTag {\n\t\t\t\t\tloadBalancer = loadBalancers[*tagDescription.LoadBalancerName]\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn true\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif fnErr != nil {\n\t\treturn nil, fnErr\n\t}\n\n\treturn loadBalancer, nil\n}\n\nfunc (c *Client) IsLoadBalancerHealthy(loadBalancerName string) (bool, error) {\n\tinstanceHealthOutput, err := c.ELB().DescribeInstanceHealth(&elb.DescribeInstanceHealthInput{\n\t\tLoadBalancerName: &loadBalancerName,\n\t})\n\tif err != nil {\n\t\treturn false, errors.WithStack(err)\n\t}\n\n\tfor _, instance := range instanceHealthOutput.InstanceStates {\n\t\tif instance == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif instance.State != nil && *instance.State != LoadBalancerStateInService.String() {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\n\treturn true, nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/elbv2.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/elbv2\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\n// https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-target-groups.html\nvar _nlbUnsupportedInstancePrefixes = strset.New(\"c1\", \"cc1\", \"cc2\", \"cg1\", \"cg2\", \"cr1\", \"g1\", \"g2\", \"hi1\", \"hs1\", \"m1\", \"m2\", \"m3\", \"t1\")\n\nfunc IsInstanceSupportedByNLB(instanceType string) (bool, error) {\n\tif err := CheckValidInstanceType(instanceType); err != nil {\n\t\treturn false, err\n\t}\n\n\tfor prefix := range _nlbUnsupportedInstancePrefixes {\n\t\tif strings.HasPrefix(instanceType, prefix) {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\n// returns the first network/application load balancer which has all of the specified tags, or nil if no load balancers match\nfunc (c *Client) FindLoadBalancerV2(tags map[string]string) (*elbv2.LoadBalancer, error) {\n\tvar loadBalancer *elbv2.LoadBalancer\n\tvar fnErr error\n\n\tparams := elbv2.DescribeLoadBalancersInput{\n\t\tPageSize: aws.Int64(20), // 20 is the limit for DescribeTags()\n\t}\n\terr := c.ELBV2().DescribeLoadBalancersPages(&params,\n\t\tfunc(page *elbv2.DescribeLoadBalancersOutput, lastPage bool) bool {\n\t\t\tarns := make([]string, len(page.LoadBalancers))\n\t\t\tloadBalancers := make(map[string]*elbv2.LoadBalancer)\n\n\t\t\tfor i := range page.LoadBalancers {\n\t\t\t\tarn := *page.LoadBalancers[i].LoadBalancerArn\n\t\t\t\tarns[i] = arn\n\t\t\t\tloadBalancers[arn] = page.LoadBalancers[i]\n\t\t\t}\n\n\t\t\ttagsOutput, err := c.ELBV2().DescribeTags(&elbv2.DescribeTagsInput{\n\t\t\t\tResourceArns: aws.StringSlice(arns),\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tfnErr = errors.WithStack(err)\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tfor _, tagDescription := range tagsOutput.TagDescriptions {\n\t\t\t\tlbTags := make(map[string]string, len(tagDescription.Tags))\n\t\t\t\tfor _, lbTag := range tagDescription.Tags {\n\t\t\t\t\tif lbTag.Key != nil && lbTag.Value != nil {\n\t\t\t\t\t\tlbTags[*lbTag.Key] = *lbTag.Value\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tmissingTag := false\n\t\t\t\tfor key, value := range tags {\n\t\t\t\t\tif lbTags[key] != value {\n\t\t\t\t\t\tmissingTag = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif !missingTag {\n\t\t\t\t\tloadBalancer = loadBalancers[*tagDescription.ResourceArn]\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn true\n\t\t})\n\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif fnErr != nil {\n\t\treturn nil, fnErr\n\t}\n\n\treturn loadBalancer, nil\n}\n\nfunc IsLoadBalancerV2Healthy(loadBalancer elbv2.LoadBalancer) bool {\n\tif loadBalancer.State == nil || loadBalancer.State.Code == nil {\n\t\treturn false\n\t}\n\n\treturn *loadBalancer.State.Code == elbv2.LoadBalancerStateEnumActive\n}\n"
  },
  {
    "path": "pkg/lib/aws/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n\t\"github.com/aws/aws-sdk-go/service/iam\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrInvalidInstanceType          = \"aws.invalid_instance_type\"\n\tErrInvalidAWSCredentials        = \"aws.invalid_aws_credentials\"\n\tErrInvalidS3aPath               = \"aws.invalid_s3a_path\"\n\tErrInvalidS3Path                = \"aws.invalid_s3_path\"\n\tErrUnexpectedMissingCredentials = \"aws.unexpected_missing_credentials\"\n\tErrAuth                         = \"aws.auth\"\n\tErrBucketInaccessible           = \"aws.bucket_inaccessible\"\n\tErrBucketNotFound               = \"aws.bucket_not_found\"\n\tErrInsufficientInstanceQuota    = \"aws.insufficient_instance_quota\"\n\tErrNoValidSpotPrices            = \"aws.no_valid_spot_prices\"\n\tErrECRExtractingCredentials     = \"aws.ecr_failed_credentials\"\n\tErrDashboardWidthOutOfRange     = \"aws.dashboard_width_ouf_of_range\"\n\tErrDashboardHeightOutOfRange    = \"aws.dashboard_height_out_of_range\"\n\tErrRegionNotConfigured          = \"aws.region_not_configured\"\n\tErrUnableToFindCredentials      = \"aws.unable_to_find_credentials\"\n\tErrNATGatewayLimitExceeded      = \"aws.nat_gateway_limit_exceeded\"\n\tErrEIPLimitExceeded             = \"aws.eip_limit_exceeded\"\n\tErrInternetGatewayLimitExceeded = \"aws.internet_gateway_limit_exceeded\"\n\tErrVPCLimitExceeded             = \"aws.vpc_limit_exceeded\"\n\tErrSecurityGroupRulesExceeded   = \"aws.security_group_rules_exceeded\"\n\tErrSecurityGroupLimitExceeded   = \"aws.security_group_limit_exceeded\"\n)\n\nfunc IsAWSError(err error) bool {\n\tif _, ok := errors.CauseOrSelf(err).(awserr.Error); ok {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc IsNotFoundErr(err error) bool {\n\treturn IsErrCode(err, \"NotFound\")\n}\n\nfunc IsNoSuchKeyErr(err error) bool {\n\treturn IsErrCode(err, s3.ErrCodeNoSuchKey)\n}\n\nfunc IsNoSuchEntityErr(err error) bool {\n\treturn IsErrCode(err, iam.ErrCodeNoSuchEntityException)\n}\n\nfunc IsNoSuchBucketErr(err error) bool {\n\treturn IsErrCode(err, s3.ErrCodeNoSuchBucket)\n}\n\nfunc IsNonExistentQueueErr(err error) bool {\n\treturn IsErrCode(err, sqs.ErrCodeQueueDoesNotExist)\n}\n\nfunc IsGenericNotFoundErr(err error) bool {\n\treturn IsNotFoundErr(err) || IsNoSuchKeyErr(err) || IsNoSuchBucketErr(err)\n}\n\nfunc IsErrCode(err error, errorCode string) bool {\n\tawsErr, ok := errors.CauseOrSelf(err).(awserr.Error)\n\tif !ok {\n\t\treturn false\n\t}\n\tif awsErr.Code() == errorCode {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc ErrorInvalidInstanceType(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidInstanceType,\n\t\tMessage: fmt.Sprintf(\"%s is not an AWS instance type (e.g. m5.large is a valid instance type)\", s.UserStr(instanceType)),\n\t})\n}\n\nfunc ErrorInvalidAWSCredentials(awsErr error) error {\n\tawsErrMsg := errors.Message(awsErr)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidAWSCredentials,\n\t\tMessage: \"invalid AWS credentials\\n\" + awsErrMsg,\n\t\tCause:   awsErr,\n\t})\n}\n\nfunc ErrorInvalidS3aPath(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidS3aPath,\n\t\tMessage: fmt.Sprintf(\"%s is not a valid s3a path (e.g. s3a://cortex-examples/pytorch/iris-classifier/weights.pth is a valid s3a path)\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorInvalidS3Path(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidS3Path,\n\t\tMessage: fmt.Sprintf(\"%s is not a valid s3 path (e.g. s3://cortex-examples/pytorch/iris-classifier/weights.pth is a valid s3 path)\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorUnexpectedMissingCredentials(awsAccessKeyID string, awsSecretAccessKey string) error {\n\tvar msg string\n\tif awsAccessKeyID == \"\" && awsSecretAccessKey == \"\" {\n\t\tmsg = \"aws access key id and aws secret access key are missing\"\n\t} else if awsAccessKeyID == \"\" {\n\t\tmsg = \"aws access key id is missing\"\n\t} else if awsSecretAccessKey == \"\" {\n\t\tmsg = \"aws secret access key is missing\"\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUnexpectedMissingCredentials,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorAuth() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAuth,\n\t\tMessage: \"unable to authenticate with AWS\",\n\t})\n}\n\nfunc ErrorBucketInaccessible(bucket string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrBucketInaccessible,\n\t\tMessage: fmt.Sprintf(\"bucket \\\"%s\\\" is not accessible with the specified AWS credentials\", bucket),\n\t})\n}\n\nfunc ErrorBucketNotFound(bucket string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrBucketNotFound,\n\t\tMessage: fmt.Sprintf(\"bucket \\\"%s\\\" not found\", bucket),\n\t})\n}\n\nfunc ErrorInsufficientInstanceQuota(instanceTypes []string, lifecycle string, region string, requiredVCPUs int64, vCPUQuota int64, quotaCode string) error {\n\turl := fmt.Sprintf(\"https://%s.console.aws.amazon.com/servicequotas/home?region=%s#!/services/ec2/quotas/%s\", region, region, quotaCode)\n\tandInstanceTypes := s.StrsAnd(instanceTypes)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInsufficientInstanceQuota,\n\t\tMessage: fmt.Sprintf(\"your cluster may require up to %d vCPU %s %s instances, but your AWS quota for %s %s instances in %s is only %d vCPU; please reduce the maximum number of %s %s instances your cluster may use (e.g. by changing max_instances and/or spot_config if applicable), or request a quota increase to at least %d vCPU here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)\", requiredVCPUs, lifecycle, andInstanceTypes, lifecycle, andInstanceTypes, region, vCPUQuota, lifecycle, andInstanceTypes, requiredVCPUs, url),\n\t})\n}\n\nfunc ErrorNoValidSpotPrices(instanceType string, region string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoValidSpotPrices,\n\t\tMessage: fmt.Sprintf(\"no spot prices were found for %s instances in %s\", instanceType, region),\n\t})\n}\n\nfunc ErrorECRExtractingCredentials() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrECRExtractingCredentials,\n\t\tMessage: \"unable to extract ECR credentials\",\n\t})\n}\n\nfunc ErrorDashboardWidthOutOfRange(width int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDashboardWidthOutOfRange,\n\t\tMessage: fmt.Sprintf(\"dashboard width %d out of range; width must be between %d and %d\", width, _dashboardMinWidthUnits, _dashboardMaxWidthUnits),\n\t})\n}\n\nfunc ErrorDashboardHeightOutOfRange(height int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDashboardHeightOutOfRange,\n\t\tMessage: fmt.Sprintf(\"dashboard height %d out of range; height must be between %d and %d\", height, _dashboardMinHeightUnits, _dashboardMaxHeightUnits),\n\t})\n}\n\nfunc ErrorRegionNotConfigured() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrRegionNotConfigured,\n\t\tMessage: \"aws region has not been configured; please set a default region (e.g. `export AWS_DEFAULT_REGION=us-west-2`)\",\n\t})\n}\n\nfunc ErrorUnableToFindCredentials() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUnableToFindCredentials,\n\t\tMessage: \"unable to find aws credentials; instructions about configuring aws credentials can be found at https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html\",\n\t})\n}\n\nfunc ErrorNATGatewayLimitExceeded(currentLimit, additionalQuotaRequired int, availabilityZones []string, region string) error {\n\turl := \"https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas\"\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNATGatewayLimitExceeded,\n\t\tMessage: fmt.Sprintf(\"NAT gateway limit of %d exceeded in availability zones %s of region %s; remove some of the existing NAT gateways or increase your quota for NAT gateways by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)\", currentLimit, s.StrsAnd(availabilityZones), region, additionalQuotaRequired, url),\n\t})\n}\n\nfunc ErrorEIPLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error {\n\turl := \"https://console.aws.amazon.com/servicequotas/home?#!/services/ec2/quotas\"\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEIPLimitExceeded,\n\t\tMessage: fmt.Sprintf(\"elastic IPs limit of %d exceeded in region %s; remove some of the existing elastic IPs or increase your quota for elastic IPs by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)\", currentLimit, region, additionalQuotaRequired, url),\n\t})\n}\n\nfunc ErrorInternetGatewayLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error {\n\turl := \"https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas\"\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInternetGatewayLimitExceeded,\n\t\tMessage: fmt.Sprintf(\"internet gateway limit of %d exceeded in region %s; remove some of the existing internet gateways or increase your quota for internet gateways by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)\", currentLimit, region, additionalQuotaRequired, url),\n\t})\n}\n\nfunc ErrorVPCLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error {\n\turl := \"https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas\"\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrVPCLimitExceeded,\n\t\tMessage: fmt.Sprintf(\"VPC limit of %d exceeded in region %s; remove some of the existing VPCs or increase your quota for VPCs by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)\", currentLimit, region, additionalQuotaRequired, url),\n\t})\n}\n\nfunc ErrorSecurityGroupRulesExceeded(currentLimit, additionalQuotaRequired int, region string) error {\n\turl := \"https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas\"\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSecurityGroupRulesExceeded,\n\t\tMessage: fmt.Sprintf(\"security group rules limit of %d exceeded in region %s; remove some node groups, use fewer availability zones, reduce the number of CIDR white lists, or increase your quota for inbound/outbound rules per security group by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)\", currentLimit, region, additionalQuotaRequired, url),\n\t})\n}\n\nfunc ErrorSecurityGroupLimitExceeded(currentLimit, additionalQuotaRequired int, region string) error {\n\turl := \"https://console.aws.amazon.com/servicequotas/home?#!/services/vpc/quotas\"\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSecurityGroupLimitExceeded,\n\t\tMessage: fmt.Sprintf(\"security group limit of %d exceeded in region %s; remove some node groups or increase your quota for security groups by at least %d here: %s (if your request was recently approved, please allow ~30 minutes for AWS to reflect this change)\", currentLimit, region, additionalQuotaRequired, url),\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/aws/gen_resource_metadata.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport requests\nimport re\nfrom string import Template\n\n# https://docs.aws.amazon.com/general/latest/gr/eks.html\n# China regions don't seem to support these endpoints (yet?)\nREGIONS = [\n    \"us-east-2\",  # Ohio\n    \"us-east-1\",  # N. Virginia\n    \"us-west-1\",  # California\n    \"us-west-2\",  # Oregon\n    \"af-south-1\",  # Cape town\n    \"ap-east-1\",  # Hong Kong\n    \"ap-south-1\",  # Mumbai\n    \"ap-northeast-3\",  # Osaka\n    \"ap-northeast-2\",  # Seoul\n    \"ap-southeast-1\",  # Singapore\n    \"ap-southeast-2\",  # Sydney\n    \"ap-northeast-1\",  # Tokyo\n    \"ca-central-1\",  # Montreal\n    \"eu-central-1\",  # Frankfurt\n    \"eu-west-1\",  # Ireland\n    \"eu-west-2\",  # London\n    \"eu-south-1\",  # Milan\n    \"eu-west-3\",  # Paris\n    \"eu-north-1\",  # Stockholm\n    \"me-south-1\",  # Bahrain\n    \"sa-east-1\",  # Sao Paulo\n    \"us-gov-east-1\",  # GovCloud US-East\n    \"us-gov-west-1\",  # GovCloud US-West\n]\n\nOUTPUT_FILE_NAME = \"resource_metadata.go\"\n\nEC2_PRICING_ENDPOINT_TEMPLATE = (\n    \"https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/{}/index.json\"\n)\n\nEKS_PRICING_ENDPOINT_TEMPLATE = (\n    \"https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEKS/current/{}/index.json\"\n)\n\ninf_per_instance_type = {\n    \"inf1.xlarge\": 1,\n    \"inf1.2xlarge\": 1,\n    \"inf1.6xlarge\": 4,\n    \"inf1.24xlarge\": 16,\n}\n\n\ndef get_instance_metadatas(pricing):\n    instance_types = set()\n    instance_mapping = {}\n\n    for _, product in pricing[\"products\"].items():\n        if product.get(\"attributes\") is None:\n            continue\n        if product[\"attributes\"].get(\"instanceType\") is None:\n            continue\n        if product[\"attributes\"].get(\"servicecode\") != \"AmazonEC2\":\n            continue\n        if product[\"attributes\"].get(\"capacitystatus\") != \"Used\":\n            continue\n\n        instance_type = product[\"attributes\"][\"instanceType\"]\n        instance_types.add(instance_type)\n\n        if product[\"attributes\"].get(\"operation\") != \"RunInstances\":\n            continue\n        if product[\"attributes\"].get(\"tenancy\") != \"Shared\":\n            continue\n        if product[\"attributes\"].get(\"operatingSystem\") != \"Linux\":\n            continue\n\n        price_dimensions = list(pricing[\"terms\"][\"OnDemand\"][product[\"sku\"]].values())[0][\n            \"priceDimensions\"\n        ]\n\n        price = list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]\n\n        metadata = {\n            \"cpu\": int(product[\"attributes\"][\"vcpu\"]),\n            \"mem\": int(\n                float(re.sub(\"[^0-9\\\\.]\", \"\", product[\"attributes\"][\"memory\"].split(\" \")[0])) * 1024\n            ),\n            \"price\": float(price),\n            \"gpu\": 0,\n        }\n        if product[\"attributes\"].get(\"gpu\") is not None:\n            metadata[\"gpu\"] = product[\"attributes\"][\"gpu\"]\n        instance_mapping[instance_type] = metadata\n\n    return instance_types, instance_mapping\n\n\ndef get_nlb_metadata(pricing):\n    for _, product in pricing[\"products\"].items():\n        if product.get(\"attributes\") is None:\n            continue\n        if product.get(\"productFamily\") != \"Load Balancer-Network\":\n            continue\n        if product[\"attributes\"].get(\"group\") != \"ELB:Balancer\":\n            continue\n        if product[\"attributes\"].get(\"operation\") != \"LoadBalancing:Network\":\n            continue\n        if \"LoadBalancerUsage\" not in product[\"attributes\"].get(\"usagetype\"):\n            continue\n\n        price_dimensions = list(pricing[\"terms\"][\"OnDemand\"][product[\"sku\"]].values())[0][\n            \"priceDimensions\"\n        ]\n        price = list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]\n        return {\"price\": float(price)}\n\n\ndef get_elb_metadata(pricing):\n    for _, product in pricing[\"products\"].items():\n        if product.get(\"attributes\") is None:\n            continue\n        if product.get(\"productFamily\") != \"Load Balancer\":\n            continue\n        if product[\"attributes\"].get(\"group\") != \"ELB:Balancer\":\n            continue\n        if product[\"attributes\"].get(\"operation\") != \"LoadBalancing\":\n            continue\n        if \"LoadBalancerUsage\" not in product[\"attributes\"].get(\"usagetype\"):\n            continue\n\n        price_dimensions = list(pricing[\"terms\"][\"OnDemand\"][product[\"sku\"]].values())[0][\n            \"priceDimensions\"\n        ]\n        price = list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]\n        return {\"price\": float(price)}\n\n\ndef get_nat_metadata(pricing):\n    for _, product in pricing[\"products\"].items():\n        if product.get(\"attributes\") is None:\n            continue\n        if product.get(\"productFamily\") != \"NAT Gateway\":\n            continue\n        if product[\"attributes\"].get(\"group\") != \"NGW:NatGateway\":\n            continue\n        if product[\"attributes\"].get(\"operation\") != \"NatGateway\":\n            continue\n        if not product[\"attributes\"].get(\"usagetype\", \"\").endswith(\"-Hours\"):\n            continue\n\n        price_dimensions = list(pricing[\"terms\"][\"OnDemand\"][product[\"sku\"]].values())[0][\n            \"priceDimensions\"\n        ]\n        price = list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]\n        return {\"price\": float(price)}\n\n\ndef get_ebs_metadata(pricing):\n    storage_mapping = {}\n\n    for _, product in pricing[\"products\"].items():\n        if product.get(\"attributes\") is None:\n            continue\n        if product.get(\"productFamily\") != \"Storage\":\n            continue\n        # ignore legacy standard storage\n        if product[\"attributes\"].get(\"volumeApiName\") == \"standard\":\n            continue\n\n        price_dimensions = list(pricing[\"terms\"][\"OnDemand\"][product[\"sku\"]].values())[0][\n            \"priceDimensions\"\n        ]\n        price = list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]\n\n        metadata = {\"type\": product[\"attributes\"].get(\"volumeApiName\"), \"price_gb\": float(price)}\n\n        # io1 has per IOPS pricing --> add pricing to metadata\n        # if storagedevice does not price per IOPS will set value to 0\n        if product[\"attributes\"].get(\"volumeApiName\") == \"io1\":\n            # go through pricing data until found data about IOPS pricing\n            for _, product_iops in pricing[\"products\"].items():\n                if product_iops.get(\"attributes\") is None:\n                    continue\n                if product_iops.get(\"productFamily\") != \"System Operation\":\n                    continue\n                if product_iops[\"attributes\"].get(\"volumeApiName\") != \"io1\":\n                    continue\n                if product_iops[\"attributes\"].get(\"group\") != \"EBS IOPS\":\n                    continue\n                if product_iops[\"attributes\"].get(\"provisioned\") != \"Yes\":\n                    continue\n\n                price_dimensions = list(pricing[\"terms\"][\"OnDemand\"][product_iops[\"sku\"]].values())[\n                    0\n                ][\"priceDimensions\"]\n                price = list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]\n\n                metadata[\"price_iops\"] = price\n                metadata[\"price_throughput\"] = 0\n                metadata[\"iops_configurable\"] = \"true\"\n                metadata[\"throughput_configurable\"] = \"false\"\n\n        elif product[\"attributes\"].get(\"volumeApiName\") == \"gp3\":\n            # go through pricing data until found data about IOPS and throughput pricing\n            for _, product_iops in pricing[\"products\"].items():\n                if product_iops.get(\"attributes\") is None:\n                    continue\n                if product_iops.get(\"productFamily\") not in [\n                    \"System Operation\",\n                    \"Provisioned Throughput\",\n                ]:\n                    continue\n                if product_iops[\"attributes\"].get(\"volumeApiName\") != \"gp3\":\n                    continue\n                if product_iops[\"attributes\"].get(\"group\") not in [\"EBS IOPS\", \"EBS Throughput\"]:\n                    continue\n                if product_iops[\"attributes\"].get(\"provisioned\") != \"Yes\":\n                    continue\n\n                price_dimensions = list(pricing[\"terms\"][\"OnDemand\"][product_iops[\"sku\"]].values())[\n                    0\n                ][\"priceDimensions\"]\n                if product_iops[\"attributes\"].get(\"group\") == \"EBS IOPS\":\n                    price_iops = list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]\n                else:\n                    price_throughput = (\n                        float(list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]) / 1000\n                    )\n\n            metadata[\"price_iops\"] = price_iops\n            metadata[\"price_throughput\"] = price_throughput\n            metadata[\"throughput_configurable\"] = \"true\"\n            metadata[\"iops_configurable\"] = \"true\"\n\n        # set default values for all other storage types\n        else:\n            metadata[\"price_iops\"] = 0\n            metadata[\"price_throughput\"] = 0\n            metadata[\"iops_configurable\"] = \"false\"\n            metadata[\"throughput_configurable\"] = \"false\"\n\n        storage_mapping[product[\"attributes\"][\"volumeApiName\"]] = metadata\n\n    return storage_mapping\n\n\ndef get_eks_price(region):\n    response = requests.get(EKS_PRICING_ENDPOINT_TEMPLATE.format(region))\n    pricing = response.json()\n\n    for _, product in pricing[\"products\"].items():\n        if product.get(\"attributes\") is None:\n            continue\n        if product.get(\"productFamily\") != \"Compute\":\n            continue\n        if product[\"attributes\"].get(\"servicecode\") != \"AmazonEKS\":\n            continue\n        if product[\"attributes\"].get(\"operation\") != \"CreateOperation\":\n            continue\n        if not product[\"attributes\"].get(\"usagetype\", \"\").endswith(\"-AmazonEKS-Hours:perCluster\"):\n            continue\n\n        price_dimensions = list(pricing[\"terms\"][\"OnDemand\"][product[\"sku\"]].values())[0][\n            \"priceDimensions\"\n        ]\n        price = list(price_dimensions.values())[0][\"pricePerUnit\"][\"USD\"]\n        return float(price)\n\n\nfile_template = Template(\n    \"\"\"/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n// This file was generated by go generate; DO NOT EDIT\n\npackage aws\n\nimport (\n    \"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n)\n\ntype InstanceMetadata struct {\n\tRegion      string             `json:\"region\"`\n\tType        string             `json:\"type\"`\n\tMemory      kresource.Quantity `json:\"memory\"`\n\tCPU         kresource.Quantity `json:\"cpu\"`\n\tGPU         int64              `json:\"gpu\"`\n\tInf         int64              `json:\"inf\"`\n\tPrice       float64            `json:\"price\"`\n}\n\ntype NLBMetadata struct {\n\tRegion string  `json:\"region\"`\n\tPrice  float64 `json:\"price\"`\n}\n\ntype ELBMetadata struct {\n\tRegion string  `json:\"region\"`\n\tPrice  float64 `json:\"price\"`\n}\n\ntype NATMetadata struct {\n\tRegion string  `json:\"region\"`\n\tPrice  float64 `json:\"price\"`\n}\n\ntype EBSMetadata struct {\n\tRegion string  `json:\"region\"`\n\tPriceGB  float64 `json:\"price_gb\"`\n\tPriceIOPS  float64 `json:\"price_iops\"`\n\tPriceThroughput float64 `json:\"price_throughput\"`\n\tIOPSConfigurable bool `json:\"iops_configurable\"`\n\tThroughputConfigurable bool `json:\"throughput_configurable\"`\n\tType  string `json:\"type\"`\n}\n\n// This set contains all known instance types. Metadata is not available for all of them.\nvar AllInstanceTypes = strset.New(\n\t${all_instance_types}\n)\n\n// This contains all instance types available in each region. Metadata is not available for all of them.\nvar InstanceTypes = map[string]strset.Set{\n\t${instance_types_map}\n}\n\n// region -> instance type -> instance metadata\nvar InstanceMetadatas = map[string]map[string]InstanceMetadata{\n\t${instance_region_map}\n}\n\n// region -> NLB metadata\nvar NLBMetadatas = map[string]NLBMetadata{\n\t${nlb_region_map}\n}\n\n// region -> ELB metadata\nvar ELBMetadatas = map[string]ELBMetadata{\n\t${elb_region_map}\n}\n\n// region -> NAT metadata\nvar NATMetadatas = map[string]NATMetadata{\n\t${nat_region_map}\n}\n\n// region -> EBS metadata\nvar EBSMetadatas = map[string]map[string]EBSMetadata{\n\t${ebs_region_map}\n}\n\n// region -> EKS price\nvar EKSPrices = map[string]float64{\n\t${eks_region_map}\n}\n\"\"\"\n)\n\ninstance_region_map_template = Template(\n    \"\"\"\"${region}\": {\n\t${instance_metadatas}\n},\n\"\"\"\n)\n\ninstance_types_map_template = Template(\n    \"\"\"\"${region}\":\n\tstrset.New(\n\t${instance_types}\n\t),\n\"\"\"\n)\n\ninstance_metadata_template = Template(\n    \"\"\"\"${type}\": {Region: \"${region}\", Type: \"${type}\", Memory: kresource.MustParse(\"${memory}Mi\"), CPU: kresource.MustParse(\"${cpu}\"), GPU: ${gpu}, Inf: ${inf}, Price: ${price}},\n\"\"\"\n)\n\nnlb_region_map_template = Template(\n    \"\"\"\"${region}\": {Region: \"${region}\", Price: ${price}},\n\"\"\"\n)\n\nelb_region_map_template = Template(\n    \"\"\"\"${region}\": {Region: \"${region}\", Price: ${price}},\n\"\"\"\n)\n\nnat_region_map_template = Template(\n    \"\"\"\"${region}\": {Region: \"${region}\", Price: ${price}},\n\"\"\"\n)\n\nebs_region_map_template = Template(\n    \"\"\"\"${region}\": {\n\t${ebs_metadata}\n},\n\"\"\"\n)\n\nebs_type_map_template = Template(\n    \"\"\"\"${type}\": {Region: \"${region}\",Type: \"${type}\", PriceGB: ${price_gb}, PriceIOPS: ${price_iops}, PriceThroughput: ${price_throughput}, IOPSConfigurable: ${iops_configurable}, ThroughputConfigurable: ${throughput_configurable}},\n\"\"\"\n)\n\neks_region_map_template = Template(\n    \"\"\"\"${region}\": ${price},\n\"\"\"\n)\n\n\ndef instanceTypeSorter(instanceType):\n    parts = instanceType.split(\".\")\n    if len(parts) != 2:\n        raise Exception(f\"unknown instance type: {instanceType}\")\n    prefix = parts[0]\n    size = parts[1]\n\n    if size == \"nano\":\n        return prefix + \".a\"\n    if size == \"micro\":\n        return prefix + \".b\"\n    if size == \"small\":\n        return prefix + \".c\"\n    if size == \"medium\":\n        return prefix + \".d\"\n    if size == \"large\":\n        return prefix + \".e\"\n    if size == \"xlarge\":\n        return prefix + \".f\"\n    if size == \"metal\":\n        return prefix + \".z\"\n\n    if not size.endswith(\"xlarge\"):\n        raise Exception(f\"unknown instance type: {instanceType}\")\n\n    num_xlarge = re.sub(\"xlarge$\", \"\", size)\n    return prefix + \".y\" + num_xlarge.zfill(5)\n\n\ndef main():\n    all_instance_types = set()\n\n    instance_types_map_str = \"\"\n    instance_region_map_str = \"\"\n    nlb_region_map_str = \"\"\n    elb_region_map_str = \"\"\n    nat_region_map_str = \"\"\n    ebs_region_map_str = \"\"\n    eks_region_map_str = \"\"\n\n    for i, region in enumerate(sorted(REGIONS), start=1):\n        print(\"generating region {}/{} ({})...\".format(i, len(REGIONS), region))\n\n        response = requests.get(EC2_PRICING_ENDPOINT_TEMPLATE.format(region))\n        pricing = response.json()\n\n        instance_types, instance_metadatas = get_instance_metadatas(pricing)\n        nlb_metadata = get_nlb_metadata(pricing)\n        elb_metadata = get_elb_metadata(pricing)\n        nat_metadata = get_nat_metadata(pricing)\n        ebs_metadata = get_ebs_metadata(pricing)\n        eks_price = get_eks_price(region)\n\n        all_instance_types.update(instance_types)\n\n        instance_types_str = \"\"\n        for instance_type in sorted(instance_types, key=instanceTypeSorter):\n            instance_types_str += f'\"{instance_type}\",\\n'\n\n        instance_metadatas_str = \"\"\n        for instance_type in sorted(instance_metadatas.keys(), key=instanceTypeSorter):\n            metadata = instance_metadatas[instance_type]\n            instance_metadatas_str += instance_metadata_template.substitute(\n                {\n                    \"region\": region,\n                    \"type\": instance_type,\n                    \"memory\": metadata[\"mem\"],\n                    \"cpu\": metadata[\"cpu\"],\n                    \"gpu\": metadata[\"gpu\"],\n                    \"inf\": inf_per_instance_type.get(instance_type, 0),\n                    \"price\": metadata[\"price\"],\n                }\n            )\n\n        ebs_metadatas_str = \"\"\n\n        for ebs_type in sorted(ebs_metadata.keys()):\n            metadata = ebs_metadata[ebs_type]\n            ebs_metadatas_str += ebs_type_map_template.substitute(\n                {\n                    \"region\": region,\n                    \"type\": ebs_type,\n                    \"price_gb\": metadata[\"price_gb\"],\n                    \"price_iops\": metadata[\"price_iops\"],\n                    \"price_throughput\": metadata[\"price_throughput\"],\n                    \"iops_configurable\": metadata[\"iops_configurable\"],\n                    \"throughput_configurable\": metadata[\"throughput_configurable\"],\n                }\n            )\n\n        instance_types_map_str += instance_types_map_template.substitute(\n            {\"region\": region, \"instance_types\": instance_types_str}\n        )\n        instance_region_map_str += instance_region_map_template.substitute(\n            {\"region\": region, \"instance_metadatas\": instance_metadatas_str}\n        )\n        nlb_region_map_str += nlb_region_map_template.substitute(\n            {\"region\": region, \"price\": nlb_metadata[\"price\"]}\n        )\n        elb_region_map_str += elb_region_map_template.substitute(\n            {\"region\": region, \"price\": elb_metadata[\"price\"]}\n        )\n        nat_region_map_str += nat_region_map_template.substitute(\n            {\"region\": region, \"price\": nat_metadata[\"price\"]}\n        )\n        ebs_region_map_str += ebs_region_map_template.substitute(\n            {\"region\": region, \"ebs_metadata\": ebs_metadatas_str}\n        )\n        eks_region_map_str += eks_region_map_template.substitute(\n            {\"region\": region, \"price\": eks_price}\n        )\n\n    all_instance_types_str = \"\"\n    for instance_type in sorted(all_instance_types, key=instanceTypeSorter):\n        all_instance_types_str += f'\"{instance_type}\",\\n'\n\n    file_str = file_template.substitute(\n        {\n            \"all_instance_types\": all_instance_types_str,\n            \"instance_types_map\": instance_types_map_str,\n            \"instance_region_map\": instance_region_map_str,\n            \"nlb_region_map\": nlb_region_map_str,\n            \"elb_region_map\": elb_region_map_str,\n            \"nat_region_map\": nat_region_map_str,\n            \"ebs_region_map\": ebs_region_map_str,\n            \"eks_region_map\": eks_region_map_str,\n        }\n    )\n\n    with open(OUTPUT_FILE_NAME, \"w\") as f:\n        print(\"writing {}...\".format(OUTPUT_FILE_NAME))\n        f.write(file_str)\n        print(\"✓ done\")\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "pkg/lib/aws/iam.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n\t\"github.com/aws/aws-sdk-go/service/iam\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nfunc PartitionFromRegion(region string) string {\n\tif strings.Contains(region, \"us-gov\") {\n\t\treturn \"aws-us-gov\"\n\t}\n\treturn \"aws\"\n}\n\nfunc administratorAccessARN(region string) string {\n\treturn fmt.Sprintf(\"arn:%s:iam::aws:policy/AdministratorAccess\", PartitionFromRegion(region))\n}\n\nfunc (c *Client) GetUser() (iam.User, error) {\n\tgetUserOutput, err := c.IAM().GetUser(nil)\n\tif err != nil {\n\t\treturn iam.User{}, errors.WithStack(err)\n\t}\n\treturn *getUserOutput.User, nil\n}\n\nfunc (c *Client) GetGroupsForUser(userName string) ([]iam.Group, error) {\n\tinput := &iam.ListGroupsForUserInput{\n\t\tUserName: &userName,\n\t}\n\n\tvar groups []iam.Group\n\n\terr := c.IAM().ListGroupsForUserPages(input, func(page *iam.ListGroupsForUserOutput, lastPage bool) bool {\n\t\tfor _, group := range page.Groups {\n\t\t\tgroups = append(groups, *group)\n\t\t}\n\t\treturn true\n\t})\n\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn groups, nil\n}\n\n// Note: root users don't have attached policies, but do have full access\nfunc (c *Client) GetManagedPoliciesForUser(userName string) ([]iam.AttachedPolicy, error) {\n\tvar policies []iam.AttachedPolicy\n\n\tuserManagedPolicies, err := c.IAM().ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{\n\t\tUserName: &userName,\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor _, policy := range userManagedPolicies.AttachedPolicies {\n\t\tpolicies = append(policies, *policy)\n\t}\n\n\tgroups, err := c.GetGroupsForUser(userName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, group := range groups {\n\t\tgroupManagedPolicies, err := c.IAM().ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{\n\t\t\tGroupName: group.GroupName,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfor _, policy := range groupManagedPolicies.AttachedPolicies {\n\t\t\tpolicies = append(policies, *policy)\n\t\t}\n\t}\n\n\treturn policies, nil\n}\n\nfunc (c *Client) isAdminUser(user iam.User) bool {\n\t// Root users may not have a user name\n\tif user.UserName == nil {\n\t\treturn true\n\t}\n\n\t// Root users may have a user name\n\tif user.Arn == nil || strings.HasSuffix(*user.Arn, \":root\") {\n\t\treturn true\n\t}\n\n\tpolicies, err := c.GetManagedPoliciesForUser(*user.UserName)\n\tif err != nil {\n\t\treturn false\n\t}\n\n\tfor _, policy := range policies {\n\t\tif *policy.PolicyArn == administratorAccessARN(c.Region) {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc (c *Client) isRoleAdmin() bool {\n\tidentity, err := c.STS().GetCallerIdentity(nil)\n\tif err != nil {\n\t\treturn false\n\t}\n\n\tarn := identity.Arn\n\tif arn == nil {\n\t\treturn false\n\t}\n\n\tif !strings.Contains(*arn, \":assumed-role/\") {\n\t\treturn false\n\t}\n\n\t// expected to be in form arn:aws:sts::account-id:assumed-role/role-name/role-session-name\n\tarnSplit := strings.Split(*arn, \"/\")\n\tif len(arnSplit) < 2 {\n\t\treturn false\n\t}\n\troleName := arnSplit[1]\n\n\tisAdmin := false\n\tc.IAM().ListAttachedRolePoliciesPages(&iam.ListAttachedRolePoliciesInput{\n\t\tRoleName: &roleName,\n\t}, func(policies *iam.ListAttachedRolePoliciesOutput, lastPage bool) bool {\n\t\tfor _, policy := range policies.AttachedPolicies {\n\t\t\tif *policy.PolicyArn == administratorAccessARN(c.Region) {\n\t\t\t\tisAdmin = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\n\t\treturn !lastPage\n\t})\n\treturn isAdmin\n}\n\nfunc (c *Client) IsAdmin() bool {\n\tuser, err := c.GetUser()\n\tif err != nil {\n\t\tawsErr, ok := errors.CauseOrSelf(err).(awserr.Error)\n\t\tif !ok {\n\t\t\treturn false\n\t\t}\n\n\t\t// this particular error is returned if GetUser() is invoked using credentials that are not for users\n\t\tif awsErr.Code() == \"ValidationError\" && strings.Contains(strings.ToLower(err.Error()), strings.ToLower(\"calling with non-User credentials\")) {\n\t\t\treturn c.isRoleAdmin()\n\t\t}\n\t\treturn false\n\t}\n\treturn c.isAdminUser(user)\n}\n\n// delete non default policy versions and then delete the policy (as required by aws)\nfunc (c *Client) DeletePolicy(policyARN string) error {\n\tpolicyVersionList, err := c.IAM().ListPolicyVersions(&iam.ListPolicyVersionsInput{\n\t\tPolicyArn: aws.String(policyARN),\n\t})\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\tfor _, policy := range policyVersionList.Versions {\n\t\tif !*policy.IsDefaultVersion {\n\t\t\t_, err = c.IAM().DeletePolicyVersion(&iam.DeletePolicyVersionInput{\n\t\t\t\tPolicyArn: aws.String(policyARN),\n\t\t\t\tVersionId: policy.VersionId,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn errors.WithStack(err)\n\t\t\t}\n\t\t}\n\t}\n\n\t_, err = c.IAM().DeletePolicy(&iam.DeletePolicyInput{\n\t\tPolicyArn: aws.String(policyARN),\n\t})\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\treturn nil\n}\n\nfunc (c *Client) GetPolicyOrNil(policyARN string) (*iam.Policy, error) {\n\tpolicyOutput, err := c.IAM().GetPolicy(&iam.GetPolicyInput{\n\t\tPolicyArn: aws.String(policyARN),\n\t})\n\tif err != nil {\n\t\tif IsErrCode(err, iam.ErrCodeNoSuchEntityException) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif policyOutput != nil {\n\t\treturn policyOutput.Policy, nil\n\t}\n\treturn nil, nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/resource_metadata.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\n// This file was generated by go generate; DO NOT EDIT\n\npackage aws\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n)\n\ntype InstanceMetadata struct {\n\tRegion string             `json:\"region\"`\n\tType   string             `json:\"type\"`\n\tMemory kresource.Quantity `json:\"memory\"`\n\tCPU    kresource.Quantity `json:\"cpu\"`\n\tGPU    int64              `json:\"gpu\"`\n\tInf    int64              `json:\"inf\"`\n\tPrice  float64            `json:\"price\"`\n}\n\ntype NLBMetadata struct {\n\tRegion string  `json:\"region\"`\n\tPrice  float64 `json:\"price\"`\n}\n\ntype ELBMetadata struct {\n\tRegion string  `json:\"region\"`\n\tPrice  float64 `json:\"price\"`\n}\n\ntype NATMetadata struct {\n\tRegion string  `json:\"region\"`\n\tPrice  float64 `json:\"price\"`\n}\n\ntype EBSMetadata struct {\n\tRegion                 string  `json:\"region\"`\n\tPriceGB                float64 `json:\"price_gb\"`\n\tPriceIOPS              float64 `json:\"price_iops\"`\n\tPriceThroughput        float64 `json:\"price_throughput\"`\n\tIOPSConfigurable       bool    `json:\"iops_configurable\"`\n\tThroughputConfigurable bool    `json:\"throughput_configurable\"`\n\tType                   string  `json:\"type\"`\n}\n\n// This set contains all known instance types. Metadata is not available for all of them.\nvar AllInstanceTypes = strset.New(\n\t\"a1.medium\",\n\t\"a1.large\",\n\t\"a1.xlarge\",\n\t\"a1.2xlarge\",\n\t\"a1.4xlarge\",\n\t\"a1.metal\",\n\t\"c1.medium\",\n\t\"c1.xlarge\",\n\t\"c3.large\",\n\t\"c3.xlarge\",\n\t\"c3.2xlarge\",\n\t\"c3.4xlarge\",\n\t\"c3.8xlarge\",\n\t\"c4.large\",\n\t\"c4.xlarge\",\n\t\"c4.2xlarge\",\n\t\"c4.4xlarge\",\n\t\"c4.8xlarge\",\n\t\"c5.large\",\n\t\"c5.xlarge\",\n\t\"c5.2xlarge\",\n\t\"c5.4xlarge\",\n\t\"c5.9xlarge\",\n\t\"c5.12xlarge\",\n\t\"c5.18xlarge\",\n\t\"c5.24xlarge\",\n\t\"c5.metal\",\n\t\"c5a.large\",\n\t\"c5a.xlarge\",\n\t\"c5a.2xlarge\",\n\t\"c5a.4xlarge\",\n\t\"c5a.8xlarge\",\n\t\"c5a.12xlarge\",\n\t\"c5a.16xlarge\",\n\t\"c5a.24xlarge\",\n\t\"c5ad.large\",\n\t\"c5ad.xlarge\",\n\t\"c5ad.2xlarge\",\n\t\"c5ad.4xlarge\",\n\t\"c5ad.8xlarge\",\n\t\"c5ad.12xlarge\",\n\t\"c5ad.16xlarge\",\n\t\"c5ad.24xlarge\",\n\t\"c5d.large\",\n\t\"c5d.xlarge\",\n\t\"c5d.2xlarge\",\n\t\"c5d.4xlarge\",\n\t\"c5d.9xlarge\",\n\t\"c5d.12xlarge\",\n\t\"c5d.18xlarge\",\n\t\"c5d.24xlarge\",\n\t\"c5d.metal\",\n\t\"c5n.large\",\n\t\"c5n.xlarge\",\n\t\"c5n.2xlarge\",\n\t\"c5n.4xlarge\",\n\t\"c5n.9xlarge\",\n\t\"c5n.18xlarge\",\n\t\"c5n.metal\",\n\t\"c6a.large\",\n\t\"c6a.xlarge\",\n\t\"c6a.2xlarge\",\n\t\"c6a.4xlarge\",\n\t\"c6a.8xlarge\",\n\t\"c6a.12xlarge\",\n\t\"c6a.16xlarge\",\n\t\"c6a.24xlarge\",\n\t\"c6a.32xlarge\",\n\t\"c6a.48xlarge\",\n\t\"c6a.metal\",\n\t\"c6g.medium\",\n\t\"c6g.large\",\n\t\"c6g.xlarge\",\n\t\"c6g.2xlarge\",\n\t\"c6g.4xlarge\",\n\t\"c6g.8xlarge\",\n\t\"c6g.12xlarge\",\n\t\"c6g.16xlarge\",\n\t\"c6g.metal\",\n\t\"c6gd.medium\",\n\t\"c6gd.large\",\n\t\"c6gd.xlarge\",\n\t\"c6gd.2xlarge\",\n\t\"c6gd.4xlarge\",\n\t\"c6gd.8xlarge\",\n\t\"c6gd.12xlarge\",\n\t\"c6gd.16xlarge\",\n\t\"c6gd.metal\",\n\t\"c6gn.medium\",\n\t\"c6gn.large\",\n\t\"c6gn.xlarge\",\n\t\"c6gn.2xlarge\",\n\t\"c6gn.4xlarge\",\n\t\"c6gn.8xlarge\",\n\t\"c6gn.12xlarge\",\n\t\"c6gn.16xlarge\",\n\t\"c6gn.metal\",\n\t\"c6i.large\",\n\t\"c6i.xlarge\",\n\t\"c6i.2xlarge\",\n\t\"c6i.4xlarge\",\n\t\"c6i.8xlarge\",\n\t\"c6i.12xlarge\",\n\t\"c6i.16xlarge\",\n\t\"c6i.24xlarge\",\n\t\"c6i.32xlarge\",\n\t\"c6i.metal\",\n\t\"cc2.8xlarge\",\n\t\"cr1.8xlarge\",\n\t\"d2.xlarge\",\n\t\"d2.2xlarge\",\n\t\"d2.4xlarge\",\n\t\"d2.8xlarge\",\n\t\"d3.xlarge\",\n\t\"d3.2xlarge\",\n\t\"d3.4xlarge\",\n\t\"d3.8xlarge\",\n\t\"d3en.xlarge\",\n\t\"d3en.2xlarge\",\n\t\"d3en.4xlarge\",\n\t\"d3en.6xlarge\",\n\t\"d3en.8xlarge\",\n\t\"d3en.12xlarge\",\n\t\"dl1.24xlarge\",\n\t\"f1.2xlarge\",\n\t\"f1.4xlarge\",\n\t\"f1.16xlarge\",\n\t\"g2.2xlarge\",\n\t\"g2.8xlarge\",\n\t\"g3.4xlarge\",\n\t\"g3.8xlarge\",\n\t\"g3.16xlarge\",\n\t\"g3s.xlarge\",\n\t\"g4ad.xlarge\",\n\t\"g4ad.2xlarge\",\n\t\"g4ad.4xlarge\",\n\t\"g4ad.8xlarge\",\n\t\"g4ad.16xlarge\",\n\t\"g4dn.xlarge\",\n\t\"g4dn.2xlarge\",\n\t\"g4dn.4xlarge\",\n\t\"g4dn.8xlarge\",\n\t\"g4dn.12xlarge\",\n\t\"g4dn.16xlarge\",\n\t\"g4dn.metal\",\n\t\"g5.xlarge\",\n\t\"g5.2xlarge\",\n\t\"g5.4xlarge\",\n\t\"g5.8xlarge\",\n\t\"g5.12xlarge\",\n\t\"g5.16xlarge\",\n\t\"g5.24xlarge\",\n\t\"g5.48xlarge\",\n\t\"g5g.xlarge\",\n\t\"g5g.2xlarge\",\n\t\"g5g.4xlarge\",\n\t\"g5g.8xlarge\",\n\t\"g5g.16xlarge\",\n\t\"g5g.metal\",\n\t\"h1.2xlarge\",\n\t\"h1.4xlarge\",\n\t\"h1.8xlarge\",\n\t\"h1.16xlarge\",\n\t\"hpc6a.48xlarge\",\n\t\"hs1.8xlarge\",\n\t\"i2.xlarge\",\n\t\"i2.2xlarge\",\n\t\"i2.4xlarge\",\n\t\"i2.8xlarge\",\n\t\"i3.large\",\n\t\"i3.xlarge\",\n\t\"i3.2xlarge\",\n\t\"i3.4xlarge\",\n\t\"i3.8xlarge\",\n\t\"i3.16xlarge\",\n\t\"i3.metal\",\n\t\"i3en.large\",\n\t\"i3en.xlarge\",\n\t\"i3en.2xlarge\",\n\t\"i3en.3xlarge\",\n\t\"i3en.6xlarge\",\n\t\"i3en.12xlarge\",\n\t\"i3en.24xlarge\",\n\t\"i3en.metal\",\n\t\"i3p.16xlarge\",\n\t\"im4gn.large\",\n\t\"im4gn.xlarge\",\n\t\"im4gn.2xlarge\",\n\t\"im4gn.4xlarge\",\n\t\"im4gn.8xlarge\",\n\t\"im4gn.16xlarge\",\n\t\"inf1.xlarge\",\n\t\"inf1.2xlarge\",\n\t\"inf1.6xlarge\",\n\t\"inf1.24xlarge\",\n\t\"is4gen.medium\",\n\t\"is4gen.large\",\n\t\"is4gen.xlarge\",\n\t\"is4gen.2xlarge\",\n\t\"is4gen.4xlarge\",\n\t\"is4gen.8xlarge\",\n\t\"m1.small\",\n\t\"m1.medium\",\n\t\"m1.large\",\n\t\"m1.xlarge\",\n\t\"m2.xlarge\",\n\t\"m2.2xlarge\",\n\t\"m2.4xlarge\",\n\t\"m3.medium\",\n\t\"m3.large\",\n\t\"m3.xlarge\",\n\t\"m3.2xlarge\",\n\t\"m4.large\",\n\t\"m4.xlarge\",\n\t\"m4.2xlarge\",\n\t\"m4.4xlarge\",\n\t\"m4.10xlarge\",\n\t\"m4.16xlarge\",\n\t\"m5.large\",\n\t\"m5.xlarge\",\n\t\"m5.2xlarge\",\n\t\"m5.4xlarge\",\n\t\"m5.8xlarge\",\n\t\"m5.12xlarge\",\n\t\"m5.16xlarge\",\n\t\"m5.24xlarge\",\n\t\"m5.metal\",\n\t\"m5a.large\",\n\t\"m5a.xlarge\",\n\t\"m5a.2xlarge\",\n\t\"m5a.4xlarge\",\n\t\"m5a.8xlarge\",\n\t\"m5a.12xlarge\",\n\t\"m5a.16xlarge\",\n\t\"m5a.24xlarge\",\n\t\"m5ad.large\",\n\t\"m5ad.xlarge\",\n\t\"m5ad.2xlarge\",\n\t\"m5ad.4xlarge\",\n\t\"m5ad.8xlarge\",\n\t\"m5ad.12xlarge\",\n\t\"m5ad.16xlarge\",\n\t\"m5ad.24xlarge\",\n\t\"m5d.large\",\n\t\"m5d.xlarge\",\n\t\"m5d.2xlarge\",\n\t\"m5d.4xlarge\",\n\t\"m5d.8xlarge\",\n\t\"m5d.12xlarge\",\n\t\"m5d.16xlarge\",\n\t\"m5d.24xlarge\",\n\t\"m5d.metal\",\n\t\"m5dn.large\",\n\t\"m5dn.xlarge\",\n\t\"m5dn.2xlarge\",\n\t\"m5dn.4xlarge\",\n\t\"m5dn.8xlarge\",\n\t\"m5dn.12xlarge\",\n\t\"m5dn.16xlarge\",\n\t\"m5dn.24xlarge\",\n\t\"m5dn.metal\",\n\t\"m5n.large\",\n\t\"m5n.xlarge\",\n\t\"m5n.2xlarge\",\n\t\"m5n.4xlarge\",\n\t\"m5n.8xlarge\",\n\t\"m5n.12xlarge\",\n\t\"m5n.16xlarge\",\n\t\"m5n.24xlarge\",\n\t\"m5n.metal\",\n\t\"m5zn.large\",\n\t\"m5zn.xlarge\",\n\t\"m5zn.2xlarge\",\n\t\"m5zn.3xlarge\",\n\t\"m5zn.6xlarge\",\n\t\"m5zn.12xlarge\",\n\t\"m5zn.metal\",\n\t\"m6a.large\",\n\t\"m6a.xlarge\",\n\t\"m6a.2xlarge\",\n\t\"m6a.4xlarge\",\n\t\"m6a.8xlarge\",\n\t\"m6a.12xlarge\",\n\t\"m6a.16xlarge\",\n\t\"m6a.24xlarge\",\n\t\"m6a.32xlarge\",\n\t\"m6a.48xlarge\",\n\t\"m6a.metal\",\n\t\"m6g.medium\",\n\t\"m6g.large\",\n\t\"m6g.xlarge\",\n\t\"m6g.2xlarge\",\n\t\"m6g.4xlarge\",\n\t\"m6g.8xlarge\",\n\t\"m6g.12xlarge\",\n\t\"m6g.16xlarge\",\n\t\"m6g.metal\",\n\t\"m6gd.medium\",\n\t\"m6gd.large\",\n\t\"m6gd.xlarge\",\n\t\"m6gd.2xlarge\",\n\t\"m6gd.4xlarge\",\n\t\"m6gd.8xlarge\",\n\t\"m6gd.12xlarge\",\n\t\"m6gd.16xlarge\",\n\t\"m6gd.metal\",\n\t\"m6i.large\",\n\t\"m6i.xlarge\",\n\t\"m6i.2xlarge\",\n\t\"m6i.4xlarge\",\n\t\"m6i.8xlarge\",\n\t\"m6i.12xlarge\",\n\t\"m6i.16xlarge\",\n\t\"m6i.24xlarge\",\n\t\"m6i.32xlarge\",\n\t\"m6i.metal\",\n\t\"mac1.metal\",\n\t\"mac2.metal\",\n\t\"p2.xlarge\",\n\t\"p2.8xlarge\",\n\t\"p2.16xlarge\",\n\t\"p3.2xlarge\",\n\t\"p3.8xlarge\",\n\t\"p3.16xlarge\",\n\t\"p3dn.24xlarge\",\n\t\"p4d.24xlarge\",\n\t\"r3.large\",\n\t\"r3.xlarge\",\n\t\"r3.2xlarge\",\n\t\"r3.4xlarge\",\n\t\"r3.8xlarge\",\n\t\"r4.large\",\n\t\"r4.xlarge\",\n\t\"r4.2xlarge\",\n\t\"r4.4xlarge\",\n\t\"r4.8xlarge\",\n\t\"r4.16xlarge\",\n\t\"r5.large\",\n\t\"r5.xlarge\",\n\t\"r5.2xlarge\",\n\t\"r5.4xlarge\",\n\t\"r5.8xlarge\",\n\t\"r5.12xlarge\",\n\t\"r5.16xlarge\",\n\t\"r5.24xlarge\",\n\t\"r5.metal\",\n\t\"r5a.large\",\n\t\"r5a.xlarge\",\n\t\"r5a.2xlarge\",\n\t\"r5a.4xlarge\",\n\t\"r5a.8xlarge\",\n\t\"r5a.12xlarge\",\n\t\"r5a.16xlarge\",\n\t\"r5a.24xlarge\",\n\t\"r5ad.large\",\n\t\"r5ad.xlarge\",\n\t\"r5ad.2xlarge\",\n\t\"r5ad.4xlarge\",\n\t\"r5ad.8xlarge\",\n\t\"r5ad.12xlarge\",\n\t\"r5ad.16xlarge\",\n\t\"r5ad.24xlarge\",\n\t\"r5b.large\",\n\t\"r5b.xlarge\",\n\t\"r5b.2xlarge\",\n\t\"r5b.4xlarge\",\n\t\"r5b.8xlarge\",\n\t\"r5b.12xlarge\",\n\t\"r5b.16xlarge\",\n\t\"r5b.24xlarge\",\n\t\"r5b.metal\",\n\t\"r5d.large\",\n\t\"r5d.xlarge\",\n\t\"r5d.2xlarge\",\n\t\"r5d.4xlarge\",\n\t\"r5d.8xlarge\",\n\t\"r5d.12xlarge\",\n\t\"r5d.16xlarge\",\n\t\"r5d.24xlarge\",\n\t\"r5d.metal\",\n\t\"r5dn.large\",\n\t\"r5dn.xlarge\",\n\t\"r5dn.2xlarge\",\n\t\"r5dn.4xlarge\",\n\t\"r5dn.8xlarge\",\n\t\"r5dn.12xlarge\",\n\t\"r5dn.16xlarge\",\n\t\"r5dn.24xlarge\",\n\t\"r5dn.metal\",\n\t\"r5n.large\",\n\t\"r5n.xlarge\",\n\t\"r5n.2xlarge\",\n\t\"r5n.4xlarge\",\n\t\"r5n.8xlarge\",\n\t\"r5n.12xlarge\",\n\t\"r5n.16xlarge\",\n\t\"r5n.24xlarge\",\n\t\"r5n.metal\",\n\t\"r6g.medium\",\n\t\"r6g.large\",\n\t\"r6g.xlarge\",\n\t\"r6g.2xlarge\",\n\t\"r6g.4xlarge\",\n\t\"r6g.8xlarge\",\n\t\"r6g.12xlarge\",\n\t\"r6g.16xlarge\",\n\t\"r6g.metal\",\n\t\"r6gd.medium\",\n\t\"r6gd.large\",\n\t\"r6gd.xlarge\",\n\t\"r6gd.2xlarge\",\n\t\"r6gd.4xlarge\",\n\t\"r6gd.8xlarge\",\n\t\"r6gd.12xlarge\",\n\t\"r6gd.16xlarge\",\n\t\"r6gd.metal\",\n\t\"r6i.large\",\n\t\"r6i.xlarge\",\n\t\"r6i.2xlarge\",\n\t\"r6i.4xlarge\",\n\t\"r6i.8xlarge\",\n\t\"r6i.12xlarge\",\n\t\"r6i.16xlarge\",\n\t\"r6i.24xlarge\",\n\t\"r6i.32xlarge\",\n\t\"r6i.metal\",\n\t\"t1.micro\",\n\t\"t2.nano\",\n\t\"t2.micro\",\n\t\"t2.small\",\n\t\"t2.medium\",\n\t\"t2.large\",\n\t\"t2.xlarge\",\n\t\"t2.2xlarge\",\n\t\"t3.nano\",\n\t\"t3.micro\",\n\t\"t3.small\",\n\t\"t3.medium\",\n\t\"t3.large\",\n\t\"t3.xlarge\",\n\t\"t3.2xlarge\",\n\t\"t3a.nano\",\n\t\"t3a.micro\",\n\t\"t3a.small\",\n\t\"t3a.medium\",\n\t\"t3a.large\",\n\t\"t3a.xlarge\",\n\t\"t3a.2xlarge\",\n\t\"t4g.nano\",\n\t\"t4g.micro\",\n\t\"t4g.small\",\n\t\"t4g.medium\",\n\t\"t4g.large\",\n\t\"t4g.xlarge\",\n\t\"t4g.2xlarge\",\n\t\"u-12tb1.112xlarge\",\n\t\"u-12tb1.metal\",\n\t\"u-18tb1.metal\",\n\t\"u-24tb1.metal\",\n\t\"u-3tb1.56xlarge\",\n\t\"u-6tb1.56xlarge\",\n\t\"u-6tb1.112xlarge\",\n\t\"u-6tb1.metal\",\n\t\"u-9tb1.112xlarge\",\n\t\"u-9tb1.metal\",\n\t\"vt1.3xlarge\",\n\t\"vt1.6xlarge\",\n\t\"vt1.24xlarge\",\n\t\"x1.16xlarge\",\n\t\"x1.32xlarge\",\n\t\"x1e.xlarge\",\n\t\"x1e.2xlarge\",\n\t\"x1e.4xlarge\",\n\t\"x1e.8xlarge\",\n\t\"x1e.16xlarge\",\n\t\"x1e.32xlarge\",\n\t\"x2gd.medium\",\n\t\"x2gd.large\",\n\t\"x2gd.xlarge\",\n\t\"x2gd.2xlarge\",\n\t\"x2gd.4xlarge\",\n\t\"x2gd.8xlarge\",\n\t\"x2gd.12xlarge\",\n\t\"x2gd.16xlarge\",\n\t\"x2gd.metal\",\n\t\"x2idn.16xlarge\",\n\t\"x2idn.24xlarge\",\n\t\"x2idn.32xlarge\",\n\t\"x2iedn.xlarge\",\n\t\"x2iedn.2xlarge\",\n\t\"x2iedn.4xlarge\",\n\t\"x2iedn.8xlarge\",\n\t\"x2iedn.16xlarge\",\n\t\"x2iedn.24xlarge\",\n\t\"x2iedn.32xlarge\",\n\t\"x2iezn.2xlarge\",\n\t\"x2iezn.4xlarge\",\n\t\"x2iezn.6xlarge\",\n\t\"x2iezn.8xlarge\",\n\t\"x2iezn.12xlarge\",\n\t\"x2iezn.metal\",\n\t\"z1d.large\",\n\t\"z1d.xlarge\",\n\t\"z1d.2xlarge\",\n\t\"z1d.3xlarge\",\n\t\"z1d.6xlarge\",\n\t\"z1d.12xlarge\",\n\t\"z1d.metal\",\n)\n\n// This contains all instance types available in each region. Metadata is not available for all of them.\nvar InstanceTypes = map[string]strset.Set{\n\t\"af-south-1\": strset.New(\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t),\n\t\"ap-east-1\": strset.New(\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t),\n\t\"ap-northeast-1\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6gn.metal\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"cc2.8xlarge\",\n\t\t\"cr1.8xlarge\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4ad.xlarge\",\n\t\t\"g4ad.2xlarge\",\n\t\t\"g4ad.4xlarge\",\n\t\t\"g4ad.8xlarge\",\n\t\t\"g4ad.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"g5g.xlarge\",\n\t\t\"g5g.2xlarge\",\n\t\t\"g5g.4xlarge\",\n\t\t\"g5g.8xlarge\",\n\t\t\"g5g.16xlarge\",\n\t\t\"g5g.metal\",\n\t\t\"hs1.8xlarge\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"p3dn.24xlarge\",\n\t\t\"p4d.24xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.metal\",\n\t\t\"vt1.3xlarge\",\n\t\t\"vt1.6xlarge\",\n\t\t\"vt1.24xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"x2idn.16xlarge\",\n\t\t\"x2idn.24xlarge\",\n\t\t\"x2idn.32xlarge\",\n\t\t\"x2iedn.xlarge\",\n\t\t\"x2iedn.2xlarge\",\n\t\t\"x2iedn.4xlarge\",\n\t\t\"x2iedn.8xlarge\",\n\t\t\"x2iedn.16xlarge\",\n\t\t\"x2iedn.24xlarge\",\n\t\t\"x2iedn.32xlarge\",\n\t\t\"x2iezn.2xlarge\",\n\t\t\"x2iezn.4xlarge\",\n\t\t\"x2iezn.6xlarge\",\n\t\t\"x2iezn.8xlarge\",\n\t\t\"x2iezn.12xlarge\",\n\t\t\"x2iezn.metal\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"ap-northeast-2\": strset.New(\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"g5g.xlarge\",\n\t\t\"g5g.2xlarge\",\n\t\t\"g5g.4xlarge\",\n\t\t\"g5g.8xlarge\",\n\t\t\"g5g.16xlarge\",\n\t\t\"g5g.metal\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"p4d.24xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"ap-northeast-3\": strset.New(\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t),\n\t\"ap-south-1\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m6a.large\",\n\t\t\"m6a.xlarge\",\n\t\t\"m6a.2xlarge\",\n\t\t\"m6a.4xlarge\",\n\t\t\"m6a.8xlarge\",\n\t\t\"m6a.12xlarge\",\n\t\t\"m6a.16xlarge\",\n\t\t\"m6a.24xlarge\",\n\t\t\"m6a.32xlarge\",\n\t\t\"m6a.48xlarge\",\n\t\t\"m6a.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"x2idn.16xlarge\",\n\t\t\"x2idn.24xlarge\",\n\t\t\"x2idn.32xlarge\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"ap-southeast-1\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"g5g.xlarge\",\n\t\t\"g5g.2xlarge\",\n\t\t\"g5g.4xlarge\",\n\t\t\"g5g.8xlarge\",\n\t\t\"g5g.16xlarge\",\n\t\t\"g5g.metal\",\n\t\t\"hs1.8xlarge\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.112xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.112xlarge\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"x2idn.16xlarge\",\n\t\t\"x2idn.24xlarge\",\n\t\t\"x2idn.32xlarge\",\n\t\t\"x2iedn.xlarge\",\n\t\t\"x2iedn.2xlarge\",\n\t\t\"x2iedn.4xlarge\",\n\t\t\"x2iedn.8xlarge\",\n\t\t\"x2iedn.16xlarge\",\n\t\t\"x2iedn.24xlarge\",\n\t\t\"x2iedn.32xlarge\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"ap-southeast-2\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"f1.2xlarge\",\n\t\t\"f1.4xlarge\",\n\t\t\"f1.16xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"hs1.8xlarge\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"ca-central-1\": strset.New(\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g4ad.xlarge\",\n\t\t\"g4ad.2xlarge\",\n\t\t\"g4ad.4xlarge\",\n\t\t\"g4ad.8xlarge\",\n\t\t\"g4ad.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t),\n\t\"eu-central-1\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"f1.2xlarge\",\n\t\t\"f1.4xlarge\",\n\t\t\"f1.16xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4ad.xlarge\",\n\t\t\"g4ad.2xlarge\",\n\t\t\"g4ad.4xlarge\",\n\t\t\"g4ad.8xlarge\",\n\t\t\"g4ad.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"p4d.24xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.112xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-3tb1.56xlarge\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.112xlarge\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"x2idn.16xlarge\",\n\t\t\"x2idn.24xlarge\",\n\t\t\"x2idn.32xlarge\",\n\t\t\"x2iedn.xlarge\",\n\t\t\"x2iedn.2xlarge\",\n\t\t\"x2iedn.4xlarge\",\n\t\t\"x2iedn.8xlarge\",\n\t\t\"x2iedn.16xlarge\",\n\t\t\"x2iedn.24xlarge\",\n\t\t\"x2iedn.32xlarge\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"eu-north-1\": strset.New(\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"mac1.metal\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t),\n\t\"eu-south-1\": strset.New(\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-3tb1.56xlarge\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t),\n\t\"eu-west-1\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6a.large\",\n\t\t\"c6a.xlarge\",\n\t\t\"c6a.2xlarge\",\n\t\t\"c6a.4xlarge\",\n\t\t\"c6a.8xlarge\",\n\t\t\"c6a.12xlarge\",\n\t\t\"c6a.16xlarge\",\n\t\t\"c6a.24xlarge\",\n\t\t\"c6a.32xlarge\",\n\t\t\"c6a.48xlarge\",\n\t\t\"c6a.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6gn.metal\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"cc2.8xlarge\",\n\t\t\"cr1.8xlarge\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"d3en.xlarge\",\n\t\t\"d3en.2xlarge\",\n\t\t\"d3en.4xlarge\",\n\t\t\"d3en.6xlarge\",\n\t\t\"d3en.8xlarge\",\n\t\t\"d3en.12xlarge\",\n\t\t\"f1.2xlarge\",\n\t\t\"f1.4xlarge\",\n\t\t\"f1.16xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4ad.xlarge\",\n\t\t\"g4ad.2xlarge\",\n\t\t\"g4ad.4xlarge\",\n\t\t\"g4ad.8xlarge\",\n\t\t\"g4ad.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"g5.xlarge\",\n\t\t\"g5.2xlarge\",\n\t\t\"g5.4xlarge\",\n\t\t\"g5.8xlarge\",\n\t\t\"g5.12xlarge\",\n\t\t\"g5.16xlarge\",\n\t\t\"g5.24xlarge\",\n\t\t\"g5.48xlarge\",\n\t\t\"h1.2xlarge\",\n\t\t\"h1.4xlarge\",\n\t\t\"h1.8xlarge\",\n\t\t\"h1.16xlarge\",\n\t\t\"hs1.8xlarge\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"im4gn.large\",\n\t\t\"im4gn.xlarge\",\n\t\t\"im4gn.2xlarge\",\n\t\t\"im4gn.4xlarge\",\n\t\t\"im4gn.8xlarge\",\n\t\t\"im4gn.16xlarge\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"is4gen.medium\",\n\t\t\"is4gen.large\",\n\t\t\"is4gen.xlarge\",\n\t\t\"is4gen.2xlarge\",\n\t\t\"is4gen.4xlarge\",\n\t\t\"is4gen.8xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6a.large\",\n\t\t\"m6a.xlarge\",\n\t\t\"m6a.2xlarge\",\n\t\t\"m6a.4xlarge\",\n\t\t\"m6a.8xlarge\",\n\t\t\"m6a.12xlarge\",\n\t\t\"m6a.16xlarge\",\n\t\t\"m6a.24xlarge\",\n\t\t\"m6a.32xlarge\",\n\t\t\"m6a.48xlarge\",\n\t\t\"m6a.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"p3dn.24xlarge\",\n\t\t\"p4d.24xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.112xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-18tb1.metal\",\n\t\t\"u-24tb1.metal\",\n\t\t\"u-3tb1.56xlarge\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.112xlarge\",\n\t\t\"u-9tb1.metal\",\n\t\t\"vt1.3xlarge\",\n\t\t\"vt1.6xlarge\",\n\t\t\"vt1.24xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"x2gd.medium\",\n\t\t\"x2gd.large\",\n\t\t\"x2gd.xlarge\",\n\t\t\"x2gd.2xlarge\",\n\t\t\"x2gd.4xlarge\",\n\t\t\"x2gd.8xlarge\",\n\t\t\"x2gd.12xlarge\",\n\t\t\"x2gd.16xlarge\",\n\t\t\"x2gd.metal\",\n\t\t\"x2idn.16xlarge\",\n\t\t\"x2idn.24xlarge\",\n\t\t\"x2idn.32xlarge\",\n\t\t\"x2iedn.xlarge\",\n\t\t\"x2iedn.2xlarge\",\n\t\t\"x2iedn.4xlarge\",\n\t\t\"x2iedn.8xlarge\",\n\t\t\"x2iedn.16xlarge\",\n\t\t\"x2iedn.24xlarge\",\n\t\t\"x2iedn.32xlarge\",\n\t\t\"x2iezn.2xlarge\",\n\t\t\"x2iezn.4xlarge\",\n\t\t\"x2iezn.6xlarge\",\n\t\t\"x2iezn.8xlarge\",\n\t\t\"x2iezn.12xlarge\",\n\t\t\"x2iezn.metal\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"eu-west-2\": strset.New(\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"f1.2xlarge\",\n\t\t\"f1.4xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4ad.xlarge\",\n\t\t\"g4ad.2xlarge\",\n\t\t\"g4ad.4xlarge\",\n\t\t\"g4ad.8xlarge\",\n\t\t\"g4ad.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"eu-west-3\": strset.New(\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t),\n\t\"me-south-1\": strset.New(\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t),\n\t\"sa-east-1\": strset.New(\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t),\n\t\"us-east-1\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6a.large\",\n\t\t\"c6a.xlarge\",\n\t\t\"c6a.2xlarge\",\n\t\t\"c6a.4xlarge\",\n\t\t\"c6a.8xlarge\",\n\t\t\"c6a.12xlarge\",\n\t\t\"c6a.16xlarge\",\n\t\t\"c6a.24xlarge\",\n\t\t\"c6a.32xlarge\",\n\t\t\"c6a.48xlarge\",\n\t\t\"c6a.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6gn.metal\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"cc2.8xlarge\",\n\t\t\"cr1.8xlarge\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"d3en.xlarge\",\n\t\t\"d3en.2xlarge\",\n\t\t\"d3en.4xlarge\",\n\t\t\"d3en.6xlarge\",\n\t\t\"d3en.8xlarge\",\n\t\t\"d3en.12xlarge\",\n\t\t\"dl1.24xlarge\",\n\t\t\"f1.2xlarge\",\n\t\t\"f1.4xlarge\",\n\t\t\"f1.16xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4ad.xlarge\",\n\t\t\"g4ad.2xlarge\",\n\t\t\"g4ad.4xlarge\",\n\t\t\"g4ad.8xlarge\",\n\t\t\"g4ad.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"g5.xlarge\",\n\t\t\"g5.2xlarge\",\n\t\t\"g5.4xlarge\",\n\t\t\"g5.8xlarge\",\n\t\t\"g5.12xlarge\",\n\t\t\"g5.16xlarge\",\n\t\t\"g5.24xlarge\",\n\t\t\"g5.48xlarge\",\n\t\t\"g5g.xlarge\",\n\t\t\"g5g.2xlarge\",\n\t\t\"g5g.4xlarge\",\n\t\t\"g5g.8xlarge\",\n\t\t\"g5g.16xlarge\",\n\t\t\"g5g.metal\",\n\t\t\"h1.2xlarge\",\n\t\t\"h1.4xlarge\",\n\t\t\"h1.8xlarge\",\n\t\t\"h1.16xlarge\",\n\t\t\"hs1.8xlarge\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"im4gn.large\",\n\t\t\"im4gn.xlarge\",\n\t\t\"im4gn.2xlarge\",\n\t\t\"im4gn.4xlarge\",\n\t\t\"im4gn.8xlarge\",\n\t\t\"im4gn.16xlarge\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"is4gen.medium\",\n\t\t\"is4gen.large\",\n\t\t\"is4gen.xlarge\",\n\t\t\"is4gen.2xlarge\",\n\t\t\"is4gen.4xlarge\",\n\t\t\"is4gen.8xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6a.large\",\n\t\t\"m6a.xlarge\",\n\t\t\"m6a.2xlarge\",\n\t\t\"m6a.4xlarge\",\n\t\t\"m6a.8xlarge\",\n\t\t\"m6a.12xlarge\",\n\t\t\"m6a.16xlarge\",\n\t\t\"m6a.24xlarge\",\n\t\t\"m6a.32xlarge\",\n\t\t\"m6a.48xlarge\",\n\t\t\"m6a.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"mac2.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"p3dn.24xlarge\",\n\t\t\"p4d.24xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.112xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-18tb1.metal\",\n\t\t\"u-24tb1.metal\",\n\t\t\"u-3tb1.56xlarge\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.112xlarge\",\n\t\t\"u-9tb1.metal\",\n\t\t\"vt1.3xlarge\",\n\t\t\"vt1.6xlarge\",\n\t\t\"vt1.24xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"x2gd.medium\",\n\t\t\"x2gd.large\",\n\t\t\"x2gd.xlarge\",\n\t\t\"x2gd.2xlarge\",\n\t\t\"x2gd.4xlarge\",\n\t\t\"x2gd.8xlarge\",\n\t\t\"x2gd.12xlarge\",\n\t\t\"x2gd.16xlarge\",\n\t\t\"x2gd.metal\",\n\t\t\"x2idn.16xlarge\",\n\t\t\"x2idn.24xlarge\",\n\t\t\"x2idn.32xlarge\",\n\t\t\"x2iedn.xlarge\",\n\t\t\"x2iedn.2xlarge\",\n\t\t\"x2iedn.4xlarge\",\n\t\t\"x2iedn.8xlarge\",\n\t\t\"x2iedn.16xlarge\",\n\t\t\"x2iedn.24xlarge\",\n\t\t\"x2iedn.32xlarge\",\n\t\t\"x2iezn.2xlarge\",\n\t\t\"x2iezn.4xlarge\",\n\t\t\"x2iezn.6xlarge\",\n\t\t\"x2iezn.8xlarge\",\n\t\t\"x2iezn.12xlarge\",\n\t\t\"x2iezn.metal\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"us-east-2\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6gn.metal\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4ad.xlarge\",\n\t\t\"g4ad.2xlarge\",\n\t\t\"g4ad.4xlarge\",\n\t\t\"g4ad.8xlarge\",\n\t\t\"g4ad.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"h1.2xlarge\",\n\t\t\"h1.4xlarge\",\n\t\t\"h1.8xlarge\",\n\t\t\"h1.16xlarge\",\n\t\t\"hpc6a.48xlarge\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"im4gn.large\",\n\t\t\"im4gn.xlarge\",\n\t\t\"im4gn.2xlarge\",\n\t\t\"im4gn.4xlarge\",\n\t\t\"im4gn.8xlarge\",\n\t\t\"im4gn.16xlarge\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"is4gen.medium\",\n\t\t\"is4gen.large\",\n\t\t\"is4gen.xlarge\",\n\t\t\"is4gen.2xlarge\",\n\t\t\"is4gen.4xlarge\",\n\t\t\"is4gen.8xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"p4d.24xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"x2gd.medium\",\n\t\t\"x2gd.large\",\n\t\t\"x2gd.xlarge\",\n\t\t\"x2gd.2xlarge\",\n\t\t\"x2gd.4xlarge\",\n\t\t\"x2gd.8xlarge\",\n\t\t\"x2gd.12xlarge\",\n\t\t\"x2gd.16xlarge\",\n\t\t\"x2gd.metal\",\n\t\t\"x2iedn.xlarge\",\n\t\t\"x2iedn.2xlarge\",\n\t\t\"x2iedn.4xlarge\",\n\t\t\"x2iedn.8xlarge\",\n\t\t\"x2iedn.16xlarge\",\n\t\t\"x2iedn.24xlarge\",\n\t\t\"x2iedn.32xlarge\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"us-gov-east-1\": strset.New(\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"p3dn.24xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-18tb1.metal\",\n\t\t\"u-24tb1.metal\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t),\n\t\"us-gov-west-1\": strset.New(\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"cc2.8xlarge\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"f1.2xlarge\",\n\t\t\"f1.4xlarge\",\n\t\t\"f1.16xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"hpc6a.48xlarge\",\n\t\t\"hs1.8xlarge\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"i3p.16xlarge\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"p3dn.24xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.112xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-18tb1.metal\",\n\t\t\"u-24tb1.metal\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.112xlarge\",\n\t\t\"u-9tb1.metal\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t),\n\t\"us-west-1\": strset.New(\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n\t\"us-west-2\": strset.New(\n\t\t\"a1.medium\",\n\t\t\"a1.large\",\n\t\t\"a1.xlarge\",\n\t\t\"a1.2xlarge\",\n\t\t\"a1.4xlarge\",\n\t\t\"a1.metal\",\n\t\t\"c1.medium\",\n\t\t\"c1.xlarge\",\n\t\t\"c3.large\",\n\t\t\"c3.xlarge\",\n\t\t\"c3.2xlarge\",\n\t\t\"c3.4xlarge\",\n\t\t\"c3.8xlarge\",\n\t\t\"c4.large\",\n\t\t\"c4.xlarge\",\n\t\t\"c4.2xlarge\",\n\t\t\"c4.4xlarge\",\n\t\t\"c4.8xlarge\",\n\t\t\"c5.large\",\n\t\t\"c5.xlarge\",\n\t\t\"c5.2xlarge\",\n\t\t\"c5.4xlarge\",\n\t\t\"c5.9xlarge\",\n\t\t\"c5.12xlarge\",\n\t\t\"c5.18xlarge\",\n\t\t\"c5.24xlarge\",\n\t\t\"c5.metal\",\n\t\t\"c5a.large\",\n\t\t\"c5a.xlarge\",\n\t\t\"c5a.2xlarge\",\n\t\t\"c5a.4xlarge\",\n\t\t\"c5a.8xlarge\",\n\t\t\"c5a.12xlarge\",\n\t\t\"c5a.16xlarge\",\n\t\t\"c5a.24xlarge\",\n\t\t\"c5ad.large\",\n\t\t\"c5ad.xlarge\",\n\t\t\"c5ad.2xlarge\",\n\t\t\"c5ad.4xlarge\",\n\t\t\"c5ad.8xlarge\",\n\t\t\"c5ad.12xlarge\",\n\t\t\"c5ad.16xlarge\",\n\t\t\"c5ad.24xlarge\",\n\t\t\"c5d.large\",\n\t\t\"c5d.xlarge\",\n\t\t\"c5d.2xlarge\",\n\t\t\"c5d.4xlarge\",\n\t\t\"c5d.9xlarge\",\n\t\t\"c5d.12xlarge\",\n\t\t\"c5d.18xlarge\",\n\t\t\"c5d.24xlarge\",\n\t\t\"c5d.metal\",\n\t\t\"c5n.large\",\n\t\t\"c5n.xlarge\",\n\t\t\"c5n.2xlarge\",\n\t\t\"c5n.4xlarge\",\n\t\t\"c5n.9xlarge\",\n\t\t\"c5n.18xlarge\",\n\t\t\"c5n.metal\",\n\t\t\"c6a.large\",\n\t\t\"c6a.xlarge\",\n\t\t\"c6a.2xlarge\",\n\t\t\"c6a.4xlarge\",\n\t\t\"c6a.8xlarge\",\n\t\t\"c6a.12xlarge\",\n\t\t\"c6a.16xlarge\",\n\t\t\"c6a.24xlarge\",\n\t\t\"c6a.32xlarge\",\n\t\t\"c6a.48xlarge\",\n\t\t\"c6a.metal\",\n\t\t\"c6g.medium\",\n\t\t\"c6g.large\",\n\t\t\"c6g.xlarge\",\n\t\t\"c6g.2xlarge\",\n\t\t\"c6g.4xlarge\",\n\t\t\"c6g.8xlarge\",\n\t\t\"c6g.12xlarge\",\n\t\t\"c6g.16xlarge\",\n\t\t\"c6g.metal\",\n\t\t\"c6gd.medium\",\n\t\t\"c6gd.large\",\n\t\t\"c6gd.xlarge\",\n\t\t\"c6gd.2xlarge\",\n\t\t\"c6gd.4xlarge\",\n\t\t\"c6gd.8xlarge\",\n\t\t\"c6gd.12xlarge\",\n\t\t\"c6gd.16xlarge\",\n\t\t\"c6gd.metal\",\n\t\t\"c6gn.medium\",\n\t\t\"c6gn.large\",\n\t\t\"c6gn.xlarge\",\n\t\t\"c6gn.2xlarge\",\n\t\t\"c6gn.4xlarge\",\n\t\t\"c6gn.8xlarge\",\n\t\t\"c6gn.12xlarge\",\n\t\t\"c6gn.16xlarge\",\n\t\t\"c6gn.metal\",\n\t\t\"c6i.large\",\n\t\t\"c6i.xlarge\",\n\t\t\"c6i.2xlarge\",\n\t\t\"c6i.4xlarge\",\n\t\t\"c6i.8xlarge\",\n\t\t\"c6i.12xlarge\",\n\t\t\"c6i.16xlarge\",\n\t\t\"c6i.24xlarge\",\n\t\t\"c6i.32xlarge\",\n\t\t\"c6i.metal\",\n\t\t\"cc2.8xlarge\",\n\t\t\"cr1.8xlarge\",\n\t\t\"d2.xlarge\",\n\t\t\"d2.2xlarge\",\n\t\t\"d2.4xlarge\",\n\t\t\"d2.8xlarge\",\n\t\t\"d3.xlarge\",\n\t\t\"d3.2xlarge\",\n\t\t\"d3.4xlarge\",\n\t\t\"d3.8xlarge\",\n\t\t\"d3en.xlarge\",\n\t\t\"d3en.2xlarge\",\n\t\t\"d3en.4xlarge\",\n\t\t\"d3en.6xlarge\",\n\t\t\"d3en.8xlarge\",\n\t\t\"d3en.12xlarge\",\n\t\t\"dl1.24xlarge\",\n\t\t\"f1.2xlarge\",\n\t\t\"f1.4xlarge\",\n\t\t\"f1.16xlarge\",\n\t\t\"g2.2xlarge\",\n\t\t\"g2.8xlarge\",\n\t\t\"g3.4xlarge\",\n\t\t\"g3.8xlarge\",\n\t\t\"g3.16xlarge\",\n\t\t\"g3s.xlarge\",\n\t\t\"g4ad.xlarge\",\n\t\t\"g4ad.2xlarge\",\n\t\t\"g4ad.4xlarge\",\n\t\t\"g4ad.8xlarge\",\n\t\t\"g4ad.16xlarge\",\n\t\t\"g4dn.xlarge\",\n\t\t\"g4dn.2xlarge\",\n\t\t\"g4dn.4xlarge\",\n\t\t\"g4dn.8xlarge\",\n\t\t\"g4dn.12xlarge\",\n\t\t\"g4dn.16xlarge\",\n\t\t\"g4dn.metal\",\n\t\t\"g5.xlarge\",\n\t\t\"g5.2xlarge\",\n\t\t\"g5.4xlarge\",\n\t\t\"g5.8xlarge\",\n\t\t\"g5.12xlarge\",\n\t\t\"g5.16xlarge\",\n\t\t\"g5.24xlarge\",\n\t\t\"g5.48xlarge\",\n\t\t\"g5g.xlarge\",\n\t\t\"g5g.2xlarge\",\n\t\t\"g5g.4xlarge\",\n\t\t\"g5g.8xlarge\",\n\t\t\"g5g.16xlarge\",\n\t\t\"g5g.metal\",\n\t\t\"h1.2xlarge\",\n\t\t\"h1.4xlarge\",\n\t\t\"h1.8xlarge\",\n\t\t\"h1.16xlarge\",\n\t\t\"hs1.8xlarge\",\n\t\t\"i2.xlarge\",\n\t\t\"i2.2xlarge\",\n\t\t\"i2.4xlarge\",\n\t\t\"i2.8xlarge\",\n\t\t\"i3.large\",\n\t\t\"i3.xlarge\",\n\t\t\"i3.2xlarge\",\n\t\t\"i3.4xlarge\",\n\t\t\"i3.8xlarge\",\n\t\t\"i3.16xlarge\",\n\t\t\"i3.metal\",\n\t\t\"i3en.large\",\n\t\t\"i3en.xlarge\",\n\t\t\"i3en.2xlarge\",\n\t\t\"i3en.3xlarge\",\n\t\t\"i3en.6xlarge\",\n\t\t\"i3en.12xlarge\",\n\t\t\"i3en.24xlarge\",\n\t\t\"i3en.metal\",\n\t\t\"im4gn.large\",\n\t\t\"im4gn.xlarge\",\n\t\t\"im4gn.2xlarge\",\n\t\t\"im4gn.4xlarge\",\n\t\t\"im4gn.8xlarge\",\n\t\t\"im4gn.16xlarge\",\n\t\t\"inf1.xlarge\",\n\t\t\"inf1.2xlarge\",\n\t\t\"inf1.6xlarge\",\n\t\t\"inf1.24xlarge\",\n\t\t\"is4gen.medium\",\n\t\t\"is4gen.large\",\n\t\t\"is4gen.xlarge\",\n\t\t\"is4gen.2xlarge\",\n\t\t\"is4gen.4xlarge\",\n\t\t\"is4gen.8xlarge\",\n\t\t\"m1.small\",\n\t\t\"m1.medium\",\n\t\t\"m1.large\",\n\t\t\"m1.xlarge\",\n\t\t\"m2.xlarge\",\n\t\t\"m2.2xlarge\",\n\t\t\"m2.4xlarge\",\n\t\t\"m3.medium\",\n\t\t\"m3.large\",\n\t\t\"m3.xlarge\",\n\t\t\"m3.2xlarge\",\n\t\t\"m4.large\",\n\t\t\"m4.xlarge\",\n\t\t\"m4.2xlarge\",\n\t\t\"m4.4xlarge\",\n\t\t\"m4.10xlarge\",\n\t\t\"m4.16xlarge\",\n\t\t\"m5.large\",\n\t\t\"m5.xlarge\",\n\t\t\"m5.2xlarge\",\n\t\t\"m5.4xlarge\",\n\t\t\"m5.8xlarge\",\n\t\t\"m5.12xlarge\",\n\t\t\"m5.16xlarge\",\n\t\t\"m5.24xlarge\",\n\t\t\"m5.metal\",\n\t\t\"m5a.large\",\n\t\t\"m5a.xlarge\",\n\t\t\"m5a.2xlarge\",\n\t\t\"m5a.4xlarge\",\n\t\t\"m5a.8xlarge\",\n\t\t\"m5a.12xlarge\",\n\t\t\"m5a.16xlarge\",\n\t\t\"m5a.24xlarge\",\n\t\t\"m5ad.large\",\n\t\t\"m5ad.xlarge\",\n\t\t\"m5ad.2xlarge\",\n\t\t\"m5ad.4xlarge\",\n\t\t\"m5ad.8xlarge\",\n\t\t\"m5ad.12xlarge\",\n\t\t\"m5ad.16xlarge\",\n\t\t\"m5ad.24xlarge\",\n\t\t\"m5d.large\",\n\t\t\"m5d.xlarge\",\n\t\t\"m5d.2xlarge\",\n\t\t\"m5d.4xlarge\",\n\t\t\"m5d.8xlarge\",\n\t\t\"m5d.12xlarge\",\n\t\t\"m5d.16xlarge\",\n\t\t\"m5d.24xlarge\",\n\t\t\"m5d.metal\",\n\t\t\"m5dn.large\",\n\t\t\"m5dn.xlarge\",\n\t\t\"m5dn.2xlarge\",\n\t\t\"m5dn.4xlarge\",\n\t\t\"m5dn.8xlarge\",\n\t\t\"m5dn.12xlarge\",\n\t\t\"m5dn.16xlarge\",\n\t\t\"m5dn.24xlarge\",\n\t\t\"m5dn.metal\",\n\t\t\"m5n.large\",\n\t\t\"m5n.xlarge\",\n\t\t\"m5n.2xlarge\",\n\t\t\"m5n.4xlarge\",\n\t\t\"m5n.8xlarge\",\n\t\t\"m5n.12xlarge\",\n\t\t\"m5n.16xlarge\",\n\t\t\"m5n.24xlarge\",\n\t\t\"m5n.metal\",\n\t\t\"m5zn.large\",\n\t\t\"m5zn.xlarge\",\n\t\t\"m5zn.2xlarge\",\n\t\t\"m5zn.3xlarge\",\n\t\t\"m5zn.6xlarge\",\n\t\t\"m5zn.12xlarge\",\n\t\t\"m5zn.metal\",\n\t\t\"m6a.large\",\n\t\t\"m6a.xlarge\",\n\t\t\"m6a.2xlarge\",\n\t\t\"m6a.4xlarge\",\n\t\t\"m6a.8xlarge\",\n\t\t\"m6a.12xlarge\",\n\t\t\"m6a.16xlarge\",\n\t\t\"m6a.24xlarge\",\n\t\t\"m6a.32xlarge\",\n\t\t\"m6a.48xlarge\",\n\t\t\"m6a.metal\",\n\t\t\"m6g.medium\",\n\t\t\"m6g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.2xlarge\",\n\t\t\"m6g.4xlarge\",\n\t\t\"m6g.8xlarge\",\n\t\t\"m6g.12xlarge\",\n\t\t\"m6g.16xlarge\",\n\t\t\"m6g.metal\",\n\t\t\"m6gd.medium\",\n\t\t\"m6gd.large\",\n\t\t\"m6gd.xlarge\",\n\t\t\"m6gd.2xlarge\",\n\t\t\"m6gd.4xlarge\",\n\t\t\"m6gd.8xlarge\",\n\t\t\"m6gd.12xlarge\",\n\t\t\"m6gd.16xlarge\",\n\t\t\"m6gd.metal\",\n\t\t\"m6i.large\",\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.2xlarge\",\n\t\t\"m6i.4xlarge\",\n\t\t\"m6i.8xlarge\",\n\t\t\"m6i.12xlarge\",\n\t\t\"m6i.16xlarge\",\n\t\t\"m6i.24xlarge\",\n\t\t\"m6i.32xlarge\",\n\t\t\"m6i.metal\",\n\t\t\"mac1.metal\",\n\t\t\"mac2.metal\",\n\t\t\"p2.xlarge\",\n\t\t\"p2.8xlarge\",\n\t\t\"p2.16xlarge\",\n\t\t\"p3.2xlarge\",\n\t\t\"p3.8xlarge\",\n\t\t\"p3.16xlarge\",\n\t\t\"p3dn.24xlarge\",\n\t\t\"p4d.24xlarge\",\n\t\t\"r3.large\",\n\t\t\"r3.xlarge\",\n\t\t\"r3.2xlarge\",\n\t\t\"r3.4xlarge\",\n\t\t\"r3.8xlarge\",\n\t\t\"r4.large\",\n\t\t\"r4.xlarge\",\n\t\t\"r4.2xlarge\",\n\t\t\"r4.4xlarge\",\n\t\t\"r4.8xlarge\",\n\t\t\"r4.16xlarge\",\n\t\t\"r5.large\",\n\t\t\"r5.xlarge\",\n\t\t\"r5.2xlarge\",\n\t\t\"r5.4xlarge\",\n\t\t\"r5.8xlarge\",\n\t\t\"r5.12xlarge\",\n\t\t\"r5.16xlarge\",\n\t\t\"r5.24xlarge\",\n\t\t\"r5.metal\",\n\t\t\"r5a.large\",\n\t\t\"r5a.xlarge\",\n\t\t\"r5a.2xlarge\",\n\t\t\"r5a.4xlarge\",\n\t\t\"r5a.8xlarge\",\n\t\t\"r5a.12xlarge\",\n\t\t\"r5a.16xlarge\",\n\t\t\"r5a.24xlarge\",\n\t\t\"r5ad.large\",\n\t\t\"r5ad.xlarge\",\n\t\t\"r5ad.2xlarge\",\n\t\t\"r5ad.4xlarge\",\n\t\t\"r5ad.8xlarge\",\n\t\t\"r5ad.12xlarge\",\n\t\t\"r5ad.16xlarge\",\n\t\t\"r5ad.24xlarge\",\n\t\t\"r5b.large\",\n\t\t\"r5b.xlarge\",\n\t\t\"r5b.2xlarge\",\n\t\t\"r5b.4xlarge\",\n\t\t\"r5b.8xlarge\",\n\t\t\"r5b.12xlarge\",\n\t\t\"r5b.16xlarge\",\n\t\t\"r5b.24xlarge\",\n\t\t\"r5b.metal\",\n\t\t\"r5d.large\",\n\t\t\"r5d.xlarge\",\n\t\t\"r5d.2xlarge\",\n\t\t\"r5d.4xlarge\",\n\t\t\"r5d.8xlarge\",\n\t\t\"r5d.12xlarge\",\n\t\t\"r5d.16xlarge\",\n\t\t\"r5d.24xlarge\",\n\t\t\"r5d.metal\",\n\t\t\"r5dn.large\",\n\t\t\"r5dn.xlarge\",\n\t\t\"r5dn.2xlarge\",\n\t\t\"r5dn.4xlarge\",\n\t\t\"r5dn.8xlarge\",\n\t\t\"r5dn.12xlarge\",\n\t\t\"r5dn.16xlarge\",\n\t\t\"r5dn.24xlarge\",\n\t\t\"r5dn.metal\",\n\t\t\"r5n.large\",\n\t\t\"r5n.xlarge\",\n\t\t\"r5n.2xlarge\",\n\t\t\"r5n.4xlarge\",\n\t\t\"r5n.8xlarge\",\n\t\t\"r5n.12xlarge\",\n\t\t\"r5n.16xlarge\",\n\t\t\"r5n.24xlarge\",\n\t\t\"r5n.metal\",\n\t\t\"r6g.medium\",\n\t\t\"r6g.large\",\n\t\t\"r6g.xlarge\",\n\t\t\"r6g.2xlarge\",\n\t\t\"r6g.4xlarge\",\n\t\t\"r6g.8xlarge\",\n\t\t\"r6g.12xlarge\",\n\t\t\"r6g.16xlarge\",\n\t\t\"r6g.metal\",\n\t\t\"r6gd.medium\",\n\t\t\"r6gd.large\",\n\t\t\"r6gd.xlarge\",\n\t\t\"r6gd.2xlarge\",\n\t\t\"r6gd.4xlarge\",\n\t\t\"r6gd.8xlarge\",\n\t\t\"r6gd.12xlarge\",\n\t\t\"r6gd.16xlarge\",\n\t\t\"r6gd.metal\",\n\t\t\"r6i.large\",\n\t\t\"r6i.xlarge\",\n\t\t\"r6i.2xlarge\",\n\t\t\"r6i.4xlarge\",\n\t\t\"r6i.8xlarge\",\n\t\t\"r6i.12xlarge\",\n\t\t\"r6i.16xlarge\",\n\t\t\"r6i.24xlarge\",\n\t\t\"r6i.32xlarge\",\n\t\t\"r6i.metal\",\n\t\t\"t1.micro\",\n\t\t\"t2.nano\",\n\t\t\"t2.micro\",\n\t\t\"t2.small\",\n\t\t\"t2.medium\",\n\t\t\"t2.large\",\n\t\t\"t2.xlarge\",\n\t\t\"t2.2xlarge\",\n\t\t\"t3.nano\",\n\t\t\"t3.micro\",\n\t\t\"t3.small\",\n\t\t\"t3.medium\",\n\t\t\"t3.large\",\n\t\t\"t3.xlarge\",\n\t\t\"t3.2xlarge\",\n\t\t\"t3a.nano\",\n\t\t\"t3a.micro\",\n\t\t\"t3a.small\",\n\t\t\"t3a.medium\",\n\t\t\"t3a.large\",\n\t\t\"t3a.xlarge\",\n\t\t\"t3a.2xlarge\",\n\t\t\"t4g.nano\",\n\t\t\"t4g.micro\",\n\t\t\"t4g.small\",\n\t\t\"t4g.medium\",\n\t\t\"t4g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.2xlarge\",\n\t\t\"u-12tb1.112xlarge\",\n\t\t\"u-12tb1.metal\",\n\t\t\"u-3tb1.56xlarge\",\n\t\t\"u-6tb1.56xlarge\",\n\t\t\"u-6tb1.112xlarge\",\n\t\t\"u-6tb1.metal\",\n\t\t\"u-9tb1.112xlarge\",\n\t\t\"u-9tb1.metal\",\n\t\t\"vt1.3xlarge\",\n\t\t\"vt1.6xlarge\",\n\t\t\"vt1.24xlarge\",\n\t\t\"x1.16xlarge\",\n\t\t\"x1.32xlarge\",\n\t\t\"x1e.xlarge\",\n\t\t\"x1e.2xlarge\",\n\t\t\"x1e.4xlarge\",\n\t\t\"x1e.8xlarge\",\n\t\t\"x1e.16xlarge\",\n\t\t\"x1e.32xlarge\",\n\t\t\"x2gd.medium\",\n\t\t\"x2gd.large\",\n\t\t\"x2gd.xlarge\",\n\t\t\"x2gd.2xlarge\",\n\t\t\"x2gd.4xlarge\",\n\t\t\"x2gd.8xlarge\",\n\t\t\"x2gd.12xlarge\",\n\t\t\"x2gd.16xlarge\",\n\t\t\"x2gd.metal\",\n\t\t\"x2iedn.xlarge\",\n\t\t\"x2iedn.2xlarge\",\n\t\t\"x2iedn.4xlarge\",\n\t\t\"x2iedn.8xlarge\",\n\t\t\"x2iedn.16xlarge\",\n\t\t\"x2iedn.24xlarge\",\n\t\t\"x2iedn.32xlarge\",\n\t\t\"x2iezn.2xlarge\",\n\t\t\"x2iezn.4xlarge\",\n\t\t\"x2iezn.6xlarge\",\n\t\t\"x2iezn.8xlarge\",\n\t\t\"x2iezn.12xlarge\",\n\t\t\"x2iezn.metal\",\n\t\t\"z1d.large\",\n\t\t\"z1d.xlarge\",\n\t\t\"z1d.2xlarge\",\n\t\t\"z1d.3xlarge\",\n\t\t\"z1d.6xlarge\",\n\t\t\"z1d.12xlarge\",\n\t\t\"z1d.metal\",\n\t),\n}\n\n// region -> instance type -> instance metadata\nvar InstanceMetadatas = map[string]map[string]InstanceMetadata{\n\t\"af-south-1\": {\n\t\t\"c5.large\":      {Region: \"af-south-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.114},\n\t\t\"c5.xlarge\":     {Region: \"af-south-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.228},\n\t\t\"c5.2xlarge\":    {Region: \"af-south-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.456},\n\t\t\"c5.4xlarge\":    {Region: \"af-south-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.912},\n\t\t\"c5.9xlarge\":    {Region: \"af-south-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.052},\n\t\t\"c5.12xlarge\":   {Region: \"af-south-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.736},\n\t\t\"c5.18xlarge\":   {Region: \"af-south-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.104},\n\t\t\"c5.24xlarge\":   {Region: \"af-south-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.472},\n\t\t\"c5.metal\":      {Region: \"af-south-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.472},\n\t\t\"c5a.large\":     {Region: \"af-south-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.103},\n\t\t\"c5a.xlarge\":    {Region: \"af-south-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.206},\n\t\t\"c5a.2xlarge\":   {Region: \"af-south-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.412},\n\t\t\"c5a.4xlarge\":   {Region: \"af-south-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.824},\n\t\t\"c5a.8xlarge\":   {Region: \"af-south-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.648},\n\t\t\"c5a.12xlarge\":  {Region: \"af-south-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.472},\n\t\t\"c5a.16xlarge\":  {Region: \"af-south-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.296},\n\t\t\"c5a.24xlarge\":  {Region: \"af-south-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.944},\n\t\t\"c5ad.large\":    {Region: \"af-south-1\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.117},\n\t\t\"c5ad.xlarge\":   {Region: \"af-south-1\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.234},\n\t\t\"c5ad.2xlarge\":  {Region: \"af-south-1\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.468},\n\t\t\"c5ad.4xlarge\":  {Region: \"af-south-1\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.936},\n\t\t\"c5ad.8xlarge\":  {Region: \"af-south-1\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.872},\n\t\t\"c5ad.12xlarge\": {Region: \"af-south-1\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.808},\n\t\t\"c5ad.16xlarge\": {Region: \"af-south-1\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.744},\n\t\t\"c5ad.24xlarge\": {Region: \"af-south-1\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.616},\n\t\t\"c5d.large\":     {Region: \"af-south-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"c5d.xlarge\":    {Region: \"af-south-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.26},\n\t\t\"c5d.2xlarge\":   {Region: \"af-south-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"c5d.4xlarge\":   {Region: \"af-south-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.04},\n\t\t\"c5d.9xlarge\":   {Region: \"af-south-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.34},\n\t\t\"c5d.12xlarge\":  {Region: \"af-south-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.12},\n\t\t\"c5d.18xlarge\":  {Region: \"af-south-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.68},\n\t\t\"c5d.24xlarge\":  {Region: \"af-south-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.24},\n\t\t\"c5d.metal\":     {Region: \"af-south-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.24},\n\t\t\"c5n.large\":     {Region: \"af-south-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.145},\n\t\t\"c5n.xlarge\":    {Region: \"af-south-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.29},\n\t\t\"c5n.2xlarge\":   {Region: \"af-south-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.581},\n\t\t\"c5n.4xlarge\":   {Region: \"af-south-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.162},\n\t\t\"c5n.9xlarge\":   {Region: \"af-south-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.614},\n\t\t\"c5n.18xlarge\":  {Region: \"af-south-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.227},\n\t\t\"c5n.metal\":     {Region: \"af-south-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.227},\n\t\t\"d2.xlarge\":     {Region: \"af-south-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.875},\n\t\t\"d2.2xlarge\":    {Region: \"af-south-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.75},\n\t\t\"d2.4xlarge\":    {Region: \"af-south-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.5},\n\t\t\"d2.8xlarge\":    {Region: \"af-south-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 7.0},\n\t\t\"g4dn.xlarge\":   {Region: \"af-south-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.698},\n\t\t\"g4dn.2xlarge\":  {Region: \"af-south-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.998},\n\t\t\"g4dn.4xlarge\":  {Region: \"af-south-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.597},\n\t\t\"g4dn.8xlarge\":  {Region: \"af-south-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.887},\n\t\t\"g4dn.12xlarge\": {Region: \"af-south-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 5.19},\n\t\t\"g4dn.16xlarge\": {Region: \"af-south-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.774},\n\t\t\"g4dn.metal\":    {Region: \"af-south-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 10.381},\n\t\t\"i3.large\":      {Region: \"af-south-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.205},\n\t\t\"i3.xlarge\":     {Region: \"af-south-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.41},\n\t\t\"i3.2xlarge\":    {Region: \"af-south-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.82},\n\t\t\"i3.4xlarge\":    {Region: \"af-south-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.64},\n\t\t\"i3.8xlarge\":    {Region: \"af-south-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.28},\n\t\t\"i3.16xlarge\":   {Region: \"af-south-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.56},\n\t\t\"i3.metal\":      {Region: \"af-south-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.56},\n\t\t\"i3en.large\":    {Region: \"af-south-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.297},\n\t\t\"i3en.xlarge\":   {Region: \"af-south-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.594},\n\t\t\"i3en.2xlarge\":  {Region: \"af-south-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.188},\n\t\t\"i3en.3xlarge\":  {Region: \"af-south-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.782},\n\t\t\"i3en.6xlarge\":  {Region: \"af-south-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.564},\n\t\t\"i3en.12xlarge\": {Region: \"af-south-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 7.128},\n\t\t\"i3en.24xlarge\": {Region: \"af-south-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 14.256},\n\t\t\"i3en.metal\":    {Region: \"af-south-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 14.256},\n\t\t\"inf1.24xlarge\": {Region: \"af-south-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 6.249},\n\t\t\"m5.large\":      {Region: \"af-south-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.127},\n\t\t\"m5.xlarge\":     {Region: \"af-south-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.254},\n\t\t\"m5.2xlarge\":    {Region: \"af-south-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.508},\n\t\t\"m5.4xlarge\":    {Region: \"af-south-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.016},\n\t\t\"m5.8xlarge\":    {Region: \"af-south-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.032},\n\t\t\"m5.12xlarge\":   {Region: \"af-south-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.048},\n\t\t\"m5.16xlarge\":   {Region: \"af-south-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.064},\n\t\t\"m5.24xlarge\":   {Region: \"af-south-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.096},\n\t\t\"m5.metal\":      {Region: \"af-south-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.096},\n\t\t\"m5d.large\":     {Region: \"af-south-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.15},\n\t\t\"m5d.xlarge\":    {Region: \"af-south-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3},\n\t\t\"m5d.2xlarge\":   {Region: \"af-south-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.6},\n\t\t\"m5d.4xlarge\":   {Region: \"af-south-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.2},\n\t\t\"m5d.8xlarge\":   {Region: \"af-south-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.4},\n\t\t\"m5d.12xlarge\":  {Region: \"af-south-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.6},\n\t\t\"m5d.16xlarge\":  {Region: \"af-south-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.8},\n\t\t\"m5d.24xlarge\":  {Region: \"af-south-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.2},\n\t\t\"m5d.metal\":     {Region: \"af-south-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.2},\n\t\t\"r5.large\":      {Region: \"af-south-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.168},\n\t\t\"r5.xlarge\":     {Region: \"af-south-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.336},\n\t\t\"r5.2xlarge\":    {Region: \"af-south-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.672},\n\t\t\"r5.4xlarge\":    {Region: \"af-south-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.344},\n\t\t\"r5.8xlarge\":    {Region: \"af-south-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"r5.12xlarge\":   {Region: \"af-south-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r5.16xlarge\":   {Region: \"af-south-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"r5.24xlarge\":   {Region: \"af-south-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"r5.metal\":      {Region: \"af-south-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"r5d.large\":     {Region: \"af-south-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.19},\n\t\t\"r5d.xlarge\":    {Region: \"af-south-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.38},\n\t\t\"r5d.2xlarge\":   {Region: \"af-south-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.76},\n\t\t\"r5d.4xlarge\":   {Region: \"af-south-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.52},\n\t\t\"r5d.8xlarge\":   {Region: \"af-south-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.04},\n\t\t\"r5d.12xlarge\":  {Region: \"af-south-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.56},\n\t\t\"r5d.16xlarge\":  {Region: \"af-south-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.08},\n\t\t\"r5d.24xlarge\":  {Region: \"af-south-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.12},\n\t\t\"r5d.metal\":     {Region: \"af-south-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.12},\n\t\t\"r5dn.large\":    {Region: \"af-south-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.221},\n\t\t\"r5dn.xlarge\":   {Region: \"af-south-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.442},\n\t\t\"r5dn.2xlarge\":  {Region: \"af-south-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.884},\n\t\t\"r5dn.4xlarge\":  {Region: \"af-south-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.768},\n\t\t\"r5dn.8xlarge\":  {Region: \"af-south-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.536},\n\t\t\"r5dn.12xlarge\": {Region: \"af-south-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.304},\n\t\t\"r5dn.16xlarge\": {Region: \"af-south-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 7.072},\n\t\t\"r5dn.24xlarge\": {Region: \"af-south-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.608},\n\t\t\"r5dn.metal\":    {Region: \"af-south-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.608},\n\t\t\"t3.nano\":       {Region: \"af-south-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0068},\n\t\t\"t3.micro\":      {Region: \"af-south-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0136},\n\t\t\"t3.small\":      {Region: \"af-south-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0271},\n\t\t\"t3.medium\":     {Region: \"af-south-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0542},\n\t\t\"t3.large\":      {Region: \"af-south-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1085},\n\t\t\"t3.xlarge\":     {Region: \"af-south-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.217},\n\t\t\"t3.2xlarge\":    {Region: \"af-south-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4339},\n\t\t\"x1.16xlarge\":   {Region: \"af-south-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.524},\n\t\t\"x1.32xlarge\":   {Region: \"af-south-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 19.048},\n\t\t\"x1e.xlarge\":    {Region: \"af-south-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.19},\n\t\t\"x1e.2xlarge\":   {Region: \"af-south-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.38},\n\t\t\"x1e.4xlarge\":   {Region: \"af-south-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.76},\n\t\t\"x1e.8xlarge\":   {Region: \"af-south-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.52},\n\t\t\"x1e.16xlarge\":  {Region: \"af-south-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 19.04},\n\t\t\"x1e.32xlarge\":  {Region: \"af-south-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 38.08},\n\t},\n\t\"ap-east-1\": {\n\t\t\"c5.large\":      {Region: \"ap-east-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"c5.xlarge\":     {Region: \"ap-east-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"c5.2xlarge\":    {Region: \"ap-east-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"c5.4xlarge\":    {Region: \"ap-east-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"c5.9xlarge\":    {Region: \"ap-east-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.944},\n\t\t\"c5.12xlarge\":   {Region: \"ap-east-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"c5.18xlarge\":   {Region: \"ap-east-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c5.24xlarge\":   {Region: \"ap-east-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.184},\n\t\t\"c5.metal\":      {Region: \"ap-east-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.184},\n\t\t\"c5a.large\":     {Region: \"ap-east-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.097},\n\t\t\"c5a.xlarge\":    {Region: \"ap-east-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.194},\n\t\t\"c5a.2xlarge\":   {Region: \"ap-east-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.388},\n\t\t\"c5a.4xlarge\":   {Region: \"ap-east-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.776},\n\t\t\"c5a.8xlarge\":   {Region: \"ap-east-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.552},\n\t\t\"c5a.12xlarge\":  {Region: \"ap-east-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.328},\n\t\t\"c5a.16xlarge\":  {Region: \"ap-east-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.104},\n\t\t\"c5a.24xlarge\":  {Region: \"ap-east-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.656},\n\t\t\"c5d.large\":     {Region: \"ap-east-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.123},\n\t\t\"c5d.xlarge\":    {Region: \"ap-east-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.246},\n\t\t\"c5d.2xlarge\":   {Region: \"ap-east-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.492},\n\t\t\"c5d.4xlarge\":   {Region: \"ap-east-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.984},\n\t\t\"c5d.9xlarge\":   {Region: \"ap-east-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.214},\n\t\t\"c5d.18xlarge\":  {Region: \"ap-east-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.428},\n\t\t\"c5n.large\":     {Region: \"ap-east-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.143},\n\t\t\"c5n.xlarge\":    {Region: \"ap-east-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.286},\n\t\t\"c5n.2xlarge\":   {Region: \"ap-east-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.572},\n\t\t\"c5n.4xlarge\":   {Region: \"ap-east-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.144},\n\t\t\"c5n.9xlarge\":   {Region: \"ap-east-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.574},\n\t\t\"c5n.18xlarge\":  {Region: \"ap-east-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.148},\n\t\t\"c5n.metal\":     {Region: \"ap-east-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.148},\n\t\t\"c6g.medium\":    {Region: \"ap-east-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.043},\n\t\t\"c6g.large\":     {Region: \"ap-east-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"c6g.xlarge\":    {Region: \"ap-east-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"c6g.2xlarge\":   {Region: \"ap-east-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"c6g.4xlarge\":   {Region: \"ap-east-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"c6g.8xlarge\":   {Region: \"ap-east-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"c6g.12xlarge\":  {Region: \"ap-east-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"c6g.16xlarge\":  {Region: \"ap-east-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"c6g.metal\":     {Region: \"ap-east-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"c6gn.medium\":   {Region: \"ap-east-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.05725},\n\t\t\"c6gn.large\":    {Region: \"ap-east-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1145},\n\t\t\"c6gn.xlarge\":   {Region: \"ap-east-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.229},\n\t\t\"c6gn.2xlarge\":  {Region: \"ap-east-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.458},\n\t\t\"c6gn.4xlarge\":  {Region: \"ap-east-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.916},\n\t\t\"c6gn.8xlarge\":  {Region: \"ap-east-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.832},\n\t\t\"c6gn.12xlarge\": {Region: \"ap-east-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.748},\n\t\t\"c6gn.16xlarge\": {Region: \"ap-east-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.664},\n\t\t\"d2.xlarge\":     {Region: \"ap-east-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.957},\n\t\t\"d2.2xlarge\":    {Region: \"ap-east-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.914},\n\t\t\"d2.4xlarge\":    {Region: \"ap-east-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.828},\n\t\t\"d2.8xlarge\":    {Region: \"ap-east-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 7.656},\n\t\t\"g4dn.xlarge\":   {Region: \"ap-east-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.81},\n\t\t\"g4dn.2xlarge\":  {Region: \"ap-east-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.158},\n\t\t\"g4dn.4xlarge\":  {Region: \"ap-east-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.854},\n\t\t\"g4dn.8xlarge\":  {Region: \"ap-east-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 3.351},\n\t\t\"g4dn.12xlarge\": {Region: \"ap-east-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 6.024},\n\t\t\"g4dn.16xlarge\": {Region: \"ap-east-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 6.702},\n\t\t\"g4dn.metal\":    {Region: \"ap-east-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 12.048},\n\t\t\"i3.large\":      {Region: \"ap-east-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.206},\n\t\t\"i3.xlarge\":     {Region: \"ap-east-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.412},\n\t\t\"i3.2xlarge\":    {Region: \"ap-east-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.824},\n\t\t\"i3.4xlarge\":    {Region: \"ap-east-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.648},\n\t\t\"i3.8xlarge\":    {Region: \"ap-east-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.296},\n\t\t\"i3.16xlarge\":   {Region: \"ap-east-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.592},\n\t\t\"i3.metal\":      {Region: \"ap-east-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.592},\n\t\t\"i3en.large\":    {Region: \"ap-east-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"i3en.xlarge\":   {Region: \"ap-east-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.597},\n\t\t\"i3en.2xlarge\":  {Region: \"ap-east-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.193},\n\t\t\"i3en.3xlarge\":  {Region: \"ap-east-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.79},\n\t\t\"i3en.6xlarge\":  {Region: \"ap-east-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.58},\n\t\t\"i3en.12xlarge\": {Region: \"ap-east-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 7.16},\n\t\t\"i3en.24xlarge\": {Region: \"ap-east-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 14.319},\n\t\t\"i3en.metal\":    {Region: \"ap-east-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 14.319},\n\t\t\"inf1.xlarge\":   {Region: \"ap-east-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.352},\n\t\t\"inf1.2xlarge\":  {Region: \"ap-east-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.558},\n\t\t\"inf1.6xlarge\":  {Region: \"ap-east-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.819},\n\t\t\"inf1.24xlarge\": {Region: \"ap-east-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 7.274},\n\t\t\"m5.large\":      {Region: \"ap-east-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.132},\n\t\t\"m5.xlarge\":     {Region: \"ap-east-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.264},\n\t\t\"m5.2xlarge\":    {Region: \"ap-east-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.528},\n\t\t\"m5.4xlarge\":    {Region: \"ap-east-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.056},\n\t\t\"m5.8xlarge\":    {Region: \"ap-east-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.112},\n\t\t\"m5.12xlarge\":   {Region: \"ap-east-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.168},\n\t\t\"m5.16xlarge\":   {Region: \"ap-east-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.224},\n\t\t\"m5.24xlarge\":   {Region: \"ap-east-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.336},\n\t\t\"m5.metal\":      {Region: \"ap-east-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.336},\n\t\t\"m5d.large\":     {Region: \"ap-east-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.155},\n\t\t\"m5d.xlarge\":    {Region: \"ap-east-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.31},\n\t\t\"m5d.2xlarge\":   {Region: \"ap-east-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.62},\n\t\t\"m5d.4xlarge\":   {Region: \"ap-east-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.24},\n\t\t\"m5d.8xlarge\":   {Region: \"ap-east-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.48},\n\t\t\"m5d.12xlarge\":  {Region: \"ap-east-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.72},\n\t\t\"m5d.16xlarge\":  {Region: \"ap-east-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.96},\n\t\t\"m5d.24xlarge\":  {Region: \"ap-east-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.44},\n\t\t\"m5d.metal\":     {Region: \"ap-east-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.44},\n\t\t\"m6g.medium\":    {Region: \"ap-east-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.053},\n\t\t\"m6g.large\":     {Region: \"ap-east-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.106},\n\t\t\"m6g.xlarge\":    {Region: \"ap-east-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.212},\n\t\t\"m6g.2xlarge\":   {Region: \"ap-east-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.424},\n\t\t\"m6g.4xlarge\":   {Region: \"ap-east-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.848},\n\t\t\"m6g.8xlarge\":   {Region: \"ap-east-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.696},\n\t\t\"m6g.12xlarge\":  {Region: \"ap-east-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"m6g.16xlarge\":  {Region: \"ap-east-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.392},\n\t\t\"m6g.metal\":     {Region: \"ap-east-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.392},\n\t\t\"r5.large\":      {Region: \"ap-east-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"r5.xlarge\":     {Region: \"ap-east-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"r5.2xlarge\":    {Region: \"ap-east-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"r5.4xlarge\":    {Region: \"ap-east-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"r5.8xlarge\":    {Region: \"ap-east-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"r5.12xlarge\":   {Region: \"ap-east-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"r5.16xlarge\":   {Region: \"ap-east-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"r5.24xlarge\":   {Region: \"ap-east-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5.metal\":      {Region: \"ap-east-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5d.large\":     {Region: \"ap-east-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.175},\n\t\t\"r5d.xlarge\":    {Region: \"ap-east-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.35},\n\t\t\"r5d.2xlarge\":   {Region: \"ap-east-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.7},\n\t\t\"r5d.4xlarge\":   {Region: \"ap-east-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.4},\n\t\t\"r5d.8xlarge\":   {Region: \"ap-east-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.8},\n\t\t\"r5d.12xlarge\":  {Region: \"ap-east-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.2},\n\t\t\"r5d.16xlarge\":  {Region: \"ap-east-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.6},\n\t\t\"r5d.24xlarge\":  {Region: \"ap-east-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"r5d.metal\":     {Region: \"ap-east-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"r5n.large\":     {Region: \"ap-east-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"r5n.xlarge\":    {Region: \"ap-east-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"r5n.2xlarge\":   {Region: \"ap-east-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"r5n.4xlarge\":   {Region: \"ap-east-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"r5n.8xlarge\":   {Region: \"ap-east-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"r5n.12xlarge\":  {Region: \"ap-east-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"r5n.16xlarge\":  {Region: \"ap-east-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.464},\n\t\t\"r5n.24xlarge\":  {Region: \"ap-east-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.696},\n\t\t\"r5n.metal\":     {Region: \"ap-east-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.696},\n\t\t\"r6g.medium\":    {Region: \"ap-east-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.067},\n\t\t\"r6g.large\":     {Region: \"ap-east-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.134},\n\t\t\"r6g.xlarge\":    {Region: \"ap-east-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.268},\n\t\t\"r6g.2xlarge\":   {Region: \"ap-east-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.536},\n\t\t\"r6g.4xlarge\":   {Region: \"ap-east-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.072},\n\t\t\"r6g.8xlarge\":   {Region: \"ap-east-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.144},\n\t\t\"r6g.12xlarge\":  {Region: \"ap-east-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.216},\n\t\t\"r6g.16xlarge\":  {Region: \"ap-east-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.288},\n\t\t\"r6g.metal\":     {Region: \"ap-east-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.288},\n\t\t\"t3.nano\":       {Region: \"ap-east-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0073},\n\t\t\"t3.micro\":      {Region: \"ap-east-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0146},\n\t\t\"t3.small\":      {Region: \"ap-east-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0292},\n\t\t\"t3.medium\":     {Region: \"ap-east-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0584},\n\t\t\"t3.large\":      {Region: \"ap-east-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1168},\n\t\t\"t3.xlarge\":     {Region: \"ap-east-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2336},\n\t\t\"t3.2xlarge\":    {Region: \"ap-east-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4672},\n\t\t\"t4g.nano\":      {Region: \"ap-east-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0058},\n\t\t\"t4g.micro\":     {Region: \"ap-east-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0116},\n\t\t\"t4g.small\":     {Region: \"ap-east-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0232},\n\t\t\"t4g.medium\":    {Region: \"ap-east-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0464},\n\t\t\"t4g.large\":     {Region: \"ap-east-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0928},\n\t\t\"t4g.xlarge\":    {Region: \"ap-east-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1856},\n\t\t\"t4g.2xlarge\":   {Region: \"ap-east-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3712},\n\t\t\"x1.16xlarge\":   {Region: \"ap-east-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 10.638},\n\t\t\"x1.32xlarge\":   {Region: \"ap-east-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 21.276},\n\t},\n\t\"ap-northeast-1\": {\n\t\t\"a1.medium\":       {Region: \"ap-northeast-1\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0321},\n\t\t\"a1.large\":        {Region: \"ap-northeast-1\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0642},\n\t\t\"a1.xlarge\":       {Region: \"ap-northeast-1\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1284},\n\t\t\"a1.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2568},\n\t\t\"a1.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.5136},\n\t\t\"a1.metal\":        {Region: \"ap-northeast-1\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.514},\n\t\t\"c1.medium\":       {Region: \"ap-northeast-1\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.158},\n\t\t\"c1.xlarge\":       {Region: \"ap-northeast-1\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.632},\n\t\t\"c3.large\":        {Region: \"ap-northeast-1\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.128},\n\t\t\"c3.xlarge\":       {Region: \"ap-northeast-1\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.255},\n\t\t\"c3.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.511},\n\t\t\"c3.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.021},\n\t\t\"c3.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.043},\n\t\t\"c4.large\":        {Region: \"ap-northeast-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"c4.xlarge\":       {Region: \"ap-northeast-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"c4.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"c4.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"c4.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"c5.large\":        {Region: \"ap-northeast-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.107},\n\t\t\"c5.xlarge\":       {Region: \"ap-northeast-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.214},\n\t\t\"c5.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.428},\n\t\t\"c5.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.856},\n\t\t\"c5.9xlarge\":      {Region: \"ap-northeast-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.926},\n\t\t\"c5.12xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.568},\n\t\t\"c5.18xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.852},\n\t\t\"c5.24xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"c5.metal\":        {Region: \"ap-northeast-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"c5a.large\":       {Region: \"ap-northeast-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"c5a.xlarge\":      {Region: \"ap-northeast-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"c5a.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"c5a.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"c5a.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"c5a.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c5a.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"c5a.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5d.large\":       {Region: \"ap-northeast-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.122},\n\t\t\"c5d.xlarge\":      {Region: \"ap-northeast-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.244},\n\t\t\"c5d.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.488},\n\t\t\"c5d.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.976},\n\t\t\"c5d.9xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.196},\n\t\t\"c5d.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.928},\n\t\t\"c5d.18xlarge\":    {Region: \"ap-northeast-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.392},\n\t\t\"c5d.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"c5d.metal\":       {Region: \"ap-northeast-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"c5n.large\":       {Region: \"ap-northeast-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"c5n.xlarge\":      {Region: \"ap-northeast-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"c5n.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"c5n.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"c5n.9xlarge\":     {Region: \"ap-northeast-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"c5n.18xlarge\":    {Region: \"ap-northeast-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"c5n.metal\":       {Region: \"ap-northeast-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"c6g.medium\":      {Region: \"ap-northeast-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0428},\n\t\t\"c6g.large\":       {Region: \"ap-northeast-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0856},\n\t\t\"c6g.xlarge\":      {Region: \"ap-northeast-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1712},\n\t\t\"c6g.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3424},\n\t\t\"c6g.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6848},\n\t\t\"c6g.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3696},\n\t\t\"c6g.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0544},\n\t\t\"c6g.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7392},\n\t\t\"c6g.metal\":       {Region: \"ap-northeast-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7392},\n\t\t\"c6gd.medium\":     {Region: \"ap-northeast-1\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.049},\n\t\t\"c6gd.large\":      {Region: \"ap-northeast-1\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.098},\n\t\t\"c6gd.xlarge\":     {Region: \"ap-northeast-1\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"c6gd.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"c6gd.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"c6gd.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.568},\n\t\t\"c6gd.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.352},\n\t\t\"c6gd.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.136},\n\t\t\"c6gd.metal\":      {Region: \"ap-northeast-1\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.136},\n\t\t\"c6gn.medium\":     {Region: \"ap-northeast-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0545},\n\t\t\"c6gn.large\":      {Region: \"ap-northeast-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.109},\n\t\t\"c6gn.xlarge\":     {Region: \"ap-northeast-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.218},\n\t\t\"c6gn.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.436},\n\t\t\"c6gn.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.872},\n\t\t\"c6gn.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.744},\n\t\t\"c6gn.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.616},\n\t\t\"c6gn.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.488},\n\t\t\"c6gn.metal\":      {Region: \"ap-northeast-1\", Type: \"c6gn.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.488},\n\t\t\"c6i.large\":       {Region: \"ap-northeast-1\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.107},\n\t\t\"c6i.xlarge\":      {Region: \"ap-northeast-1\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.214},\n\t\t\"c6i.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.428},\n\t\t\"c6i.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.856},\n\t\t\"c6i.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.712},\n\t\t\"c6i.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.568},\n\t\t\"c6i.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.424},\n\t\t\"c6i.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"c6i.32xlarge\":    {Region: \"ap-northeast-1\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.848},\n\t\t\"c6i.metal\":       {Region: \"ap-northeast-1\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.848},\n\t\t\"cc2.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"cc2.8xlarge\", Memory: kresource.MustParse(\"61952Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.349},\n\t\t\"cr1.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"cr1.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.105},\n\t\t\"d2.xlarge\":       {Region: \"ap-northeast-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.844},\n\t\t\"d2.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.688},\n\t\t\"d2.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.376},\n\t\t\"d2.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.752},\n\t\t\"d3.xlarge\":       {Region: \"ap-northeast-1\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.724},\n\t\t\"d3.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.448},\n\t\t\"d3.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.897},\n\t\t\"d3.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 5.79344},\n\t\t\"g2.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.898},\n\t\t\"g2.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 3.592},\n\t\t\"g3.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.58},\n\t\t\"g3.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 3.16},\n\t\t\"g3.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 6.32},\n\t\t\"g3s.xlarge\":      {Region: \"ap-northeast-1\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.04},\n\t\t\"g4ad.xlarge\":     {Region: \"ap-northeast-1\", Type: \"g4ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.51082},\n\t\t\"g4ad.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"g4ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.7303},\n\t\t\"g4ad.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"g4ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.17},\n\t\t\"g4ad.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"g4ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.34},\n\t\t\"g4ad.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"g4ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 4.68},\n\t\t\"g4dn.xlarge\":     {Region: \"ap-northeast-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.71},\n\t\t\"g4dn.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.015},\n\t\t\"g4dn.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.625},\n\t\t\"g4dn.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.938},\n\t\t\"g4dn.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 5.281},\n\t\t\"g4dn.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.875},\n\t\t\"g4dn.metal\":      {Region: \"ap-northeast-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 10.562},\n\t\t\"g5g.xlarge\":      {Region: \"ap-northeast-1\", Type: \"g5g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.5669},\n\t\t\"g5g.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"g5g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.7505},\n\t\t\"g5g.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"g5g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.1176},\n\t\t\"g5g.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"g5g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 1.8519},\n\t\t\"g5g.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"g5g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 3.7039},\n\t\t\"g5g.metal\":       {Region: \"ap-northeast-1\", Type: \"g5g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 3.7039},\n\t\t\"hs1.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"hs1.8xlarge\", Memory: kresource.MustParse(\"119808Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 5.4},\n\t\t\"i2.xlarge\":       {Region: \"ap-northeast-1\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.001},\n\t\t\"i2.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.001},\n\t\t\"i2.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.002},\n\t\t\"i2.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.004},\n\t\t\"i3.large\":        {Region: \"ap-northeast-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.183},\n\t\t\"i3.xlarge\":       {Region: \"ap-northeast-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.366},\n\t\t\"i3.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.732},\n\t\t\"i3.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.464},\n\t\t\"i3.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.928},\n\t\t\"i3.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"i3.metal\":        {Region: \"ap-northeast-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"i3en.large\":      {Region: \"ap-northeast-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"i3en.xlarge\":     {Region: \"ap-northeast-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"i3en.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"i3en.3xlarge\":    {Region: \"ap-northeast-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.596},\n\t\t\"i3en.6xlarge\":    {Region: \"ap-northeast-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"i3en.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"i3en.24xlarge\":   {Region: \"ap-northeast-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.768},\n\t\t\"i3en.metal\":      {Region: \"ap-northeast-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.768},\n\t\t\"inf1.xlarge\":     {Region: \"ap-northeast-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.308},\n\t\t\"inf1.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.489},\n\t\t\"inf1.6xlarge\":    {Region: \"ap-northeast-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.594},\n\t\t\"inf1.24xlarge\":   {Region: \"ap-northeast-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 6.376},\n\t\t\"m1.small\":        {Region: \"ap-northeast-1\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.061},\n\t\t\"m1.medium\":       {Region: \"ap-northeast-1\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.122},\n\t\t\"m1.large\":        {Region: \"ap-northeast-1\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.243},\n\t\t\"m1.xlarge\":       {Region: \"ap-northeast-1\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.486},\n\t\t\"m2.xlarge\":       {Region: \"ap-northeast-1\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.287},\n\t\t\"m2.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.575},\n\t\t\"m2.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.15},\n\t\t\"m3.medium\":       {Region: \"ap-northeast-1\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m3.large\":        {Region: \"ap-northeast-1\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.193},\n\t\t\"m3.xlarge\":       {Region: \"ap-northeast-1\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.385},\n\t\t\"m3.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.77},\n\t\t\"m4.large\":        {Region: \"ap-northeast-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.129},\n\t\t\"m4.xlarge\":       {Region: \"ap-northeast-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.258},\n\t\t\"m4.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.516},\n\t\t\"m4.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.032},\n\t\t\"m4.10xlarge\":     {Region: \"ap-northeast-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.58},\n\t\t\"m4.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"m5.large\":        {Region: \"ap-northeast-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.124},\n\t\t\"m5.xlarge\":       {Region: \"ap-northeast-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.248},\n\t\t\"m5.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.496},\n\t\t\"m5.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.992},\n\t\t\"m5.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.984},\n\t\t\"m5.12xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.976},\n\t\t\"m5.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.968},\n\t\t\"m5.24xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"m5.metal\":        {Region: \"ap-northeast-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"m5a.large\":       {Region: \"ap-northeast-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.112},\n\t\t\"m5a.xlarge\":      {Region: \"ap-northeast-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.224},\n\t\t\"m5a.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.448},\n\t\t\"m5a.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.896},\n\t\t\"m5a.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.792},\n\t\t\"m5a.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"m5a.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.584},\n\t\t\"m5a.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m5ad.large\":      {Region: \"ap-northeast-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.134},\n\t\t\"m5ad.xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.268},\n\t\t\"m5ad.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.536},\n\t\t\"m5ad.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.072},\n\t\t\"m5ad.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.144},\n\t\t\"m5ad.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.216},\n\t\t\"m5ad.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.288},\n\t\t\"m5ad.24xlarge\":   {Region: \"ap-northeast-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.432},\n\t\t\"m5d.large\":       {Region: \"ap-northeast-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.146},\n\t\t\"m5d.xlarge\":      {Region: \"ap-northeast-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.292},\n\t\t\"m5d.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.584},\n\t\t\"m5d.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.168},\n\t\t\"m5d.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.336},\n\t\t\"m5d.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.504},\n\t\t\"m5d.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.672},\n\t\t\"m5d.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.008},\n\t\t\"m5d.metal\":       {Region: \"ap-northeast-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.008},\n\t\t\"m5dn.large\":      {Region: \"ap-northeast-1\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.175},\n\t\t\"m5dn.xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.35},\n\t\t\"m5dn.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.7},\n\t\t\"m5dn.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.4},\n\t\t\"m5dn.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.8},\n\t\t\"m5dn.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.2},\n\t\t\"m5dn.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.6},\n\t\t\"m5dn.24xlarge\":   {Region: \"ap-northeast-1\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"m5dn.metal\":      {Region: \"ap-northeast-1\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"m5n.large\":       {Region: \"ap-northeast-1\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.153},\n\t\t\"m5n.xlarge\":      {Region: \"ap-northeast-1\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.306},\n\t\t\"m5n.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.612},\n\t\t\"m5n.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.224},\n\t\t\"m5n.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"m5n.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"m5n.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"m5n.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"m5n.metal\":       {Region: \"ap-northeast-1\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"m5zn.large\":      {Region: \"ap-northeast-1\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2134},\n\t\t\"m5zn.xlarge\":     {Region: \"ap-northeast-1\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.4267},\n\t\t\"m5zn.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.8534},\n\t\t\"m5zn.3xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.2801},\n\t\t\"m5zn.6xlarge\":    {Region: \"ap-northeast-1\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.5602},\n\t\t\"m5zn.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.1204},\n\t\t\"m5zn.metal\":      {Region: \"ap-northeast-1\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.1204},\n\t\t\"m6g.medium\":      {Region: \"ap-northeast-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0495},\n\t\t\"m6g.large\":       {Region: \"ap-northeast-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.099},\n\t\t\"m6g.xlarge\":      {Region: \"ap-northeast-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.198},\n\t\t\"m6g.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.396},\n\t\t\"m6g.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.792},\n\t\t\"m6g.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.584},\n\t\t\"m6g.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.376},\n\t\t\"m6g.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.168},\n\t\t\"m6g.metal\":       {Region: \"ap-northeast-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.168},\n\t\t\"m6gd.medium\":     {Region: \"ap-northeast-1\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0585},\n\t\t\"m6gd.large\":      {Region: \"ap-northeast-1\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.117},\n\t\t\"m6gd.xlarge\":     {Region: \"ap-northeast-1\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.234},\n\t\t\"m6gd.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.468},\n\t\t\"m6gd.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.936},\n\t\t\"m6gd.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.872},\n\t\t\"m6gd.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.808},\n\t\t\"m6gd.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.744},\n\t\t\"m6gd.metal\":      {Region: \"ap-northeast-1\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.744},\n\t\t\"m6i.large\":       {Region: \"ap-northeast-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.124},\n\t\t\"m6i.xlarge\":      {Region: \"ap-northeast-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.248},\n\t\t\"m6i.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.496},\n\t\t\"m6i.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.992},\n\t\t\"m6i.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.984},\n\t\t\"m6i.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.976},\n\t\t\"m6i.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.968},\n\t\t\"m6i.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"m6i.32xlarge\":    {Region: \"ap-northeast-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.936},\n\t\t\"m6i.metal\":       {Region: \"ap-northeast-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.936},\n\t\t\"p2.xlarge\":       {Region: \"ap-northeast-1\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.542},\n\t\t\"p2.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 12.336},\n\t\t\"p2.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 24.672},\n\t\t\"p3.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 4.194},\n\t\t\"p3.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 16.776},\n\t\t\"p3.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 33.552},\n\t\t\"p3dn.24xlarge\":   {Region: \"ap-northeast-1\", Type: \"p3dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 42.783},\n\t\t\"p4d.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"p4d.24xlarge\", Memory: kresource.MustParse(\"1179648Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 44.92215},\n\t\t\"r3.large\":        {Region: \"ap-northeast-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"r3.xlarge\":       {Region: \"ap-northeast-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.399},\n\t\t\"r3.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.798},\n\t\t\"r3.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.596},\n\t\t\"r3.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"r4.large\":        {Region: \"ap-northeast-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.16},\n\t\t\"r4.xlarge\":       {Region: \"ap-northeast-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.32},\n\t\t\"r4.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.64},\n\t\t\"r4.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.28},\n\t\t\"r4.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.56},\n\t\t\"r4.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.12},\n\t\t\"r5.large\":        {Region: \"ap-northeast-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r5.xlarge\":       {Region: \"ap-northeast-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r5.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r5.4xlarge\":      {Region: \"ap-northeast-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r5.8xlarge\":      {Region: \"ap-northeast-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r5.12xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r5.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r5.24xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5.metal\":        {Region: \"ap-northeast-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5a.large\":       {Region: \"ap-northeast-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.137},\n\t\t\"r5a.xlarge\":      {Region: \"ap-northeast-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.274},\n\t\t\"r5a.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.548},\n\t\t\"r5a.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.096},\n\t\t\"r5a.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.192},\n\t\t\"r5a.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.288},\n\t\t\"r5a.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.384},\n\t\t\"r5a.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.576},\n\t\t\"r5ad.large\":      {Region: \"ap-northeast-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.159},\n\t\t\"r5ad.xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.318},\n\t\t\"r5ad.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.636},\n\t\t\"r5ad.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.272},\n\t\t\"r5ad.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"r5ad.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.816},\n\t\t\"r5ad.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"r5ad.24xlarge\":   {Region: \"ap-northeast-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.632},\n\t\t\"r5b.large\":       {Region: \"ap-northeast-1\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.181},\n\t\t\"r5b.xlarge\":      {Region: \"ap-northeast-1\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.362},\n\t\t\"r5b.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.724},\n\t\t\"r5b.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.448},\n\t\t\"r5b.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.896},\n\t\t\"r5b.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.344},\n\t\t\"r5b.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"r5b.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.688},\n\t\t\"r5b.metal\":       {Region: \"ap-northeast-1\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.688},\n\t\t\"r5d.large\":       {Region: \"ap-northeast-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.174},\n\t\t\"r5d.xlarge\":      {Region: \"ap-northeast-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.348},\n\t\t\"r5d.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.696},\n\t\t\"r5d.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.392},\n\t\t\"r5d.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.784},\n\t\t\"r5d.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"r5d.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.568},\n\t\t\"r5d.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.352},\n\t\t\"r5d.metal\":       {Region: \"ap-northeast-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.352},\n\t\t\"r5dn.large\":      {Region: \"ap-northeast-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.203},\n\t\t\"r5dn.xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.406},\n\t\t\"r5dn.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.812},\n\t\t\"r5dn.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.624},\n\t\t\"r5dn.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.248},\n\t\t\"r5dn.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.872},\n\t\t\"r5dn.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.496},\n\t\t\"r5dn.24xlarge\":   {Region: \"ap-northeast-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.744},\n\t\t\"r5dn.metal\":      {Region: \"ap-northeast-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.744},\n\t\t\"r5n.large\":       {Region: \"ap-northeast-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.181},\n\t\t\"r5n.xlarge\":      {Region: \"ap-northeast-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.362},\n\t\t\"r5n.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.724},\n\t\t\"r5n.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.448},\n\t\t\"r5n.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.896},\n\t\t\"r5n.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.344},\n\t\t\"r5n.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"r5n.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.688},\n\t\t\"r5n.metal\":       {Region: \"ap-northeast-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.688},\n\t\t\"r6g.medium\":      {Region: \"ap-northeast-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0608},\n\t\t\"r6g.large\":       {Region: \"ap-northeast-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1216},\n\t\t\"r6g.xlarge\":      {Region: \"ap-northeast-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2432},\n\t\t\"r6g.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4864},\n\t\t\"r6g.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9728},\n\t\t\"r6g.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.9456},\n\t\t\"r6g.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.9184},\n\t\t\"r6g.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8912},\n\t\t\"r6g.metal\":       {Region: \"ap-northeast-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8912},\n\t\t\"r6gd.medium\":     {Region: \"ap-northeast-1\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0695},\n\t\t\"r6gd.large\":      {Region: \"ap-northeast-1\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.139},\n\t\t\"r6gd.xlarge\":     {Region: \"ap-northeast-1\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.278},\n\t\t\"r6gd.2xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.556},\n\t\t\"r6gd.4xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.112},\n\t\t\"r6gd.8xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.224},\n\t\t\"r6gd.12xlarge\":   {Region: \"ap-northeast-1\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"r6gd.16xlarge\":   {Region: \"ap-northeast-1\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.448},\n\t\t\"r6gd.metal\":      {Region: \"ap-northeast-1\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.448},\n\t\t\"r6i.large\":       {Region: \"ap-northeast-1\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r6i.xlarge\":      {Region: \"ap-northeast-1\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r6i.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r6i.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r6i.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r6i.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r6i.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r6i.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r6i.32xlarge\":    {Region: \"ap-northeast-1\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.728},\n\t\t\"r6i.metal\":       {Region: \"ap-northeast-1\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.728},\n\t\t\"t1.micro\":        {Region: \"ap-northeast-1\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.026},\n\t\t\"t2.nano\":         {Region: \"ap-northeast-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0076},\n\t\t\"t2.micro\":        {Region: \"ap-northeast-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0152},\n\t\t\"t2.small\":        {Region: \"ap-northeast-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0304},\n\t\t\"t2.medium\":       {Region: \"ap-northeast-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0608},\n\t\t\"t2.large\":        {Region: \"ap-northeast-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1216},\n\t\t\"t2.xlarge\":       {Region: \"ap-northeast-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2432},\n\t\t\"t2.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4864},\n\t\t\"t3.nano\":         {Region: \"ap-northeast-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0068},\n\t\t\"t3.micro\":        {Region: \"ap-northeast-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0136},\n\t\t\"t3.small\":        {Region: \"ap-northeast-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0272},\n\t\t\"t3.medium\":       {Region: \"ap-northeast-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0544},\n\t\t\"t3.large\":        {Region: \"ap-northeast-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1088},\n\t\t\"t3.xlarge\":       {Region: \"ap-northeast-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2176},\n\t\t\"t3.2xlarge\":      {Region: \"ap-northeast-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4352},\n\t\t\"t3a.nano\":        {Region: \"ap-northeast-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0061},\n\t\t\"t3a.micro\":       {Region: \"ap-northeast-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0122},\n\t\t\"t3a.small\":       {Region: \"ap-northeast-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0245},\n\t\t\"t3a.medium\":      {Region: \"ap-northeast-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.049},\n\t\t\"t3a.large\":       {Region: \"ap-northeast-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0979},\n\t\t\"t3a.xlarge\":      {Region: \"ap-northeast-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1958},\n\t\t\"t3a.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3917},\n\t\t\"t4g.nano\":        {Region: \"ap-northeast-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0054},\n\t\t\"t4g.micro\":       {Region: \"ap-northeast-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0108},\n\t\t\"t4g.small\":       {Region: \"ap-northeast-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0216},\n\t\t\"t4g.medium\":      {Region: \"ap-northeast-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0432},\n\t\t\"t4g.large\":       {Region: \"ap-northeast-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0864},\n\t\t\"t4g.xlarge\":      {Region: \"ap-northeast-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1728},\n\t\t\"t4g.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3456},\n\t\t\"vt1.3xlarge\":     {Region: \"ap-northeast-1\", Type: \"vt1.3xlarge\", Memory: kresource.MustParse(\"24576Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 0.81824},\n\t\t\"vt1.6xlarge\":     {Region: \"ap-northeast-1\", Type: \"vt1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 1.63647},\n\t\t\"vt1.24xlarge\":    {Region: \"ap-northeast-1\", Type: \"vt1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.54588},\n\t\t\"x1.16xlarge\":     {Region: \"ap-northeast-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.671},\n\t\t\"x1.32xlarge\":     {Region: \"ap-northeast-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 19.341},\n\t\t\"x1e.xlarge\":      {Region: \"ap-northeast-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.209},\n\t\t\"x1e.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.418},\n\t\t\"x1e.4xlarge\":     {Region: \"ap-northeast-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.836},\n\t\t\"x1e.8xlarge\":     {Region: \"ap-northeast-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.672},\n\t\t\"x1e.16xlarge\":    {Region: \"ap-northeast-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 19.344},\n\t\t\"x1e.32xlarge\":    {Region: \"ap-northeast-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 38.688},\n\t\t\"x2idn.16xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2idn.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.6705},\n\t\t\"x2idn.24xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2idn.24xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 14.50575},\n\t\t\"x2idn.32xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2idn.32xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 19.341},\n\t\t\"x2iedn.xlarge\":   {Region: \"ap-northeast-1\", Type: \"x2iedn.xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.20881},\n\t\t\"x2iedn.2xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2iedn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.41763},\n\t\t\"x2iedn.4xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2iedn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.83525},\n\t\t\"x2iedn.8xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2iedn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.6705},\n\t\t\"x2iedn.16xlarge\": {Region: \"ap-northeast-1\", Type: \"x2iedn.16xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 19.341},\n\t\t\"x2iedn.24xlarge\": {Region: \"ap-northeast-1\", Type: \"x2iedn.24xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 29.0115},\n\t\t\"x2iedn.32xlarge\": {Region: \"ap-northeast-1\", Type: \"x2iedn.32xlarge\", Memory: kresource.MustParse(\"4194304Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 38.682},\n\t\t\"x2iezn.2xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2iezn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.418},\n\t\t\"x2iezn.4xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2iezn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.836},\n\t\t\"x2iezn.6xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2iezn.6xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 7.254},\n\t\t\"x2iezn.8xlarge\":  {Region: \"ap-northeast-1\", Type: \"x2iezn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.672},\n\t\t\"x2iezn.12xlarge\": {Region: \"ap-northeast-1\", Type: \"x2iezn.12xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 14.508},\n\t\t\"x2iezn.metal\":    {Region: \"ap-northeast-1\", Type: \"x2iezn.metal\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 14.508},\n\t\t\"z1d.large\":       {Region: \"ap-northeast-1\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.227},\n\t\t\"z1d.xlarge\":      {Region: \"ap-northeast-1\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.454},\n\t\t\"z1d.2xlarge\":     {Region: \"ap-northeast-1\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.908},\n\t\t\"z1d.3xlarge\":     {Region: \"ap-northeast-1\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.362},\n\t\t\"z1d.6xlarge\":     {Region: \"ap-northeast-1\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.724},\n\t\t\"z1d.12xlarge\":    {Region: \"ap-northeast-1\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.448},\n\t\t\"z1d.metal\":       {Region: \"ap-northeast-1\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.448},\n\t},\n\t\"ap-northeast-2\": {\n\t\t\"c3.large\":         {Region: \"ap-northeast-2\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.115},\n\t\t\"c3.xlarge\":        {Region: \"ap-northeast-2\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.23},\n\t\t\"c3.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.46},\n\t\t\"c3.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.919},\n\t\t\"c3.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.839},\n\t\t\"c4.large\":         {Region: \"ap-northeast-2\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.114},\n\t\t\"c4.xlarge\":        {Region: \"ap-northeast-2\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.227},\n\t\t\"c4.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.454},\n\t\t\"c4.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.907},\n\t\t\"c4.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.815},\n\t\t\"c5.large\":         {Region: \"ap-northeast-2\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"c5.xlarge\":        {Region: \"ap-northeast-2\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"c5.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"c5.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"c5.9xlarge\":       {Region: \"ap-northeast-2\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"c5.12xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c5.18xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"c5.24xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5.metal\":         {Region: \"ap-northeast-2\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5a.large\":        {Region: \"ap-northeast-2\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"c5a.xlarge\":       {Region: \"ap-northeast-2\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"c5a.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"c5a.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"c5a.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"c5a.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"c5a.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"c5a.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"c5d.large\":        {Region: \"ap-northeast-2\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.11},\n\t\t\"c5d.xlarge\":       {Region: \"ap-northeast-2\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.22},\n\t\t\"c5d.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.44},\n\t\t\"c5d.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.88},\n\t\t\"c5d.9xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.98},\n\t\t\"c5d.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.64},\n\t\t\"c5d.18xlarge\":     {Region: \"ap-northeast-2\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.96},\n\t\t\"c5d.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.28},\n\t\t\"c5d.metal\":        {Region: \"ap-northeast-2\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.28},\n\t\t\"c5n.large\":        {Region: \"ap-northeast-2\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.122},\n\t\t\"c5n.xlarge\":       {Region: \"ap-northeast-2\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.244},\n\t\t\"c5n.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.488},\n\t\t\"c5n.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.976},\n\t\t\"c5n.9xlarge\":      {Region: \"ap-northeast-2\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.196},\n\t\t\"c5n.18xlarge\":     {Region: \"ap-northeast-2\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.392},\n\t\t\"c5n.metal\":        {Region: \"ap-northeast-2\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.392},\n\t\t\"c6g.medium\":       {Region: \"ap-northeast-2\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0385},\n\t\t\"c6g.large\":        {Region: \"ap-northeast-2\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.077},\n\t\t\"c6g.xlarge\":       {Region: \"ap-northeast-2\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"c6g.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"c6g.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"c6g.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.232},\n\t\t\"c6g.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.848},\n\t\t\"c6g.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"c6g.metal\":        {Region: \"ap-northeast-2\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"c6i.large\":        {Region: \"ap-northeast-2\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"c6i.xlarge\":       {Region: \"ap-northeast-2\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"c6i.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"c6i.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"c6i.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"c6i.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c6i.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"c6i.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c6i.32xlarge\":     {Region: \"ap-northeast-2\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.144},\n\t\t\"c6i.metal\":        {Region: \"ap-northeast-2\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.144},\n\t\t\"d2.xlarge\":        {Region: \"ap-northeast-2\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.844},\n\t\t\"d2.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.688},\n\t\t\"d2.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.376},\n\t\t\"d2.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.752},\n\t\t\"g2.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.898},\n\t\t\"g2.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 3.592},\n\t\t\"g3.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.42},\n\t\t\"g3.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.84},\n\t\t\"g3.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 5.68},\n\t\t\"g3s.xlarge\":       {Region: \"ap-northeast-2\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.934},\n\t\t\"g4dn.xlarge\":      {Region: \"ap-northeast-2\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.647},\n\t\t\"g4dn.2xlarge\":     {Region: \"ap-northeast-2\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.925},\n\t\t\"g4dn.4xlarge\":     {Region: \"ap-northeast-2\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.481},\n\t\t\"g4dn.8xlarge\":     {Region: \"ap-northeast-2\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.677},\n\t\t\"g4dn.12xlarge\":    {Region: \"ap-northeast-2\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.812},\n\t\t\"g4dn.16xlarge\":    {Region: \"ap-northeast-2\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.353},\n\t\t\"g4dn.metal\":       {Region: \"ap-northeast-2\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 9.624},\n\t\t\"g5g.xlarge\":       {Region: \"ap-northeast-2\", Type: \"g5g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.5166},\n\t\t\"g5g.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"g5g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.6839},\n\t\t\"g5g.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"g5g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.0185},\n\t\t\"g5g.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"g5g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 1.6876},\n\t\t\"g5g.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"g5g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 3.3752},\n\t\t\"g5g.metal\":        {Region: \"ap-northeast-2\", Type: \"g5g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 3.3752},\n\t\t\"i2.xlarge\":        {Region: \"ap-northeast-2\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.001},\n\t\t\"i2.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.001},\n\t\t\"i2.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.002},\n\t\t\"i2.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.004},\n\t\t\"i3.large\":         {Region: \"ap-northeast-2\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.183},\n\t\t\"i3.xlarge\":        {Region: \"ap-northeast-2\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.366},\n\t\t\"i3.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.732},\n\t\t\"i3.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.464},\n\t\t\"i3.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.928},\n\t\t\"i3.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"i3.metal\":         {Region: \"ap-northeast-2\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"i3en.large\":       {Region: \"ap-northeast-2\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"i3en.xlarge\":      {Region: \"ap-northeast-2\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"i3en.2xlarge\":     {Region: \"ap-northeast-2\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"i3en.3xlarge\":     {Region: \"ap-northeast-2\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.596},\n\t\t\"i3en.6xlarge\":     {Region: \"ap-northeast-2\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"i3en.12xlarge\":    {Region: \"ap-northeast-2\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"i3en.24xlarge\":    {Region: \"ap-northeast-2\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.768},\n\t\t\"i3en.metal\":       {Region: \"ap-northeast-2\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.768},\n\t\t\"inf1.xlarge\":      {Region: \"ap-northeast-2\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.281},\n\t\t\"inf1.2xlarge\":     {Region: \"ap-northeast-2\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.446},\n\t\t\"inf1.6xlarge\":     {Region: \"ap-northeast-2\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.453},\n\t\t\"inf1.24xlarge\":    {Region: \"ap-northeast-2\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.812},\n\t\t\"m3.medium\":        {Region: \"ap-northeast-2\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.091},\n\t\t\"m3.large\":         {Region: \"ap-northeast-2\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.183},\n\t\t\"m3.xlarge\":        {Region: \"ap-northeast-2\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.366},\n\t\t\"m3.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.732},\n\t\t\"m4.large\":         {Region: \"ap-northeast-2\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.123},\n\t\t\"m4.xlarge\":        {Region: \"ap-northeast-2\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.246},\n\t\t\"m4.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.492},\n\t\t\"m4.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.984},\n\t\t\"m4.10xlarge\":      {Region: \"ap-northeast-2\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.46},\n\t\t\"m4.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.936},\n\t\t\"m5.large\":         {Region: \"ap-northeast-2\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.118},\n\t\t\"m5.xlarge\":        {Region: \"ap-northeast-2\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.236},\n\t\t\"m5.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.472},\n\t\t\"m5.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.944},\n\t\t\"m5.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.888},\n\t\t\"m5.12xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.832},\n\t\t\"m5.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.776},\n\t\t\"m5.24xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.664},\n\t\t\"m5.metal\":         {Region: \"ap-northeast-2\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.664},\n\t\t\"m5a.large\":        {Region: \"ap-northeast-2\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.106},\n\t\t\"m5a.xlarge\":       {Region: \"ap-northeast-2\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.212},\n\t\t\"m5a.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.424},\n\t\t\"m5a.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.848},\n\t\t\"m5a.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.696},\n\t\t\"m5a.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"m5a.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.392},\n\t\t\"m5a.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"m5ad.large\":       {Region: \"ap-northeast-2\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.127},\n\t\t\"m5ad.xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.254},\n\t\t\"m5ad.2xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.508},\n\t\t\"m5ad.4xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.016},\n\t\t\"m5ad.8xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.032},\n\t\t\"m5ad.12xlarge\":    {Region: \"ap-northeast-2\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.048},\n\t\t\"m5ad.16xlarge\":    {Region: \"ap-northeast-2\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.064},\n\t\t\"m5ad.24xlarge\":    {Region: \"ap-northeast-2\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.096},\n\t\t\"m5d.large\":        {Region: \"ap-northeast-2\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.139},\n\t\t\"m5d.xlarge\":       {Region: \"ap-northeast-2\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.278},\n\t\t\"m5d.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.556},\n\t\t\"m5d.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.112},\n\t\t\"m5d.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.224},\n\t\t\"m5d.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"m5d.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.448},\n\t\t\"m5d.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.672},\n\t\t\"m5d.metal\":        {Region: \"ap-northeast-2\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.672},\n\t\t\"m5zn.large\":       {Region: \"ap-northeast-2\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.203},\n\t\t\"m5zn.xlarge\":      {Region: \"ap-northeast-2\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.406},\n\t\t\"m5zn.2xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.812},\n\t\t\"m5zn.3xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.218},\n\t\t\"m5zn.6xlarge\":     {Region: \"ap-northeast-2\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.436},\n\t\t\"m5zn.12xlarge\":    {Region: \"ap-northeast-2\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.872},\n\t\t\"m5zn.metal\":       {Region: \"ap-northeast-2\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.872},\n\t\t\"m6g.medium\":       {Region: \"ap-northeast-2\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.047},\n\t\t\"m6g.large\":        {Region: \"ap-northeast-2\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.094},\n\t\t\"m6g.xlarge\":       {Region: \"ap-northeast-2\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.188},\n\t\t\"m6g.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.376},\n\t\t\"m6g.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.752},\n\t\t\"m6g.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.504},\n\t\t\"m6g.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.256},\n\t\t\"m6g.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.008},\n\t\t\"m6g.metal\":        {Region: \"ap-northeast-2\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.008},\n\t\t\"m6i.large\":        {Region: \"ap-northeast-2\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.118},\n\t\t\"m6i.xlarge\":       {Region: \"ap-northeast-2\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.236},\n\t\t\"m6i.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.472},\n\t\t\"m6i.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.944},\n\t\t\"m6i.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.888},\n\t\t\"m6i.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.832},\n\t\t\"m6i.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.776},\n\t\t\"m6i.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.664},\n\t\t\"m6i.32xlarge\":     {Region: \"ap-northeast-2\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.552},\n\t\t\"m6i.metal\":        {Region: \"ap-northeast-2\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.552},\n\t\t\"p2.xlarge\":        {Region: \"ap-northeast-2\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.465},\n\t\t\"p2.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 11.72},\n\t\t\"p2.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 23.44},\n\t\t\"p3.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 4.234},\n\t\t\"p3.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 16.936},\n\t\t\"p3.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 33.872},\n\t\t\"p4d.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"p4d.24xlarge\", Memory: kresource.MustParse(\"1179648Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 45.38848},\n\t\t\"r3.large\":         {Region: \"ap-northeast-2\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"r3.xlarge\":        {Region: \"ap-northeast-2\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.399},\n\t\t\"r3.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.798},\n\t\t\"r3.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.596},\n\t\t\"r3.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"r4.large\":         {Region: \"ap-northeast-2\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.16},\n\t\t\"r4.xlarge\":        {Region: \"ap-northeast-2\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.32},\n\t\t\"r4.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.64},\n\t\t\"r4.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.28},\n\t\t\"r4.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.56},\n\t\t\"r4.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.12},\n\t\t\"r5.large\":         {Region: \"ap-northeast-2\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r5.xlarge\":        {Region: \"ap-northeast-2\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r5.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r5.4xlarge\":       {Region: \"ap-northeast-2\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r5.8xlarge\":       {Region: \"ap-northeast-2\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r5.12xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r5.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r5.24xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5.metal\":         {Region: \"ap-northeast-2\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5a.large\":        {Region: \"ap-northeast-2\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"r5a.xlarge\":       {Region: \"ap-northeast-2\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"r5a.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"r5a.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"r5a.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"r5a.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"r5a.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"r5a.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"r5ad.large\":       {Region: \"ap-northeast-2\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.158},\n\t\t\"r5ad.xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.316},\n\t\t\"r5ad.2xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.632},\n\t\t\"r5ad.4xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.264},\n\t\t\"r5ad.8xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.528},\n\t\t\"r5ad.12xlarge\":    {Region: \"ap-northeast-2\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.792},\n\t\t\"r5ad.16xlarge\":    {Region: \"ap-northeast-2\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.056},\n\t\t\"r5ad.24xlarge\":    {Region: \"ap-northeast-2\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.584},\n\t\t\"r5b.large\":        {Region: \"ap-northeast-2\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.178},\n\t\t\"r5b.xlarge\":       {Region: \"ap-northeast-2\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.356},\n\t\t\"r5b.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.712},\n\t\t\"r5b.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.424},\n\t\t\"r5b.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.848},\n\t\t\"r5b.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.272},\n\t\t\"r5b.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.696},\n\t\t\"r5b.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5b.metal\":        {Region: \"ap-northeast-2\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5d.large\":        {Region: \"ap-northeast-2\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.173},\n\t\t\"r5d.xlarge\":       {Region: \"ap-northeast-2\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.346},\n\t\t\"r5d.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.692},\n\t\t\"r5d.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.384},\n\t\t\"r5d.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.768},\n\t\t\"r5d.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.152},\n\t\t\"r5d.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.536},\n\t\t\"r5d.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.304},\n\t\t\"r5d.metal\":        {Region: \"ap-northeast-2\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.304},\n\t\t\"r5dn.large\":       {Region: \"ap-northeast-2\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.199},\n\t\t\"r5dn.xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.398},\n\t\t\"r5dn.2xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.796},\n\t\t\"r5dn.4xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.592},\n\t\t\"r5dn.8xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.184},\n\t\t\"r5dn.12xlarge\":    {Region: \"ap-northeast-2\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.776},\n\t\t\"r5dn.16xlarge\":    {Region: \"ap-northeast-2\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.368},\n\t\t\"r5dn.24xlarge\":    {Region: \"ap-northeast-2\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.552},\n\t\t\"r5dn.metal\":       {Region: \"ap-northeast-2\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.552},\n\t\t\"r5n.large\":        {Region: \"ap-northeast-2\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.178},\n\t\t\"r5n.xlarge\":       {Region: \"ap-northeast-2\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.356},\n\t\t\"r5n.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.712},\n\t\t\"r5n.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.424},\n\t\t\"r5n.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.848},\n\t\t\"r5n.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.272},\n\t\t\"r5n.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.696},\n\t\t\"r5n.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5n.metal\":        {Region: \"ap-northeast-2\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r6g.medium\":       {Region: \"ap-northeast-2\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.061},\n\t\t\"r6g.large\":        {Region: \"ap-northeast-2\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.122},\n\t\t\"r6g.xlarge\":       {Region: \"ap-northeast-2\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.244},\n\t\t\"r6g.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.488},\n\t\t\"r6g.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.976},\n\t\t\"r6g.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.952},\n\t\t\"r6g.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.928},\n\t\t\"r6g.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.904},\n\t\t\"r6g.metal\":        {Region: \"ap-northeast-2\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.904},\n\t\t\"r6i.large\":        {Region: \"ap-northeast-2\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r6i.xlarge\":       {Region: \"ap-northeast-2\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r6i.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r6i.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r6i.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r6i.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r6i.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r6i.24xlarge\":     {Region: \"ap-northeast-2\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r6i.32xlarge\":     {Region: \"ap-northeast-2\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.728},\n\t\t\"r6i.metal\":        {Region: \"ap-northeast-2\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.728},\n\t\t\"t2.nano\":          {Region: \"ap-northeast-2\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0072},\n\t\t\"t2.micro\":         {Region: \"ap-northeast-2\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0144},\n\t\t\"t2.small\":         {Region: \"ap-northeast-2\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0288},\n\t\t\"t2.medium\":        {Region: \"ap-northeast-2\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0576},\n\t\t\"t2.large\":         {Region: \"ap-northeast-2\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1152},\n\t\t\"t2.xlarge\":        {Region: \"ap-northeast-2\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2304},\n\t\t\"t2.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4608},\n\t\t\"t3.nano\":          {Region: \"ap-northeast-2\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0065},\n\t\t\"t3.micro\":         {Region: \"ap-northeast-2\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.013},\n\t\t\"t3.small\":         {Region: \"ap-northeast-2\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.026},\n\t\t\"t3.medium\":        {Region: \"ap-northeast-2\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.052},\n\t\t\"t3.large\":         {Region: \"ap-northeast-2\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.104},\n\t\t\"t3.xlarge\":        {Region: \"ap-northeast-2\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.208},\n\t\t\"t3.2xlarge\":       {Region: \"ap-northeast-2\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.416},\n\t\t\"t3a.nano\":         {Region: \"ap-northeast-2\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0059},\n\t\t\"t3a.micro\":        {Region: \"ap-northeast-2\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0117},\n\t\t\"t3a.small\":        {Region: \"ap-northeast-2\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0234},\n\t\t\"t3a.medium\":       {Region: \"ap-northeast-2\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0468},\n\t\t\"t3a.large\":        {Region: \"ap-northeast-2\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0936},\n\t\t\"t3a.xlarge\":       {Region: \"ap-northeast-2\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1872},\n\t\t\"t3a.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3744},\n\t\t\"t4g.nano\":         {Region: \"ap-northeast-2\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0052},\n\t\t\"t4g.micro\":        {Region: \"ap-northeast-2\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0104},\n\t\t\"t4g.small\":        {Region: \"ap-northeast-2\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0208},\n\t\t\"t4g.medium\":       {Region: \"ap-northeast-2\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0416},\n\t\t\"t4g.large\":        {Region: \"ap-northeast-2\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0832},\n\t\t\"t4g.xlarge\":       {Region: \"ap-northeast-2\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1664},\n\t\t\"t4g.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3328},\n\t\t\"u-6tb1.56xlarge\":  {Region: \"ap-northeast-2\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 55.9796},\n\t\t\"u-6tb1.112xlarge\": {Region: \"ap-northeast-2\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 65.867},\n\t\t\"x1.16xlarge\":      {Region: \"ap-northeast-2\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.671},\n\t\t\"x1.32xlarge\":      {Region: \"ap-northeast-2\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 19.341},\n\t\t\"x1e.xlarge\":       {Region: \"ap-northeast-2\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.209},\n\t\t\"x1e.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.418},\n\t\t\"x1e.4xlarge\":      {Region: \"ap-northeast-2\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.836},\n\t\t\"x1e.8xlarge\":      {Region: \"ap-northeast-2\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.672},\n\t\t\"x1e.16xlarge\":     {Region: \"ap-northeast-2\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 19.344},\n\t\t\"x1e.32xlarge\":     {Region: \"ap-northeast-2\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 38.688},\n\t\t\"z1d.large\":        {Region: \"ap-northeast-2\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.225},\n\t\t\"z1d.xlarge\":       {Region: \"ap-northeast-2\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.45},\n\t\t\"z1d.2xlarge\":      {Region: \"ap-northeast-2\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.9},\n\t\t\"z1d.3xlarge\":      {Region: \"ap-northeast-2\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.35},\n\t\t\"z1d.6xlarge\":      {Region: \"ap-northeast-2\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.7},\n\t\t\"z1d.12xlarge\":     {Region: \"ap-northeast-2\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.4},\n\t\t\"z1d.metal\":        {Region: \"ap-northeast-2\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.4},\n\t},\n\t\"ap-northeast-3\": {\n\t\t\"c3.large\":      {Region: \"ap-northeast-3\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.128},\n\t\t\"c3.xlarge\":     {Region: \"ap-northeast-3\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.256},\n\t\t\"c3.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.512},\n\t\t\"c3.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.024},\n\t\t\"c3.8xlarge\":    {Region: \"ap-northeast-3\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.048},\n\t\t\"c4.large\":      {Region: \"ap-northeast-3\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"c4.xlarge\":     {Region: \"ap-northeast-3\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"c4.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"c4.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"c4.8xlarge\":    {Region: \"ap-northeast-3\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"c5.large\":      {Region: \"ap-northeast-3\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.107},\n\t\t\"c5.xlarge\":     {Region: \"ap-northeast-3\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.214},\n\t\t\"c5.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.428},\n\t\t\"c5.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.856},\n\t\t\"c5.9xlarge\":    {Region: \"ap-northeast-3\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.926},\n\t\t\"c5.12xlarge\":   {Region: \"ap-northeast-3\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.568},\n\t\t\"c5.18xlarge\":   {Region: \"ap-northeast-3\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.852},\n\t\t\"c5.24xlarge\":   {Region: \"ap-northeast-3\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"c5.metal\":      {Region: \"ap-northeast-3\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"c5d.large\":     {Region: \"ap-northeast-3\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.122},\n\t\t\"c5d.xlarge\":    {Region: \"ap-northeast-3\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.244},\n\t\t\"c5d.2xlarge\":   {Region: \"ap-northeast-3\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.488},\n\t\t\"c5d.4xlarge\":   {Region: \"ap-northeast-3\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.976},\n\t\t\"c5d.9xlarge\":   {Region: \"ap-northeast-3\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.196},\n\t\t\"c5d.12xlarge\":  {Region: \"ap-northeast-3\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.928},\n\t\t\"c5d.18xlarge\":  {Region: \"ap-northeast-3\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.392},\n\t\t\"c5d.24xlarge\":  {Region: \"ap-northeast-3\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"c5d.metal\":     {Region: \"ap-northeast-3\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"d2.xlarge\":     {Region: \"ap-northeast-3\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.844},\n\t\t\"d2.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.688},\n\t\t\"d2.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.376},\n\t\t\"d2.8xlarge\":    {Region: \"ap-northeast-3\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.752},\n\t\t\"g4dn.xlarge\":   {Region: \"ap-northeast-3\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.71},\n\t\t\"g4dn.2xlarge\":  {Region: \"ap-northeast-3\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.015},\n\t\t\"g4dn.4xlarge\":  {Region: \"ap-northeast-3\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.625},\n\t\t\"g4dn.8xlarge\":  {Region: \"ap-northeast-3\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.938},\n\t\t\"g4dn.12xlarge\": {Region: \"ap-northeast-3\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 5.281},\n\t\t\"g4dn.16xlarge\": {Region: \"ap-northeast-3\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.875},\n\t\t\"g4dn.metal\":    {Region: \"ap-northeast-3\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 10.562},\n\t\t\"i3.large\":      {Region: \"ap-northeast-3\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.183},\n\t\t\"i3.xlarge\":     {Region: \"ap-northeast-3\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.366},\n\t\t\"i3.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.732},\n\t\t\"i3.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.464},\n\t\t\"i3.8xlarge\":    {Region: \"ap-northeast-3\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.928},\n\t\t\"i3.16xlarge\":   {Region: \"ap-northeast-3\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"i3.metal\":      {Region: \"ap-northeast-3\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"i3en.large\":    {Region: \"ap-northeast-3\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"i3en.xlarge\":   {Region: \"ap-northeast-3\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"i3en.2xlarge\":  {Region: \"ap-northeast-3\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"i3en.3xlarge\":  {Region: \"ap-northeast-3\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.596},\n\t\t\"i3en.6xlarge\":  {Region: \"ap-northeast-3\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"i3en.12xlarge\": {Region: \"ap-northeast-3\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"i3en.24xlarge\": {Region: \"ap-northeast-3\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.768},\n\t\t\"i3en.metal\":    {Region: \"ap-northeast-3\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.768},\n\t\t\"m3.medium\":     {Region: \"ap-northeast-3\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m3.large\":      {Region: \"ap-northeast-3\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m3.xlarge\":     {Region: \"ap-northeast-3\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m3.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m4.large\":      {Region: \"ap-northeast-3\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.129},\n\t\t\"m4.xlarge\":     {Region: \"ap-northeast-3\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.258},\n\t\t\"m4.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.516},\n\t\t\"m4.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.032},\n\t\t\"m4.10xlarge\":   {Region: \"ap-northeast-3\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.58},\n\t\t\"m4.16xlarge\":   {Region: \"ap-northeast-3\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"m5.large\":      {Region: \"ap-northeast-3\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.124},\n\t\t\"m5.xlarge\":     {Region: \"ap-northeast-3\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.248},\n\t\t\"m5.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.496},\n\t\t\"m5.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.992},\n\t\t\"m5.8xlarge\":    {Region: \"ap-northeast-3\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.984},\n\t\t\"m5.12xlarge\":   {Region: \"ap-northeast-3\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.976},\n\t\t\"m5.16xlarge\":   {Region: \"ap-northeast-3\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.968},\n\t\t\"m5.24xlarge\":   {Region: \"ap-northeast-3\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"m5.metal\":      {Region: \"ap-northeast-3\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"m5d.large\":     {Region: \"ap-northeast-3\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.146},\n\t\t\"m5d.xlarge\":    {Region: \"ap-northeast-3\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.292},\n\t\t\"m5d.2xlarge\":   {Region: \"ap-northeast-3\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.584},\n\t\t\"m5d.4xlarge\":   {Region: \"ap-northeast-3\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.168},\n\t\t\"m5d.8xlarge\":   {Region: \"ap-northeast-3\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.336},\n\t\t\"m5d.12xlarge\":  {Region: \"ap-northeast-3\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.504},\n\t\t\"m5d.16xlarge\":  {Region: \"ap-northeast-3\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.672},\n\t\t\"m5d.24xlarge\":  {Region: \"ap-northeast-3\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.008},\n\t\t\"m5d.metal\":     {Region: \"ap-northeast-3\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.008},\n\t\t\"r3.large\":      {Region: \"ap-northeast-3\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"r3.xlarge\":     {Region: \"ap-northeast-3\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"r3.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"r3.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.6},\n\t\t\"r3.8xlarge\":    {Region: \"ap-northeast-3\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"r4.large\":      {Region: \"ap-northeast-3\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.16},\n\t\t\"r4.xlarge\":     {Region: \"ap-northeast-3\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.32},\n\t\t\"r4.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.64},\n\t\t\"r4.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.28},\n\t\t\"r4.8xlarge\":    {Region: \"ap-northeast-3\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.56},\n\t\t\"r4.16xlarge\":   {Region: \"ap-northeast-3\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.12},\n\t\t\"r5.large\":      {Region: \"ap-northeast-3\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r5.xlarge\":     {Region: \"ap-northeast-3\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r5.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r5.4xlarge\":    {Region: \"ap-northeast-3\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r5.8xlarge\":    {Region: \"ap-northeast-3\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r5.12xlarge\":   {Region: \"ap-northeast-3\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r5.16xlarge\":   {Region: \"ap-northeast-3\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r5.24xlarge\":   {Region: \"ap-northeast-3\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5.metal\":      {Region: \"ap-northeast-3\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5d.large\":     {Region: \"ap-northeast-3\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.174},\n\t\t\"r5d.xlarge\":    {Region: \"ap-northeast-3\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.348},\n\t\t\"r5d.2xlarge\":   {Region: \"ap-northeast-3\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.696},\n\t\t\"r5d.4xlarge\":   {Region: \"ap-northeast-3\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.392},\n\t\t\"r5d.8xlarge\":   {Region: \"ap-northeast-3\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.784},\n\t\t\"r5d.12xlarge\":  {Region: \"ap-northeast-3\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"r5d.16xlarge\":  {Region: \"ap-northeast-3\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.568},\n\t\t\"r5d.24xlarge\":  {Region: \"ap-northeast-3\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.352},\n\t\t\"r5d.metal\":     {Region: \"ap-northeast-3\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.352},\n\t\t\"t2.nano\":       {Region: \"ap-northeast-3\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0076},\n\t\t\"t2.micro\":      {Region: \"ap-northeast-3\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0152},\n\t\t\"t2.small\":      {Region: \"ap-northeast-3\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0304},\n\t\t\"t2.medium\":     {Region: \"ap-northeast-3\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0608},\n\t\t\"t2.large\":      {Region: \"ap-northeast-3\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1216},\n\t\t\"t2.xlarge\":     {Region: \"ap-northeast-3\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2432},\n\t\t\"t2.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4864},\n\t\t\"t3.nano\":       {Region: \"ap-northeast-3\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0068},\n\t\t\"t3.micro\":      {Region: \"ap-northeast-3\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0136},\n\t\t\"t3.small\":      {Region: \"ap-northeast-3\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0272},\n\t\t\"t3.medium\":     {Region: \"ap-northeast-3\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0544},\n\t\t\"t3.large\":      {Region: \"ap-northeast-3\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1088},\n\t\t\"t3.xlarge\":     {Region: \"ap-northeast-3\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2176},\n\t\t\"t3.2xlarge\":    {Region: \"ap-northeast-3\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4352},\n\t\t\"x1.16xlarge\":   {Region: \"ap-northeast-3\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.671},\n\t\t\"x1.32xlarge\":   {Region: \"ap-northeast-3\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 19.342},\n\t\t\"x1e.xlarge\":    {Region: \"ap-northeast-3\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.209},\n\t\t\"x1e.2xlarge\":   {Region: \"ap-northeast-3\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.418},\n\t\t\"x1e.4xlarge\":   {Region: \"ap-northeast-3\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.836},\n\t\t\"x1e.8xlarge\":   {Region: \"ap-northeast-3\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.672},\n\t\t\"x1e.16xlarge\":  {Region: \"ap-northeast-3\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 19.344},\n\t\t\"x1e.32xlarge\":  {Region: \"ap-northeast-3\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 38.688},\n\t},\n\t\"ap-south-1\": {\n\t\t\"a1.medium\":        {Region: \"ap-south-1\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0255},\n\t\t\"a1.large\":         {Region: \"ap-south-1\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.051},\n\t\t\"a1.xlarge\":        {Region: \"ap-south-1\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.102},\n\t\t\"a1.2xlarge\":       {Region: \"ap-south-1\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.204},\n\t\t\"a1.4xlarge\":       {Region: \"ap-south-1\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"a1.metal\":         {Region: \"ap-south-1\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"c4.large\":         {Region: \"ap-south-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"c4.xlarge\":        {Region: \"ap-south-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"c4.2xlarge\":       {Region: \"ap-south-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"c4.4xlarge\":       {Region: \"ap-south-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"c4.8xlarge\":       {Region: \"ap-south-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.6},\n\t\t\"c5.large\":         {Region: \"ap-south-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c5.xlarge\":        {Region: \"ap-south-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c5.2xlarge\":       {Region: \"ap-south-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c5.4xlarge\":       {Region: \"ap-south-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c5.9xlarge\":       {Region: \"ap-south-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.53},\n\t\t\"c5.12xlarge\":      {Region: \"ap-south-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c5.18xlarge\":      {Region: \"ap-south-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.06},\n\t\t\"c5.24xlarge\":      {Region: \"ap-south-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c5.metal\":         {Region: \"ap-south-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c5a.large\":        {Region: \"ap-south-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.047},\n\t\t\"c5a.xlarge\":       {Region: \"ap-south-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.094},\n\t\t\"c5a.2xlarge\":      {Region: \"ap-south-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.188},\n\t\t\"c5a.4xlarge\":      {Region: \"ap-south-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.376},\n\t\t\"c5a.8xlarge\":      {Region: \"ap-south-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 0.752},\n\t\t\"c5a.12xlarge\":     {Region: \"ap-south-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.128},\n\t\t\"c5a.16xlarge\":     {Region: \"ap-south-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.504},\n\t\t\"c5a.24xlarge\":     {Region: \"ap-south-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 2.256},\n\t\t\"c5d.large\":        {Region: \"ap-south-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.099},\n\t\t\"c5d.xlarge\":       {Region: \"ap-south-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.198},\n\t\t\"c5d.2xlarge\":      {Region: \"ap-south-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.396},\n\t\t\"c5d.4xlarge\":      {Region: \"ap-south-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.792},\n\t\t\"c5d.9xlarge\":      {Region: \"ap-south-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.782},\n\t\t\"c5d.12xlarge\":     {Region: \"ap-south-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.376},\n\t\t\"c5d.18xlarge\":     {Region: \"ap-south-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.564},\n\t\t\"c5d.24xlarge\":     {Region: \"ap-south-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.752},\n\t\t\"c5d.metal\":        {Region: \"ap-south-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.752},\n\t\t\"c5n.large\":        {Region: \"ap-south-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"c5n.xlarge\":       {Region: \"ap-south-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"c5n.2xlarge\":      {Region: \"ap-south-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"c5n.4xlarge\":      {Region: \"ap-south-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"c5n.9xlarge\":      {Region: \"ap-south-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.944},\n\t\t\"c5n.18xlarge\":     {Region: \"ap-south-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c5n.metal\":        {Region: \"ap-south-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c6g.medium\":       {Region: \"ap-south-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0213},\n\t\t\"c6g.large\":        {Region: \"ap-south-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0426},\n\t\t\"c6g.xlarge\":       {Region: \"ap-south-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.0852},\n\t\t\"c6g.2xlarge\":      {Region: \"ap-south-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.1704},\n\t\t\"c6g.4xlarge\":      {Region: \"ap-south-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.3408},\n\t\t\"c6g.8xlarge\":      {Region: \"ap-south-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 0.6816},\n\t\t\"c6g.12xlarge\":     {Region: \"ap-south-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.0224},\n\t\t\"c6g.16xlarge\":     {Region: \"ap-south-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.3632},\n\t\t\"c6g.metal\":        {Region: \"ap-south-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.3632},\n\t\t\"c6gd.medium\":      {Region: \"ap-south-1\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0245},\n\t\t\"c6gd.large\":       {Region: \"ap-south-1\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.049},\n\t\t\"c6gd.xlarge\":      {Region: \"ap-south-1\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.098},\n\t\t\"c6gd.2xlarge\":     {Region: \"ap-south-1\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"c6gd.4xlarge\":     {Region: \"ap-south-1\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"c6gd.8xlarge\":     {Region: \"ap-south-1\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"c6gd.12xlarge\":    {Region: \"ap-south-1\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.176},\n\t\t\"c6gd.16xlarge\":    {Region: \"ap-south-1\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.568},\n\t\t\"c6gd.metal\":       {Region: \"ap-south-1\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.568},\n\t\t\"c6gn.medium\":      {Region: \"ap-south-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.04325},\n\t\t\"c6gn.large\":       {Region: \"ap-south-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0865},\n\t\t\"c6gn.xlarge\":      {Region: \"ap-south-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.173},\n\t\t\"c6gn.2xlarge\":     {Region: \"ap-south-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.346},\n\t\t\"c6gn.4xlarge\":     {Region: \"ap-south-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.692},\n\t\t\"c6gn.8xlarge\":     {Region: \"ap-south-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.384},\n\t\t\"c6gn.12xlarge\":    {Region: \"ap-south-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.076},\n\t\t\"c6gn.16xlarge\":    {Region: \"ap-south-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.768},\n\t\t\"c6i.large\":        {Region: \"ap-south-1\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c6i.xlarge\":       {Region: \"ap-south-1\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c6i.2xlarge\":      {Region: \"ap-south-1\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c6i.4xlarge\":      {Region: \"ap-south-1\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c6i.8xlarge\":      {Region: \"ap-south-1\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.36},\n\t\t\"c6i.12xlarge\":     {Region: \"ap-south-1\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c6i.16xlarge\":     {Region: \"ap-south-1\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.72},\n\t\t\"c6i.24xlarge\":     {Region: \"ap-south-1\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c6i.32xlarge\":     {Region: \"ap-south-1\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.44},\n\t\t\"c6i.metal\":        {Region: \"ap-south-1\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.44},\n\t\t\"d2.xlarge\":        {Region: \"ap-south-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.827},\n\t\t\"d2.2xlarge\":       {Region: \"ap-south-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.653},\n\t\t\"d2.4xlarge\":       {Region: \"ap-south-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.306},\n\t\t\"d2.8xlarge\":       {Region: \"ap-south-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.612},\n\t\t\"d3.xlarge\":        {Region: \"ap-south-1\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"d3.2xlarge\":       {Region: \"ap-south-1\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.047},\n\t\t\"d3.4xlarge\":       {Region: \"ap-south-1\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.095},\n\t\t\"d3.8xlarge\":       {Region: \"ap-south-1\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.18904},\n\t\t\"g4dn.xlarge\":      {Region: \"ap-south-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.579},\n\t\t\"g4dn.2xlarge\":     {Region: \"ap-south-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.828},\n\t\t\"g4dn.4xlarge\":     {Region: \"ap-south-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.325},\n\t\t\"g4dn.8xlarge\":     {Region: \"ap-south-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.395},\n\t\t\"g4dn.12xlarge\":    {Region: \"ap-south-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.306},\n\t\t\"g4dn.16xlarge\":    {Region: \"ap-south-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.791},\n\t\t\"g4dn.metal\":       {Region: \"ap-south-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 8.612},\n\t\t\"i2.xlarge\":        {Region: \"ap-south-1\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.967},\n\t\t\"i2.2xlarge\":       {Region: \"ap-south-1\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.933},\n\t\t\"i2.4xlarge\":       {Region: \"ap-south-1\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.867},\n\t\t\"i2.8xlarge\":       {Region: \"ap-south-1\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 7.733},\n\t\t\"i3.large\":         {Region: \"ap-south-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.177},\n\t\t\"i3.xlarge\":        {Region: \"ap-south-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.354},\n\t\t\"i3.2xlarge\":       {Region: \"ap-south-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.708},\n\t\t\"i3.4xlarge\":       {Region: \"ap-south-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.416},\n\t\t\"i3.8xlarge\":       {Region: \"ap-south-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.832},\n\t\t\"i3.16xlarge\":      {Region: \"ap-south-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.664},\n\t\t\"i3.metal\":         {Region: \"ap-south-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.664},\n\t\t\"i3en.large\":       {Region: \"ap-south-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.257},\n\t\t\"i3en.xlarge\":      {Region: \"ap-south-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.514},\n\t\t\"i3en.2xlarge\":     {Region: \"ap-south-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.028},\n\t\t\"i3en.3xlarge\":     {Region: \"ap-south-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.542},\n\t\t\"i3en.6xlarge\":     {Region: \"ap-south-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.084},\n\t\t\"i3en.12xlarge\":    {Region: \"ap-south-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.168},\n\t\t\"i3en.24xlarge\":    {Region: \"ap-south-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.336},\n\t\t\"i3en.metal\":       {Region: \"ap-south-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.336},\n\t\t\"inf1.xlarge\":      {Region: \"ap-south-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.24},\n\t\t\"inf1.2xlarge\":     {Region: \"ap-south-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.381},\n\t\t\"inf1.6xlarge\":     {Region: \"ap-south-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.241},\n\t\t\"inf1.24xlarge\":    {Region: \"ap-south-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 4.965},\n\t\t\"m4.large\":         {Region: \"ap-south-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.105},\n\t\t\"m4.xlarge\":        {Region: \"ap-south-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.21},\n\t\t\"m4.2xlarge\":       {Region: \"ap-south-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.42},\n\t\t\"m4.4xlarge\":       {Region: \"ap-south-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.84},\n\t\t\"m4.10xlarge\":      {Region: \"ap-south-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.1},\n\t\t\"m4.16xlarge\":      {Region: \"ap-south-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.36},\n\t\t\"m5.large\":         {Region: \"ap-south-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"m5.xlarge\":        {Region: \"ap-south-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"m5.2xlarge\":       {Region: \"ap-south-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"m5.4xlarge\":       {Region: \"ap-south-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"m5.8xlarge\":       {Region: \"ap-south-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"m5.12xlarge\":      {Region: \"ap-south-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"m5.16xlarge\":      {Region: \"ap-south-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"m5.24xlarge\":      {Region: \"ap-south-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"m5.metal\":         {Region: \"ap-south-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"m5a.large\":        {Region: \"ap-south-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.056},\n\t\t\"m5a.xlarge\":       {Region: \"ap-south-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.111},\n\t\t\"m5a.2xlarge\":      {Region: \"ap-south-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.222},\n\t\t\"m5a.4xlarge\":      {Region: \"ap-south-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.444},\n\t\t\"m5a.8xlarge\":      {Region: \"ap-south-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 0.889},\n\t\t\"m5a.12xlarge\":     {Region: \"ap-south-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.333},\n\t\t\"m5a.16xlarge\":     {Region: \"ap-south-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.778},\n\t\t\"m5a.24xlarge\":     {Region: \"ap-south-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 2.666},\n\t\t\"m5ad.large\":       {Region: \"ap-south-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.067},\n\t\t\"m5ad.xlarge\":      {Region: \"ap-south-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.134},\n\t\t\"m5ad.2xlarge\":     {Region: \"ap-south-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.268},\n\t\t\"m5ad.4xlarge\":     {Region: \"ap-south-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.537},\n\t\t\"m5ad.8xlarge\":     {Region: \"ap-south-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.074},\n\t\t\"m5ad.12xlarge\":    {Region: \"ap-south-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.61},\n\t\t\"m5ad.16xlarge\":    {Region: \"ap-south-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.147},\n\t\t\"m5ad.24xlarge\":    {Region: \"ap-south-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.221},\n\t\t\"m5d.large\":        {Region: \"ap-south-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.122},\n\t\t\"m5d.xlarge\":       {Region: \"ap-south-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.244},\n\t\t\"m5d.2xlarge\":      {Region: \"ap-south-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.488},\n\t\t\"m5d.4xlarge\":      {Region: \"ap-south-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.976},\n\t\t\"m5d.8xlarge\":      {Region: \"ap-south-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.952},\n\t\t\"m5d.12xlarge\":     {Region: \"ap-south-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.928},\n\t\t\"m5d.16xlarge\":     {Region: \"ap-south-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.904},\n\t\t\"m5d.24xlarge\":     {Region: \"ap-south-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"m5d.metal\":        {Region: \"ap-south-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"m6a.large\":        {Region: \"ap-south-1\", Type: \"m6a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.05555},\n\t\t\"m6a.xlarge\":       {Region: \"ap-south-1\", Type: \"m6a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1111},\n\t\t\"m6a.2xlarge\":      {Region: \"ap-south-1\", Type: \"m6a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2222},\n\t\t\"m6a.4xlarge\":      {Region: \"ap-south-1\", Type: \"m6a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.4444},\n\t\t\"m6a.8xlarge\":      {Region: \"ap-south-1\", Type: \"m6a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 0.8888},\n\t\t\"m6a.12xlarge\":     {Region: \"ap-south-1\", Type: \"m6a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.3332},\n\t\t\"m6a.16xlarge\":     {Region: \"ap-south-1\", Type: \"m6a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.7776},\n\t\t\"m6a.24xlarge\":     {Region: \"ap-south-1\", Type: \"m6a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 2.6664},\n\t\t\"m6a.32xlarge\":     {Region: \"ap-south-1\", Type: \"m6a.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 3.5552},\n\t\t\"m6a.48xlarge\":     {Region: \"ap-south-1\", Type: \"m6a.48xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 5.3328},\n\t\t\"m6a.metal\":        {Region: \"ap-south-1\", Type: \"m6a.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 5.3328},\n\t\t\"m6g.medium\":       {Region: \"ap-south-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0253},\n\t\t\"m6g.large\":        {Region: \"ap-south-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0506},\n\t\t\"m6g.xlarge\":       {Region: \"ap-south-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1012},\n\t\t\"m6g.2xlarge\":      {Region: \"ap-south-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2024},\n\t\t\"m6g.4xlarge\":      {Region: \"ap-south-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.4048},\n\t\t\"m6g.8xlarge\":      {Region: \"ap-south-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 0.8096},\n\t\t\"m6g.12xlarge\":     {Region: \"ap-south-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.2144},\n\t\t\"m6g.16xlarge\":     {Region: \"ap-south-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.6192},\n\t\t\"m6g.metal\":        {Region: \"ap-south-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.6192},\n\t\t\"m6gd.medium\":      {Region: \"ap-south-1\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0302},\n\t\t\"m6gd.large\":       {Region: \"ap-south-1\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0604},\n\t\t\"m6gd.xlarge\":      {Region: \"ap-south-1\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1208},\n\t\t\"m6gd.2xlarge\":     {Region: \"ap-south-1\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2416},\n\t\t\"m6gd.4xlarge\":     {Region: \"ap-south-1\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.4832},\n\t\t\"m6gd.8xlarge\":     {Region: \"ap-south-1\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 0.9664},\n\t\t\"m6gd.12xlarge\":    {Region: \"ap-south-1\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.4496},\n\t\t\"m6gd.16xlarge\":    {Region: \"ap-south-1\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.9328},\n\t\t\"m6gd.metal\":       {Region: \"ap-south-1\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 1.9328},\n\t\t\"m6i.large\":        {Region: \"ap-south-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"m6i.xlarge\":       {Region: \"ap-south-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"m6i.2xlarge\":      {Region: \"ap-south-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"m6i.4xlarge\":      {Region: \"ap-south-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"m6i.8xlarge\":      {Region: \"ap-south-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"m6i.12xlarge\":     {Region: \"ap-south-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"m6i.16xlarge\":     {Region: \"ap-south-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"m6i.24xlarge\":     {Region: \"ap-south-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"m6i.32xlarge\":     {Region: \"ap-south-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.464},\n\t\t\"m6i.metal\":        {Region: \"ap-south-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.464},\n\t\t\"p2.xlarge\":        {Region: \"ap-south-1\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.718},\n\t\t\"p2.8xlarge\":       {Region: \"ap-south-1\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 13.744},\n\t\t\"p2.16xlarge\":      {Region: \"ap-south-1\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 27.488},\n\t\t\"r3.large\":         {Region: \"ap-south-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.19},\n\t\t\"r3.xlarge\":        {Region: \"ap-south-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.379},\n\t\t\"r3.2xlarge\":       {Region: \"ap-south-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.758},\n\t\t\"r3.4xlarge\":       {Region: \"ap-south-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.516},\n\t\t\"r3.8xlarge\":       {Region: \"ap-south-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.032},\n\t\t\"r4.large\":         {Region: \"ap-south-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.137},\n\t\t\"r4.xlarge\":        {Region: \"ap-south-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.274},\n\t\t\"r4.2xlarge\":       {Region: \"ap-south-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.548},\n\t\t\"r4.4xlarge\":       {Region: \"ap-south-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.096},\n\t\t\"r4.8xlarge\":       {Region: \"ap-south-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.192},\n\t\t\"r4.16xlarge\":      {Region: \"ap-south-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.384},\n\t\t\"r5.large\":         {Region: \"ap-south-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"r5.xlarge\":        {Region: \"ap-south-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.26},\n\t\t\"r5.2xlarge\":       {Region: \"ap-south-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"r5.4xlarge\":       {Region: \"ap-south-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.04},\n\t\t\"r5.8xlarge\":       {Region: \"ap-south-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.08},\n\t\t\"r5.12xlarge\":      {Region: \"ap-south-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.12},\n\t\t\"r5.16xlarge\":      {Region: \"ap-south-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.16},\n\t\t\"r5.24xlarge\":      {Region: \"ap-south-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.24},\n\t\t\"r5.metal\":         {Region: \"ap-south-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.24},\n\t\t\"r5a.large\":        {Region: \"ap-south-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.072},\n\t\t\"r5a.xlarge\":       {Region: \"ap-south-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.143},\n\t\t\"r5a.2xlarge\":      {Region: \"ap-south-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.286},\n\t\t\"r5a.4xlarge\":      {Region: \"ap-south-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.572},\n\t\t\"r5a.8xlarge\":      {Region: \"ap-south-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.144},\n\t\t\"r5a.12xlarge\":     {Region: \"ap-south-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.716},\n\t\t\"r5a.16xlarge\":     {Region: \"ap-south-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.288},\n\t\t\"r5a.24xlarge\":     {Region: \"ap-south-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.432},\n\t\t\"r5ad.large\":       {Region: \"ap-south-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.083},\n\t\t\"r5ad.xlarge\":      {Region: \"ap-south-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.166},\n\t\t\"r5ad.2xlarge\":     {Region: \"ap-south-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.332},\n\t\t\"r5ad.4xlarge\":     {Region: \"ap-south-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.664},\n\t\t\"r5ad.8xlarge\":     {Region: \"ap-south-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.329},\n\t\t\"r5ad.12xlarge\":    {Region: \"ap-south-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.993},\n\t\t\"r5ad.16xlarge\":    {Region: \"ap-south-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.658},\n\t\t\"r5ad.24xlarge\":    {Region: \"ap-south-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.986},\n\t\t\"r5d.large\":        {Region: \"ap-south-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.151},\n\t\t\"r5d.xlarge\":       {Region: \"ap-south-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.302},\n\t\t\"r5d.2xlarge\":      {Region: \"ap-south-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.604},\n\t\t\"r5d.4xlarge\":      {Region: \"ap-south-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.208},\n\t\t\"r5d.8xlarge\":      {Region: \"ap-south-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.416},\n\t\t\"r5d.12xlarge\":     {Region: \"ap-south-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.624},\n\t\t\"r5d.16xlarge\":     {Region: \"ap-south-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.832},\n\t\t\"r5d.24xlarge\":     {Region: \"ap-south-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r5d.metal\":        {Region: \"ap-south-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r5n.large\":        {Region: \"ap-south-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.153},\n\t\t\"r5n.xlarge\":       {Region: \"ap-south-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.306},\n\t\t\"r5n.2xlarge\":      {Region: \"ap-south-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.612},\n\t\t\"r5n.4xlarge\":      {Region: \"ap-south-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.224},\n\t\t\"r5n.8xlarge\":      {Region: \"ap-south-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"r5n.12xlarge\":     {Region: \"ap-south-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"r5n.16xlarge\":     {Region: \"ap-south-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"r5n.24xlarge\":     {Region: \"ap-south-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"r5n.metal\":        {Region: \"ap-south-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"r6g.medium\":       {Region: \"ap-south-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0325},\n\t\t\"r6g.large\":        {Region: \"ap-south-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.065},\n\t\t\"r6g.xlarge\":       {Region: \"ap-south-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"r6g.2xlarge\":      {Region: \"ap-south-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.26},\n\t\t\"r6g.4xlarge\":      {Region: \"ap-south-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"r6g.8xlarge\":      {Region: \"ap-south-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.04},\n\t\t\"r6g.12xlarge\":     {Region: \"ap-south-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.56},\n\t\t\"r6g.16xlarge\":     {Region: \"ap-south-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.08},\n\t\t\"r6g.metal\":        {Region: \"ap-south-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.08},\n\t\t\"r6gd.medium\":      {Region: \"ap-south-1\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0374},\n\t\t\"r6gd.large\":       {Region: \"ap-south-1\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0747},\n\t\t\"r6gd.xlarge\":      {Region: \"ap-south-1\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1495},\n\t\t\"r6gd.2xlarge\":     {Region: \"ap-south-1\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.299},\n\t\t\"r6gd.4xlarge\":     {Region: \"ap-south-1\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.5979},\n\t\t\"r6gd.8xlarge\":     {Region: \"ap-south-1\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.1958},\n\t\t\"r6gd.12xlarge\":    {Region: \"ap-south-1\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.7938},\n\t\t\"r6gd.16xlarge\":    {Region: \"ap-south-1\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.3917},\n\t\t\"r6gd.metal\":       {Region: \"ap-south-1\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.3917},\n\t\t\"r6i.large\":        {Region: \"ap-south-1\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"r6i.xlarge\":       {Region: \"ap-south-1\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.26},\n\t\t\"r6i.2xlarge\":      {Region: \"ap-south-1\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"r6i.4xlarge\":      {Region: \"ap-south-1\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.04},\n\t\t\"r6i.8xlarge\":      {Region: \"ap-south-1\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.08},\n\t\t\"r6i.12xlarge\":     {Region: \"ap-south-1\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.12},\n\t\t\"r6i.16xlarge\":     {Region: \"ap-south-1\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.16},\n\t\t\"r6i.24xlarge\":     {Region: \"ap-south-1\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.24},\n\t\t\"r6i.32xlarge\":     {Region: \"ap-south-1\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.32},\n\t\t\"r6i.metal\":        {Region: \"ap-south-1\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.32},\n\t\t\"t2.nano\":          {Region: \"ap-south-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0062},\n\t\t\"t2.micro\":         {Region: \"ap-south-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0124},\n\t\t\"t2.small\":         {Region: \"ap-south-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0248},\n\t\t\"t2.medium\":        {Region: \"ap-south-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0496},\n\t\t\"t2.large\":         {Region: \"ap-south-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0992},\n\t\t\"t2.xlarge\":        {Region: \"ap-south-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1984},\n\t\t\"t2.2xlarge\":       {Region: \"ap-south-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3968},\n\t\t\"t3.nano\":          {Region: \"ap-south-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0056},\n\t\t\"t3.micro\":         {Region: \"ap-south-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0112},\n\t\t\"t3.small\":         {Region: \"ap-south-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0224},\n\t\t\"t3.medium\":        {Region: \"ap-south-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0448},\n\t\t\"t3.large\":         {Region: \"ap-south-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0896},\n\t\t\"t3.xlarge\":        {Region: \"ap-south-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1792},\n\t\t\"t3.2xlarge\":       {Region: \"ap-south-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3584},\n\t\t\"t3a.nano\":         {Region: \"ap-south-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0031},\n\t\t\"t3a.micro\":        {Region: \"ap-south-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0062},\n\t\t\"t3a.small\":        {Region: \"ap-south-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0123},\n\t\t\"t3a.medium\":       {Region: \"ap-south-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0246},\n\t\t\"t3a.large\":        {Region: \"ap-south-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0493},\n\t\t\"t3a.xlarge\":       {Region: \"ap-south-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.0986},\n\t\t\"t3a.2xlarge\":      {Region: \"ap-south-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.1971},\n\t\t\"t4g.nano\":         {Region: \"ap-south-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0028},\n\t\t\"t4g.micro\":        {Region: \"ap-south-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0056},\n\t\t\"t4g.small\":        {Region: \"ap-south-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0112},\n\t\t\"t4g.medium\":       {Region: \"ap-south-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0224},\n\t\t\"t4g.large\":        {Region: \"ap-south-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0448},\n\t\t\"t4g.xlarge\":       {Region: \"ap-south-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.0896},\n\t\t\"t4g.2xlarge\":      {Region: \"ap-south-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.1792},\n\t\t\"u-6tb1.56xlarge\":  {Region: \"ap-south-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 47.87676},\n\t\t\"u-6tb1.112xlarge\": {Region: \"ap-south-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 56.333},\n\t\t\"x1.16xlarge\":      {Region: \"ap-south-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.881},\n\t\t\"x1.32xlarge\":      {Region: \"ap-south-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 13.762},\n\t\t\"x1e.xlarge\":       {Region: \"ap-south-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.86},\n\t\t\"x1e.2xlarge\":      {Region: \"ap-south-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.72},\n\t\t\"x1e.4xlarge\":      {Region: \"ap-south-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.44},\n\t\t\"x1e.8xlarge\":      {Region: \"ap-south-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.88},\n\t\t\"x1e.16xlarge\":     {Region: \"ap-south-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.76},\n\t\t\"x1e.32xlarge\":     {Region: \"ap-south-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 27.52},\n\t\t\"x2idn.16xlarge\":   {Region: \"ap-south-1\", Type: \"x2idn.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.881},\n\t\t\"x2idn.24xlarge\":   {Region: \"ap-south-1\", Type: \"x2idn.24xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.3215},\n\t\t\"x2idn.32xlarge\":   {Region: \"ap-south-1\", Type: \"x2idn.32xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 13.762},\n\t\t\"z1d.large\":        {Region: \"ap-south-1\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"z1d.xlarge\":       {Region: \"ap-south-1\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"z1d.2xlarge\":      {Region: \"ap-south-1\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"z1d.3xlarge\":      {Region: \"ap-south-1\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.176},\n\t\t\"z1d.6xlarge\":      {Region: \"ap-south-1\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.352},\n\t\t\"z1d.12xlarge\":     {Region: \"ap-south-1\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.704},\n\t\t\"z1d.metal\":        {Region: \"ap-south-1\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.704},\n\t},\n\t\"ap-southeast-1\": {\n\t\t\"a1.medium\":         {Region: \"ap-southeast-1\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0294},\n\t\t\"a1.large\":          {Region: \"ap-southeast-1\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0588},\n\t\t\"a1.xlarge\":         {Region: \"ap-southeast-1\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1176},\n\t\t\"a1.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2352},\n\t\t\"a1.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.4704},\n\t\t\"a1.metal\":          {Region: \"ap-southeast-1\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.47},\n\t\t\"c1.medium\":         {Region: \"ap-southeast-1\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.164},\n\t\t\"c1.xlarge\":         {Region: \"ap-southeast-1\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.655},\n\t\t\"c3.large\":          {Region: \"ap-southeast-1\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.132},\n\t\t\"c3.xlarge\":         {Region: \"ap-southeast-1\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.265},\n\t\t\"c3.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.529},\n\t\t\"c3.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.058},\n\t\t\"c3.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.117},\n\t\t\"c4.large\":          {Region: \"ap-southeast-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.115},\n\t\t\"c4.xlarge\":         {Region: \"ap-southeast-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.231},\n\t\t\"c4.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.462},\n\t\t\"c4.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.924},\n\t\t\"c4.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.848},\n\t\t\"c5.large\":          {Region: \"ap-southeast-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.098},\n\t\t\"c5.xlarge\":         {Region: \"ap-southeast-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"c5.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"c5.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"c5.9xlarge\":        {Region: \"ap-southeast-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.764},\n\t\t\"c5.12xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.352},\n\t\t\"c5.18xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.528},\n\t\t\"c5.24xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.704},\n\t\t\"c5.metal\":          {Region: \"ap-southeast-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.704},\n\t\t\"c5a.large\":         {Region: \"ap-southeast-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.088},\n\t\t\"c5a.xlarge\":        {Region: \"ap-southeast-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.176},\n\t\t\"c5a.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.352},\n\t\t\"c5a.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.704},\n\t\t\"c5a.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.408},\n\t\t\"c5a.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.112},\n\t\t\"c5a.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.816},\n\t\t\"c5a.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.224},\n\t\t\"c5ad.large\":        {Region: \"ap-southeast-1\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"c5ad.xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"c5ad.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"c5ad.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"c5ad.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"c5ad.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"c5ad.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"c5ad.24xlarge\":     {Region: \"ap-southeast-1\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c5d.large\":         {Region: \"ap-southeast-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.112},\n\t\t\"c5d.xlarge\":        {Region: \"ap-southeast-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.224},\n\t\t\"c5d.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.448},\n\t\t\"c5d.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.896},\n\t\t\"c5d.9xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"c5d.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"c5d.18xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"c5d.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"c5d.metal\":         {Region: \"ap-southeast-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"c5n.large\":         {Region: \"ap-southeast-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.124},\n\t\t\"c5n.xlarge\":        {Region: \"ap-southeast-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.248},\n\t\t\"c5n.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.496},\n\t\t\"c5n.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.992},\n\t\t\"c5n.9xlarge\":       {Region: \"ap-southeast-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.232},\n\t\t\"c5n.18xlarge\":      {Region: \"ap-southeast-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"c5n.metal\":         {Region: \"ap-southeast-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"c6g.medium\":        {Region: \"ap-southeast-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0392},\n\t\t\"c6g.large\":         {Region: \"ap-southeast-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0784},\n\t\t\"c6g.xlarge\":        {Region: \"ap-southeast-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1568},\n\t\t\"c6g.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3136},\n\t\t\"c6g.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6272},\n\t\t\"c6g.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.2544},\n\t\t\"c6g.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.8816},\n\t\t\"c6g.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.5088},\n\t\t\"c6g.metal\":         {Region: \"ap-southeast-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.5088},\n\t\t\"c6gd.medium\":       {Region: \"ap-southeast-1\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.045},\n\t\t\"c6gd.large\":        {Region: \"ap-southeast-1\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.09},\n\t\t\"c6gd.xlarge\":       {Region: \"ap-southeast-1\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.18},\n\t\t\"c6gd.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.36},\n\t\t\"c6gd.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.72},\n\t\t\"c6gd.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.44},\n\t\t\"c6gd.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.16},\n\t\t\"c6gd.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"c6gd.metal\":        {Region: \"ap-southeast-1\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"c6gn.medium\":       {Region: \"ap-southeast-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0495},\n\t\t\"c6gn.large\":        {Region: \"ap-southeast-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.099},\n\t\t\"c6gn.xlarge\":       {Region: \"ap-southeast-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.198},\n\t\t\"c6gn.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.396},\n\t\t\"c6gn.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.792},\n\t\t\"c6gn.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.584},\n\t\t\"c6gn.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.376},\n\t\t\"c6gn.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.168},\n\t\t\"c6i.large\":         {Region: \"ap-southeast-1\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.098},\n\t\t\"c6i.xlarge\":        {Region: \"ap-southeast-1\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"c6i.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"c6i.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"c6i.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.568},\n\t\t\"c6i.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.352},\n\t\t\"c6i.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.136},\n\t\t\"c6i.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.704},\n\t\t\"c6i.32xlarge\":      {Region: \"ap-southeast-1\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.272},\n\t\t\"c6i.metal\":         {Region: \"ap-southeast-1\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.272},\n\t\t\"d2.xlarge\":         {Region: \"ap-southeast-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.87},\n\t\t\"d2.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.74},\n\t\t\"d2.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.48},\n\t\t\"d2.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.96},\n\t\t\"d3.xlarge\":         {Region: \"ap-southeast-1\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.626},\n\t\t\"d3.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.252},\n\t\t\"d3.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.504},\n\t\t\"d3.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 5.00808},\n\t\t\"g2.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.0},\n\t\t\"g2.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 4.0},\n\t\t\"g3.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.67},\n\t\t\"g3.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 3.34},\n\t\t\"g3.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 6.68},\n\t\t\"g4dn.xlarge\":       {Region: \"ap-southeast-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.736},\n\t\t\"g4dn.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.052},\n\t\t\"g4dn.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.685},\n\t\t\"g4dn.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 3.045},\n\t\t\"g4dn.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 5.474},\n\t\t\"g4dn.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 6.089},\n\t\t\"g4dn.metal\":        {Region: \"ap-southeast-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 10.948},\n\t\t\"g5g.xlarge\":        {Region: \"ap-southeast-1\", Type: \"g5g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.5877},\n\t\t\"g5g.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"g5g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.778},\n\t\t\"g5g.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"g5g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.1586},\n\t\t\"g5g.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"g5g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 1.9198},\n\t\t\"g5g.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"g5g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 3.8395},\n\t\t\"g5g.metal\":         {Region: \"ap-southeast-1\", Type: \"g5g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 3.8395},\n\t\t\"hs1.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"hs1.8xlarge\", Memory: kresource.MustParse(\"119808Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 5.57},\n\t\t\"i2.xlarge\":         {Region: \"ap-southeast-1\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.018},\n\t\t\"i2.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.035},\n\t\t\"i2.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.07},\n\t\t\"i2.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.14},\n\t\t\"i3.large\":          {Region: \"ap-southeast-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.187},\n\t\t\"i3.xlarge\":         {Region: \"ap-southeast-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.374},\n\t\t\"i3.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.748},\n\t\t\"i3.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.496},\n\t\t\"i3.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.992},\n\t\t\"i3.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.984},\n\t\t\"i3.metal\":          {Region: \"ap-southeast-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.984},\n\t\t\"i3en.large\":        {Region: \"ap-southeast-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.271},\n\t\t\"i3en.xlarge\":       {Region: \"ap-southeast-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.542},\n\t\t\"i3en.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.084},\n\t\t\"i3en.3xlarge\":      {Region: \"ap-southeast-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.626},\n\t\t\"i3en.6xlarge\":      {Region: \"ap-southeast-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.252},\n\t\t\"i3en.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.504},\n\t\t\"i3en.24xlarge\":     {Region: \"ap-southeast-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.008},\n\t\t\"i3en.metal\":        {Region: \"ap-southeast-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.008},\n\t\t\"inf1.xlarge\":       {Region: \"ap-southeast-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.308},\n\t\t\"inf1.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.489},\n\t\t\"inf1.6xlarge\":      {Region: \"ap-southeast-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.594},\n\t\t\"inf1.24xlarge\":     {Region: \"ap-southeast-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 6.376},\n\t\t\"m1.small\":          {Region: \"ap-southeast-1\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.058},\n\t\t\"m1.medium\":         {Region: \"ap-southeast-1\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.117},\n\t\t\"m1.large\":          {Region: \"ap-southeast-1\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.233},\n\t\t\"m1.xlarge\":         {Region: \"ap-southeast-1\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.467},\n\t\t\"m2.xlarge\":         {Region: \"ap-southeast-1\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.296},\n\t\t\"m2.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"m2.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.183},\n\t\t\"m3.medium\":         {Region: \"ap-southeast-1\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.098},\n\t\t\"m3.large\":          {Region: \"ap-southeast-1\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"m3.xlarge\":         {Region: \"ap-southeast-1\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"m3.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"m4.large\":          {Region: \"ap-southeast-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.125},\n\t\t\"m4.xlarge\":         {Region: \"ap-southeast-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.25},\n\t\t\"m4.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5},\n\t\t\"m4.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"m4.10xlarge\":       {Region: \"ap-southeast-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.5},\n\t\t\"m4.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.0},\n\t\t\"m5.large\":          {Region: \"ap-southeast-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"m5.xlarge\":         {Region: \"ap-southeast-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"m5.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"m5.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.96},\n\t\t\"m5.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.92},\n\t\t\"m5.12xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m5.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.84},\n\t\t\"m5.24xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m5.metal\":          {Region: \"ap-southeast-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m5a.large\":         {Region: \"ap-southeast-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"m5a.xlarge\":        {Region: \"ap-southeast-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"m5a.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"m5a.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"m5a.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"m5a.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"m5a.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"m5a.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.184},\n\t\t\"m5ad.large\":        {Region: \"ap-southeast-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.129},\n\t\t\"m5ad.xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.258},\n\t\t\"m5ad.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.516},\n\t\t\"m5ad.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.032},\n\t\t\"m5ad.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"m5ad.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.096},\n\t\t\"m5ad.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"m5ad.24xlarge\":     {Region: \"ap-southeast-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.192},\n\t\t\"m5d.large\":         {Region: \"ap-southeast-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.141},\n\t\t\"m5d.xlarge\":        {Region: \"ap-southeast-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.282},\n\t\t\"m5d.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.564},\n\t\t\"m5d.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.128},\n\t\t\"m5d.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.256},\n\t\t\"m5d.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.384},\n\t\t\"m5d.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.512},\n\t\t\"m5d.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.768},\n\t\t\"m5d.metal\":         {Region: \"ap-southeast-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.768},\n\t\t\"m5dn.large\":        {Region: \"ap-southeast-1\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"m5dn.xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"m5dn.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"m5dn.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"m5dn.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"m5dn.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"m5dn.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"m5dn.24xlarge\":     {Region: \"ap-southeast-1\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"m5dn.metal\":        {Region: \"ap-southeast-1\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"m5n.large\":         {Region: \"ap-southeast-1\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.146},\n\t\t\"m5n.xlarge\":        {Region: \"ap-southeast-1\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.292},\n\t\t\"m5n.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.584},\n\t\t\"m5n.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.168},\n\t\t\"m5n.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.336},\n\t\t\"m5n.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.504},\n\t\t\"m5n.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.672},\n\t\t\"m5n.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.008},\n\t\t\"m5n.metal\":         {Region: \"ap-southeast-1\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.008},\n\t\t\"m5zn.large\":        {Region: \"ap-southeast-1\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2065},\n\t\t\"m5zn.xlarge\":       {Region: \"ap-southeast-1\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.413},\n\t\t\"m5zn.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.826},\n\t\t\"m5zn.3xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.239},\n\t\t\"m5zn.6xlarge\":      {Region: \"ap-southeast-1\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.478},\n\t\t\"m5zn.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.956},\n\t\t\"m5zn.metal\":        {Region: \"ap-southeast-1\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.956},\n\t\t\"m6g.medium\":        {Region: \"ap-southeast-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.048},\n\t\t\"m6g.large\":         {Region: \"ap-southeast-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m6g.xlarge\":        {Region: \"ap-southeast-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m6g.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m6g.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m6g.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m6g.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m6g.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m6g.metal\":         {Region: \"ap-southeast-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m6gd.medium\":       {Region: \"ap-southeast-1\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0565},\n\t\t\"m6gd.large\":        {Region: \"ap-southeast-1\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"m6gd.xlarge\":       {Region: \"ap-southeast-1\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"m6gd.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"m6gd.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"m6gd.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"m6gd.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"m6gd.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"m6gd.metal\":        {Region: \"ap-southeast-1\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"m6i.large\":         {Region: \"ap-southeast-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"m6i.xlarge\":        {Region: \"ap-southeast-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"m6i.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"m6i.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.96},\n\t\t\"m6i.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.92},\n\t\t\"m6i.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m6i.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.84},\n\t\t\"m6i.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m6i.32xlarge\":      {Region: \"ap-southeast-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.68},\n\t\t\"m6i.metal\":         {Region: \"ap-southeast-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.68},\n\t\t\"p2.xlarge\":         {Region: \"ap-southeast-1\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.718},\n\t\t\"p2.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 13.744},\n\t\t\"p2.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 27.488},\n\t\t\"p3.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 4.234},\n\t\t\"p3.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 16.936},\n\t\t\"p3.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 33.872},\n\t\t\"r3.large\":          {Region: \"ap-southeast-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"r3.xlarge\":         {Region: \"ap-southeast-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.399},\n\t\t\"r3.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.798},\n\t\t\"r3.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.596},\n\t\t\"r3.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"r4.large\":          {Region: \"ap-southeast-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.16},\n\t\t\"r4.xlarge\":         {Region: \"ap-southeast-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.32},\n\t\t\"r4.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.64},\n\t\t\"r4.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.28},\n\t\t\"r4.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.56},\n\t\t\"r4.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.12},\n\t\t\"r5.large\":          {Region: \"ap-southeast-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r5.xlarge\":         {Region: \"ap-southeast-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r5.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r5.4xlarge\":        {Region: \"ap-southeast-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r5.8xlarge\":        {Region: \"ap-southeast-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r5.12xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r5.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r5.24xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5.metal\":          {Region: \"ap-southeast-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5a.large\":         {Region: \"ap-southeast-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"r5a.xlarge\":        {Region: \"ap-southeast-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"r5a.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"r5a.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"r5a.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"r5a.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"r5a.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"r5a.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"r5ad.large\":        {Region: \"ap-southeast-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.159},\n\t\t\"r5ad.xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.318},\n\t\t\"r5ad.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.636},\n\t\t\"r5ad.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.272},\n\t\t\"r5ad.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"r5ad.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.816},\n\t\t\"r5ad.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"r5ad.24xlarge\":     {Region: \"ap-southeast-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.632},\n\t\t\"r5b.large\":         {Region: \"ap-southeast-1\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.178},\n\t\t\"r5b.xlarge\":        {Region: \"ap-southeast-1\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.356},\n\t\t\"r5b.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.712},\n\t\t\"r5b.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.424},\n\t\t\"r5b.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.848},\n\t\t\"r5b.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.272},\n\t\t\"r5b.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.696},\n\t\t\"r5b.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5b.metal\":         {Region: \"ap-southeast-1\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5d.large\":         {Region: \"ap-southeast-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.174},\n\t\t\"r5d.xlarge\":        {Region: \"ap-southeast-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.348},\n\t\t\"r5d.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.696},\n\t\t\"r5d.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.392},\n\t\t\"r5d.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.784},\n\t\t\"r5d.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"r5d.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.568},\n\t\t\"r5d.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.352},\n\t\t\"r5d.metal\":         {Region: \"ap-southeast-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.352},\n\t\t\"r5dn.large\":        {Region: \"ap-southeast-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"r5dn.xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"r5dn.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"r5dn.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.6},\n\t\t\"r5dn.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"r5dn.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.8},\n\t\t\"r5dn.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.4},\n\t\t\"r5dn.24xlarge\":     {Region: \"ap-southeast-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.6},\n\t\t\"r5dn.metal\":        {Region: \"ap-southeast-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.6},\n\t\t\"r5n.large\":         {Region: \"ap-southeast-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.178},\n\t\t\"r5n.xlarge\":        {Region: \"ap-southeast-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.356},\n\t\t\"r5n.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.712},\n\t\t\"r5n.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.424},\n\t\t\"r5n.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.848},\n\t\t\"r5n.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.272},\n\t\t\"r5n.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.696},\n\t\t\"r5n.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5n.metal\":         {Region: \"ap-southeast-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r6g.medium\":        {Region: \"ap-southeast-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0608},\n\t\t\"r6g.large\":         {Region: \"ap-southeast-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1216},\n\t\t\"r6g.xlarge\":        {Region: \"ap-southeast-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2432},\n\t\t\"r6g.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4864},\n\t\t\"r6g.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9728},\n\t\t\"r6g.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.9456},\n\t\t\"r6g.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.9184},\n\t\t\"r6g.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8912},\n\t\t\"r6g.metal\":         {Region: \"ap-southeast-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8912},\n\t\t\"r6gd.medium\":       {Region: \"ap-southeast-1\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0695},\n\t\t\"r6gd.large\":        {Region: \"ap-southeast-1\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.139},\n\t\t\"r6gd.xlarge\":       {Region: \"ap-southeast-1\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.278},\n\t\t\"r6gd.2xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.556},\n\t\t\"r6gd.4xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.112},\n\t\t\"r6gd.8xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.224},\n\t\t\"r6gd.12xlarge\":     {Region: \"ap-southeast-1\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"r6gd.16xlarge\":     {Region: \"ap-southeast-1\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.448},\n\t\t\"r6gd.metal\":        {Region: \"ap-southeast-1\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.448},\n\t\t\"r6i.large\":         {Region: \"ap-southeast-1\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r6i.xlarge\":        {Region: \"ap-southeast-1\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r6i.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r6i.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r6i.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r6i.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r6i.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r6i.24xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r6i.32xlarge\":      {Region: \"ap-southeast-1\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.728},\n\t\t\"r6i.metal\":         {Region: \"ap-southeast-1\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.728},\n\t\t\"t1.micro\":          {Region: \"ap-southeast-1\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.02},\n\t\t\"t2.nano\":           {Region: \"ap-southeast-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0073},\n\t\t\"t2.micro\":          {Region: \"ap-southeast-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0146},\n\t\t\"t2.small\":          {Region: \"ap-southeast-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0292},\n\t\t\"t2.medium\":         {Region: \"ap-southeast-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0584},\n\t\t\"t2.large\":          {Region: \"ap-southeast-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1168},\n\t\t\"t2.xlarge\":         {Region: \"ap-southeast-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2336},\n\t\t\"t2.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4672},\n\t\t\"t3.nano\":           {Region: \"ap-southeast-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0066},\n\t\t\"t3.micro\":          {Region: \"ap-southeast-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0132},\n\t\t\"t3.small\":          {Region: \"ap-southeast-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0264},\n\t\t\"t3.medium\":         {Region: \"ap-southeast-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0528},\n\t\t\"t3.large\":          {Region: \"ap-southeast-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1056},\n\t\t\"t3.xlarge\":         {Region: \"ap-southeast-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2112},\n\t\t\"t3.2xlarge\":        {Region: \"ap-southeast-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4224},\n\t\t\"t3a.nano\":          {Region: \"ap-southeast-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0059},\n\t\t\"t3a.micro\":         {Region: \"ap-southeast-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0118},\n\t\t\"t3a.small\":         {Region: \"ap-southeast-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0236},\n\t\t\"t3a.medium\":        {Region: \"ap-southeast-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0472},\n\t\t\"t3a.large\":         {Region: \"ap-southeast-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0944},\n\t\t\"t3a.xlarge\":        {Region: \"ap-southeast-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1888},\n\t\t\"t3a.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3776},\n\t\t\"t4g.nano\":          {Region: \"ap-southeast-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0053},\n\t\t\"t4g.micro\":         {Region: \"ap-southeast-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0106},\n\t\t\"t4g.small\":         {Region: \"ap-southeast-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0212},\n\t\t\"t4g.medium\":        {Region: \"ap-southeast-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0424},\n\t\t\"t4g.large\":         {Region: \"ap-southeast-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0848},\n\t\t\"t4g.xlarge\":        {Region: \"ap-southeast-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1696},\n\t\t\"t4g.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3392},\n\t\t\"u-12tb1.112xlarge\": {Region: \"ap-southeast-1\", Type: \"u-12tb1.112xlarge\", Memory: kresource.MustParse(\"12582912Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 131.733},\n\t\t\"u-6tb1.56xlarge\":   {Region: \"ap-southeast-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 55.9796},\n\t\t\"u-6tb1.112xlarge\":  {Region: \"ap-southeast-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 65.867},\n\t\t\"u-9tb1.112xlarge\":  {Region: \"ap-southeast-1\", Type: \"u-9tb1.112xlarge\", Memory: kresource.MustParse(\"9437184Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 98.8},\n\t\t\"x1.16xlarge\":       {Region: \"ap-southeast-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.671},\n\t\t\"x1.32xlarge\":       {Region: \"ap-southeast-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 19.341},\n\t\t\"x1e.xlarge\":        {Region: \"ap-southeast-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.209},\n\t\t\"x1e.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.418},\n\t\t\"x1e.4xlarge\":       {Region: \"ap-southeast-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.836},\n\t\t\"x1e.8xlarge\":       {Region: \"ap-southeast-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.672},\n\t\t\"x1e.16xlarge\":      {Region: \"ap-southeast-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 19.344},\n\t\t\"x1e.32xlarge\":      {Region: \"ap-southeast-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 38.688},\n\t\t\"x2idn.16xlarge\":    {Region: \"ap-southeast-1\", Type: \"x2idn.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.6705},\n\t\t\"x2idn.24xlarge\":    {Region: \"ap-southeast-1\", Type: \"x2idn.24xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 14.50575},\n\t\t\"x2idn.32xlarge\":    {Region: \"ap-southeast-1\", Type: \"x2idn.32xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 19.341},\n\t\t\"x2iedn.xlarge\":     {Region: \"ap-southeast-1\", Type: \"x2iedn.xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.20881},\n\t\t\"x2iedn.2xlarge\":    {Region: \"ap-southeast-1\", Type: \"x2iedn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.41763},\n\t\t\"x2iedn.4xlarge\":    {Region: \"ap-southeast-1\", Type: \"x2iedn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.83525},\n\t\t\"x2iedn.8xlarge\":    {Region: \"ap-southeast-1\", Type: \"x2iedn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.6705},\n\t\t\"x2iedn.16xlarge\":   {Region: \"ap-southeast-1\", Type: \"x2iedn.16xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 19.341},\n\t\t\"x2iedn.24xlarge\":   {Region: \"ap-southeast-1\", Type: \"x2iedn.24xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 29.0115},\n\t\t\"x2iedn.32xlarge\":   {Region: \"ap-southeast-1\", Type: \"x2iedn.32xlarge\", Memory: kresource.MustParse(\"4194304Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 38.682},\n\t\t\"z1d.large\":         {Region: \"ap-southeast-1\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"z1d.xlarge\":        {Region: \"ap-southeast-1\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"z1d.2xlarge\":       {Region: \"ap-southeast-1\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"z1d.3xlarge\":       {Region: \"ap-southeast-1\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.356},\n\t\t\"z1d.6xlarge\":       {Region: \"ap-southeast-1\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"z1d.12xlarge\":      {Region: \"ap-southeast-1\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"z1d.metal\":         {Region: \"ap-southeast-1\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.424},\n\t},\n\t\"ap-southeast-2\": {\n\t\t\"a1.medium\":        {Region: \"ap-southeast-2\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0333},\n\t\t\"a1.large\":         {Region: \"ap-southeast-2\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0666},\n\t\t\"a1.xlarge\":        {Region: \"ap-southeast-2\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1332},\n\t\t\"a1.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2664},\n\t\t\"a1.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.5328},\n\t\t\"a1.metal\":         {Region: \"ap-southeast-2\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.533},\n\t\t\"c1.medium\":        {Region: \"ap-southeast-2\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.164},\n\t\t\"c1.xlarge\":        {Region: \"ap-southeast-2\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.655},\n\t\t\"c3.large\":         {Region: \"ap-southeast-2\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.132},\n\t\t\"c3.xlarge\":        {Region: \"ap-southeast-2\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.265},\n\t\t\"c3.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.529},\n\t\t\"c3.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.058},\n\t\t\"c3.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.117},\n\t\t\"c4.large\":         {Region: \"ap-southeast-2\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"c4.xlarge\":        {Region: \"ap-southeast-2\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.261},\n\t\t\"c4.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.522},\n\t\t\"c4.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.042},\n\t\t\"c4.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.085},\n\t\t\"c5.large\":         {Region: \"ap-southeast-2\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.111},\n\t\t\"c5.xlarge\":        {Region: \"ap-southeast-2\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.222},\n\t\t\"c5.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.444},\n\t\t\"c5.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.888},\n\t\t\"c5.9xlarge\":       {Region: \"ap-southeast-2\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.998},\n\t\t\"c5.12xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.664},\n\t\t\"c5.18xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.996},\n\t\t\"c5.24xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.328},\n\t\t\"c5.metal\":         {Region: \"ap-southeast-2\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.328},\n\t\t\"c5a.large\":        {Region: \"ap-southeast-2\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"c5a.xlarge\":       {Region: \"ap-southeast-2\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"c5a.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"c5a.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"c5a.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6},\n\t\t\"c5a.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.4},\n\t\t\"c5a.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"c5a.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.8},\n\t\t\"c5ad.large\":       {Region: \"ap-southeast-2\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"c5ad.xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"c5ad.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"c5ad.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"c5ad.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"c5ad.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"c5ad.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"c5ad.24xlarge\":    {Region: \"ap-southeast-2\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"c5d.large\":        {Region: \"ap-southeast-2\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"c5d.xlarge\":       {Region: \"ap-southeast-2\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"c5d.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"c5d.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"c5d.9xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.268},\n\t\t\"c5d.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"c5d.18xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.536},\n\t\t\"c5d.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"c5d.metal\":        {Region: \"ap-southeast-2\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"c5n.large\":        {Region: \"ap-southeast-2\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.141},\n\t\t\"c5n.xlarge\":       {Region: \"ap-southeast-2\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.282},\n\t\t\"c5n.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.564},\n\t\t\"c5n.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.128},\n\t\t\"c5n.9xlarge\":      {Region: \"ap-southeast-2\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.538},\n\t\t\"c5n.18xlarge\":     {Region: \"ap-southeast-2\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.076},\n\t\t\"c5n.metal\":        {Region: \"ap-southeast-2\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.076},\n\t\t\"c6g.medium\":       {Region: \"ap-southeast-2\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0444},\n\t\t\"c6g.large\":        {Region: \"ap-southeast-2\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0888},\n\t\t\"c6g.xlarge\":       {Region: \"ap-southeast-2\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1776},\n\t\t\"c6g.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3552},\n\t\t\"c6g.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7104},\n\t\t\"c6g.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.4208},\n\t\t\"c6g.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.1312},\n\t\t\"c6g.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8416},\n\t\t\"c6g.metal\":        {Region: \"ap-southeast-2\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8416},\n\t\t\"c6gd.medium\":      {Region: \"ap-southeast-2\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0505},\n\t\t\"c6gd.large\":       {Region: \"ap-southeast-2\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"c6gd.xlarge\":      {Region: \"ap-southeast-2\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"c6gd.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"c6gd.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"c6gd.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"c6gd.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"c6gd.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"c6gd.metal\":       {Region: \"ap-southeast-2\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"c6gn.medium\":      {Region: \"ap-southeast-2\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0565},\n\t\t\"c6gn.large\":       {Region: \"ap-southeast-2\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"c6gn.xlarge\":      {Region: \"ap-southeast-2\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"c6gn.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"c6gn.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"c6gn.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"c6gn.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"c6gn.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"c6i.large\":        {Region: \"ap-southeast-2\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.111},\n\t\t\"c6i.xlarge\":       {Region: \"ap-southeast-2\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.222},\n\t\t\"c6i.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.444},\n\t\t\"c6i.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.888},\n\t\t\"c6i.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.776},\n\t\t\"c6i.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.664},\n\t\t\"c6i.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"c6i.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.328},\n\t\t\"c6i.32xlarge\":     {Region: \"ap-southeast-2\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"c6i.metal\":        {Region: \"ap-southeast-2\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"d2.xlarge\":        {Region: \"ap-southeast-2\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.87},\n\t\t\"d2.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.74},\n\t\t\"d2.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.48},\n\t\t\"d2.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.96},\n\t\t\"d3.xlarge\":        {Region: \"ap-southeast-2\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.626},\n\t\t\"d3.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.252},\n\t\t\"d3.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.504},\n\t\t\"d3.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 5.00808},\n\t\t\"f1.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"f1.2xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.981},\n\t\t\"f1.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"f1.4xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.962},\n\t\t\"f1.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"f1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 15.848},\n\t\t\"g2.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.898},\n\t\t\"g2.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 3.592},\n\t\t\"g3.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.754},\n\t\t\"g3.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 3.508},\n\t\t\"g3.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 7.016},\n\t\t\"g3s.xlarge\":       {Region: \"ap-southeast-2\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.154},\n\t\t\"g4dn.xlarge\":      {Region: \"ap-southeast-2\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.684},\n\t\t\"g4dn.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.978},\n\t\t\"g4dn.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.566},\n\t\t\"g4dn.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.83},\n\t\t\"g4dn.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 5.087},\n\t\t\"g4dn.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.659},\n\t\t\"g4dn.metal\":       {Region: \"ap-southeast-2\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 10.174},\n\t\t\"hs1.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"hs1.8xlarge\", Memory: kresource.MustParse(\"119808Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 5.57},\n\t\t\"i2.xlarge\":        {Region: \"ap-southeast-2\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.018},\n\t\t\"i2.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.035},\n\t\t\"i2.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.07},\n\t\t\"i2.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.14},\n\t\t\"i3.large\":         {Region: \"ap-southeast-2\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.187},\n\t\t\"i3.xlarge\":        {Region: \"ap-southeast-2\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.374},\n\t\t\"i3.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.748},\n\t\t\"i3.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.496},\n\t\t\"i3.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.992},\n\t\t\"i3.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.984},\n\t\t\"i3.metal\":         {Region: \"ap-southeast-2\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.984},\n\t\t\"i3en.large\":       {Region: \"ap-southeast-2\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.271},\n\t\t\"i3en.xlarge\":      {Region: \"ap-southeast-2\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.542},\n\t\t\"i3en.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.084},\n\t\t\"i3en.3xlarge\":     {Region: \"ap-southeast-2\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.626},\n\t\t\"i3en.6xlarge\":     {Region: \"ap-southeast-2\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.252},\n\t\t\"i3en.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.504},\n\t\t\"i3en.24xlarge\":    {Region: \"ap-southeast-2\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.008},\n\t\t\"i3en.metal\":       {Region: \"ap-southeast-2\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.008},\n\t\t\"inf1.xlarge\":      {Region: \"ap-southeast-2\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.285},\n\t\t\"inf1.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.453},\n\t\t\"inf1.6xlarge\":     {Region: \"ap-southeast-2\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.475},\n\t\t\"inf1.24xlarge\":    {Region: \"ap-southeast-2\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.902},\n\t\t\"m1.small\":         {Region: \"ap-southeast-2\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.058},\n\t\t\"m1.medium\":        {Region: \"ap-southeast-2\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.117},\n\t\t\"m1.large\":         {Region: \"ap-southeast-2\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.233},\n\t\t\"m1.xlarge\":        {Region: \"ap-southeast-2\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.467},\n\t\t\"m2.xlarge\":        {Region: \"ap-southeast-2\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.296},\n\t\t\"m2.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"m2.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.183},\n\t\t\"m3.medium\":        {Region: \"ap-southeast-2\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.093},\n\t\t\"m3.large\":         {Region: \"ap-southeast-2\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"m3.xlarge\":        {Region: \"ap-southeast-2\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"m3.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.745},\n\t\t\"m4.large\":         {Region: \"ap-southeast-2\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.125},\n\t\t\"m4.xlarge\":        {Region: \"ap-southeast-2\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.25},\n\t\t\"m4.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5},\n\t\t\"m4.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"m4.10xlarge\":      {Region: \"ap-southeast-2\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.5},\n\t\t\"m4.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.0},\n\t\t\"m5.large\":         {Region: \"ap-southeast-2\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"m5.xlarge\":        {Region: \"ap-southeast-2\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"m5.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"m5.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.96},\n\t\t\"m5.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.92},\n\t\t\"m5.12xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m5.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.84},\n\t\t\"m5.24xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m5.metal\":         {Region: \"ap-southeast-2\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m5a.large\":        {Region: \"ap-southeast-2\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"m5a.xlarge\":       {Region: \"ap-southeast-2\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"m5a.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"m5a.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"m5a.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"m5a.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"m5a.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"m5a.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.184},\n\t\t\"m5ad.large\":       {Region: \"ap-southeast-2\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"m5ad.xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.26},\n\t\t\"m5ad.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"m5ad.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.04},\n\t\t\"m5ad.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.08},\n\t\t\"m5ad.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.12},\n\t\t\"m5ad.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.16},\n\t\t\"m5ad.24xlarge\":    {Region: \"ap-southeast-2\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.24},\n\t\t\"m5d.large\":        {Region: \"ap-southeast-2\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.142},\n\t\t\"m5d.xlarge\":       {Region: \"ap-southeast-2\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.284},\n\t\t\"m5d.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.568},\n\t\t\"m5d.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.136},\n\t\t\"m5d.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.272},\n\t\t\"m5d.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.408},\n\t\t\"m5d.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.544},\n\t\t\"m5d.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.816},\n\t\t\"m5d.metal\":        {Region: \"ap-southeast-2\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.816},\n\t\t\"m5zn.large\":       {Region: \"ap-southeast-2\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2065},\n\t\t\"m5zn.xlarge\":      {Region: \"ap-southeast-2\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.413},\n\t\t\"m5zn.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.826},\n\t\t\"m5zn.3xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.239},\n\t\t\"m5zn.6xlarge\":     {Region: \"ap-southeast-2\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.478},\n\t\t\"m5zn.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.956},\n\t\t\"m5zn.metal\":       {Region: \"ap-southeast-2\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.956},\n\t\t\"m6g.medium\":       {Region: \"ap-southeast-2\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.048},\n\t\t\"m6g.large\":        {Region: \"ap-southeast-2\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m6g.xlarge\":       {Region: \"ap-southeast-2\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m6g.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m6g.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m6g.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m6g.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m6g.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m6g.metal\":        {Region: \"ap-southeast-2\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m6gd.medium\":      {Region: \"ap-southeast-2\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.057},\n\t\t\"m6gd.large\":       {Region: \"ap-southeast-2\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.114},\n\t\t\"m6gd.xlarge\":      {Region: \"ap-southeast-2\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.228},\n\t\t\"m6gd.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.456},\n\t\t\"m6gd.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.912},\n\t\t\"m6gd.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.824},\n\t\t\"m6gd.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.736},\n\t\t\"m6gd.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"m6gd.metal\":       {Region: \"ap-southeast-2\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"m6i.large\":        {Region: \"ap-southeast-2\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"m6i.xlarge\":       {Region: \"ap-southeast-2\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"m6i.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"m6i.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.96},\n\t\t\"m6i.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.92},\n\t\t\"m6i.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m6i.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.84},\n\t\t\"m6i.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m6i.32xlarge\":     {Region: \"ap-southeast-2\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.68},\n\t\t\"m6i.metal\":        {Region: \"ap-southeast-2\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.68},\n\t\t\"p2.xlarge\":        {Region: \"ap-southeast-2\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.542},\n\t\t\"p2.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 12.336},\n\t\t\"p2.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 24.672},\n\t\t\"p3.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 4.234},\n\t\t\"p3.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 16.936},\n\t\t\"p3.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 33.872},\n\t\t\"r3.large\":         {Region: \"ap-southeast-2\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"r3.xlarge\":        {Region: \"ap-southeast-2\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.399},\n\t\t\"r3.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.798},\n\t\t\"r3.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.596},\n\t\t\"r3.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"r4.large\":         {Region: \"ap-southeast-2\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1596},\n\t\t\"r4.xlarge\":        {Region: \"ap-southeast-2\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3192},\n\t\t\"r4.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.6384},\n\t\t\"r4.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.2768},\n\t\t\"r4.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.5536},\n\t\t\"r4.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.1072},\n\t\t\"r5.large\":         {Region: \"ap-southeast-2\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.151},\n\t\t\"r5.xlarge\":        {Region: \"ap-southeast-2\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.302},\n\t\t\"r5.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.604},\n\t\t\"r5.4xlarge\":       {Region: \"ap-southeast-2\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.208},\n\t\t\"r5.8xlarge\":       {Region: \"ap-southeast-2\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.416},\n\t\t\"r5.12xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.624},\n\t\t\"r5.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.832},\n\t\t\"r5.24xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r5.metal\":         {Region: \"ap-southeast-2\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r5a.large\":        {Region: \"ap-southeast-2\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"r5a.xlarge\":       {Region: \"ap-southeast-2\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"r5a.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"r5a.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"r5a.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"r5a.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"r5a.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"r5a.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"r5ad.large\":       {Region: \"ap-southeast-2\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.159},\n\t\t\"r5ad.xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.318},\n\t\t\"r5ad.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.636},\n\t\t\"r5ad.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.272},\n\t\t\"r5ad.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"r5ad.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.816},\n\t\t\"r5ad.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"r5ad.24xlarge\":    {Region: \"ap-southeast-2\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.632},\n\t\t\"r5d.large\":        {Region: \"ap-southeast-2\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.174},\n\t\t\"r5d.xlarge\":       {Region: \"ap-southeast-2\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.348},\n\t\t\"r5d.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.696},\n\t\t\"r5d.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.392},\n\t\t\"r5d.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.784},\n\t\t\"r5d.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"r5d.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.568},\n\t\t\"r5d.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.352},\n\t\t\"r5d.metal\":        {Region: \"ap-southeast-2\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.352},\n\t\t\"r5dn.large\":       {Region: \"ap-southeast-2\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.204},\n\t\t\"r5dn.xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"r5dn.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.816},\n\t\t\"r5dn.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.632},\n\t\t\"r5dn.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"r5dn.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"r5dn.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"r5dn.24xlarge\":    {Region: \"ap-southeast-2\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.792},\n\t\t\"r5dn.metal\":       {Region: \"ap-southeast-2\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.792},\n\t\t\"r5n.large\":        {Region: \"ap-southeast-2\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.181},\n\t\t\"r5n.xlarge\":       {Region: \"ap-southeast-2\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.362},\n\t\t\"r5n.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.724},\n\t\t\"r5n.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.448},\n\t\t\"r5n.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.896},\n\t\t\"r5n.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.344},\n\t\t\"r5n.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"r5n.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.688},\n\t\t\"r5n.metal\":        {Region: \"ap-southeast-2\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.688},\n\t\t\"r6g.medium\":       {Region: \"ap-southeast-2\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0604},\n\t\t\"r6g.large\":        {Region: \"ap-southeast-2\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1208},\n\t\t\"r6g.xlarge\":       {Region: \"ap-southeast-2\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2416},\n\t\t\"r6g.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4832},\n\t\t\"r6g.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9664},\n\t\t\"r6g.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.9328},\n\t\t\"r6g.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.8992},\n\t\t\"r6g.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8656},\n\t\t\"r6g.metal\":        {Region: \"ap-southeast-2\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8656},\n\t\t\"r6gd.medium\":      {Region: \"ap-southeast-2\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0695},\n\t\t\"r6gd.large\":       {Region: \"ap-southeast-2\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.139},\n\t\t\"r6gd.xlarge\":      {Region: \"ap-southeast-2\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.278},\n\t\t\"r6gd.2xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.556},\n\t\t\"r6gd.4xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.112},\n\t\t\"r6gd.8xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.224},\n\t\t\"r6gd.12xlarge\":    {Region: \"ap-southeast-2\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"r6gd.16xlarge\":    {Region: \"ap-southeast-2\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.448},\n\t\t\"r6gd.metal\":       {Region: \"ap-southeast-2\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.448},\n\t\t\"r6i.large\":        {Region: \"ap-southeast-2\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.151},\n\t\t\"r6i.xlarge\":       {Region: \"ap-southeast-2\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.302},\n\t\t\"r6i.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.604},\n\t\t\"r6i.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.208},\n\t\t\"r6i.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.416},\n\t\t\"r6i.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.624},\n\t\t\"r6i.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.832},\n\t\t\"r6i.24xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r6i.32xlarge\":     {Region: \"ap-southeast-2\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.664},\n\t\t\"r6i.metal\":        {Region: \"ap-southeast-2\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.664},\n\t\t\"t1.micro\":         {Region: \"ap-southeast-2\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.02},\n\t\t\"t2.nano\":          {Region: \"ap-southeast-2\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0073},\n\t\t\"t2.micro\":         {Region: \"ap-southeast-2\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0146},\n\t\t\"t2.small\":         {Region: \"ap-southeast-2\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0292},\n\t\t\"t2.medium\":        {Region: \"ap-southeast-2\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0584},\n\t\t\"t2.large\":         {Region: \"ap-southeast-2\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1168},\n\t\t\"t2.xlarge\":        {Region: \"ap-southeast-2\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2336},\n\t\t\"t2.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4672},\n\t\t\"t3.nano\":          {Region: \"ap-southeast-2\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0066},\n\t\t\"t3.micro\":         {Region: \"ap-southeast-2\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0132},\n\t\t\"t3.small\":         {Region: \"ap-southeast-2\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0264},\n\t\t\"t3.medium\":        {Region: \"ap-southeast-2\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0528},\n\t\t\"t3.large\":         {Region: \"ap-southeast-2\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1056},\n\t\t\"t3.xlarge\":        {Region: \"ap-southeast-2\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2112},\n\t\t\"t3.2xlarge\":       {Region: \"ap-southeast-2\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4224},\n\t\t\"t3a.nano\":         {Region: \"ap-southeast-2\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0059},\n\t\t\"t3a.micro\":        {Region: \"ap-southeast-2\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0119},\n\t\t\"t3a.small\":        {Region: \"ap-southeast-2\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0238},\n\t\t\"t3a.medium\":       {Region: \"ap-southeast-2\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0475},\n\t\t\"t3a.large\":        {Region: \"ap-southeast-2\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.095},\n\t\t\"t3a.xlarge\":       {Region: \"ap-southeast-2\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1901},\n\t\t\"t3a.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3802},\n\t\t\"t4g.nano\":         {Region: \"ap-southeast-2\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0053},\n\t\t\"t4g.micro\":        {Region: \"ap-southeast-2\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0106},\n\t\t\"t4g.small\":        {Region: \"ap-southeast-2\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0212},\n\t\t\"t4g.medium\":       {Region: \"ap-southeast-2\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0424},\n\t\t\"t4g.large\":        {Region: \"ap-southeast-2\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0848},\n\t\t\"t4g.xlarge\":       {Region: \"ap-southeast-2\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1696},\n\t\t\"t4g.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3392},\n\t\t\"u-6tb1.56xlarge\":  {Region: \"ap-southeast-2\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 55.61075},\n\t\t\"u-6tb1.112xlarge\": {Region: \"ap-southeast-2\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 65.433},\n\t\t\"x1.16xlarge\":      {Region: \"ap-southeast-2\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.671},\n\t\t\"x1.32xlarge\":      {Region: \"ap-southeast-2\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 19.341},\n\t\t\"x1e.xlarge\":       {Region: \"ap-southeast-2\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.209},\n\t\t\"x1e.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.418},\n\t\t\"x1e.4xlarge\":      {Region: \"ap-southeast-2\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.836},\n\t\t\"x1e.8xlarge\":      {Region: \"ap-southeast-2\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.672},\n\t\t\"x1e.16xlarge\":     {Region: \"ap-southeast-2\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 19.344},\n\t\t\"x1e.32xlarge\":     {Region: \"ap-southeast-2\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 38.688},\n\t\t\"z1d.large\":        {Region: \"ap-southeast-2\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"z1d.xlarge\":       {Region: \"ap-southeast-2\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"z1d.2xlarge\":      {Region: \"ap-southeast-2\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"z1d.3xlarge\":      {Region: \"ap-southeast-2\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.356},\n\t\t\"z1d.6xlarge\":      {Region: \"ap-southeast-2\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"z1d.12xlarge\":     {Region: \"ap-southeast-2\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"z1d.metal\":        {Region: \"ap-southeast-2\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.424},\n\t},\n\t\"ca-central-1\": {\n\t\t\"c4.large\":      {Region: \"ca-central-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.11},\n\t\t\"c4.xlarge\":     {Region: \"ca-central-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.218},\n\t\t\"c4.2xlarge\":    {Region: \"ca-central-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.438},\n\t\t\"c4.4xlarge\":    {Region: \"ca-central-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.876},\n\t\t\"c4.8xlarge\":    {Region: \"ca-central-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.75},\n\t\t\"c5.large\":      {Region: \"ca-central-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.093},\n\t\t\"c5.xlarge\":     {Region: \"ca-central-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"c5.2xlarge\":    {Region: \"ca-central-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"c5.4xlarge\":    {Region: \"ca-central-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.744},\n\t\t\"c5.9xlarge\":    {Region: \"ca-central-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.674},\n\t\t\"c5.12xlarge\":   {Region: \"ca-central-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.232},\n\t\t\"c5.18xlarge\":   {Region: \"ca-central-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.348},\n\t\t\"c5.24xlarge\":   {Region: \"ca-central-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"c5.metal\":      {Region: \"ca-central-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"c5a.large\":     {Region: \"ca-central-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.084},\n\t\t\"c5a.xlarge\":    {Region: \"ca-central-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.168},\n\t\t\"c5a.2xlarge\":   {Region: \"ca-central-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.336},\n\t\t\"c5a.4xlarge\":   {Region: \"ca-central-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.672},\n\t\t\"c5a.8xlarge\":   {Region: \"ca-central-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.344},\n\t\t\"c5a.12xlarge\":  {Region: \"ca-central-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"c5a.16xlarge\":  {Region: \"ca-central-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"c5a.24xlarge\":  {Region: \"ca-central-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"c5d.large\":     {Region: \"ca-central-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.106},\n\t\t\"c5d.xlarge\":    {Region: \"ca-central-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.212},\n\t\t\"c5d.2xlarge\":   {Region: \"ca-central-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.424},\n\t\t\"c5d.4xlarge\":   {Region: \"ca-central-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.848},\n\t\t\"c5d.9xlarge\":   {Region: \"ca-central-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.908},\n\t\t\"c5d.12xlarge\":  {Region: \"ca-central-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"c5d.18xlarge\":  {Region: \"ca-central-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.816},\n\t\t\"c5d.24xlarge\":  {Region: \"ca-central-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"c5d.metal\":     {Region: \"ca-central-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"c5n.large\":     {Region: \"ca-central-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.118},\n\t\t\"c5n.xlarge\":    {Region: \"ca-central-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.236},\n\t\t\"c5n.2xlarge\":   {Region: \"ca-central-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.472},\n\t\t\"c5n.4xlarge\":   {Region: \"ca-central-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.944},\n\t\t\"c5n.9xlarge\":   {Region: \"ca-central-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.124},\n\t\t\"c5n.18xlarge\":  {Region: \"ca-central-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.248},\n\t\t\"c5n.metal\":     {Region: \"ca-central-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.248},\n\t\t\"c6g.medium\":    {Region: \"ca-central-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0372},\n\t\t\"c6g.large\":     {Region: \"ca-central-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0744},\n\t\t\"c6g.xlarge\":    {Region: \"ca-central-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1488},\n\t\t\"c6g.2xlarge\":   {Region: \"ca-central-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2976},\n\t\t\"c6g.4xlarge\":   {Region: \"ca-central-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.5952},\n\t\t\"c6g.8xlarge\":   {Region: \"ca-central-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.1904},\n\t\t\"c6g.12xlarge\":  {Region: \"ca-central-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.7856},\n\t\t\"c6g.16xlarge\":  {Region: \"ca-central-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.3808},\n\t\t\"c6g.metal\":     {Region: \"ca-central-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.3808},\n\t\t\"c6gd.medium\":   {Region: \"ca-central-1\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0424},\n\t\t\"c6gd.large\":    {Region: \"ca-central-1\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0848},\n\t\t\"c6gd.xlarge\":   {Region: \"ca-central-1\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1696},\n\t\t\"c6gd.2xlarge\":  {Region: \"ca-central-1\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3392},\n\t\t\"c6gd.4xlarge\":  {Region: \"ca-central-1\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6784},\n\t\t\"c6gd.8xlarge\":  {Region: \"ca-central-1\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3568},\n\t\t\"c6gd.12xlarge\": {Region: \"ca-central-1\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0352},\n\t\t\"c6gd.16xlarge\": {Region: \"ca-central-1\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7136},\n\t\t\"c6gd.metal\":    {Region: \"ca-central-1\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7136},\n\t\t\"c6gn.medium\":   {Region: \"ca-central-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0473},\n\t\t\"c6gn.large\":    {Region: \"ca-central-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0946},\n\t\t\"c6gn.xlarge\":   {Region: \"ca-central-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1892},\n\t\t\"c6gn.2xlarge\":  {Region: \"ca-central-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3784},\n\t\t\"c6gn.4xlarge\":  {Region: \"ca-central-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7568},\n\t\t\"c6gn.8xlarge\":  {Region: \"ca-central-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.5136},\n\t\t\"c6gn.12xlarge\": {Region: \"ca-central-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.2704},\n\t\t\"c6gn.16xlarge\": {Region: \"ca-central-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.0272},\n\t\t\"c6i.large\":     {Region: \"ca-central-1\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.093},\n\t\t\"c6i.xlarge\":    {Region: \"ca-central-1\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"c6i.2xlarge\":   {Region: \"ca-central-1\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"c6i.4xlarge\":   {Region: \"ca-central-1\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.744},\n\t\t\"c6i.8xlarge\":   {Region: \"ca-central-1\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.488},\n\t\t\"c6i.12xlarge\":  {Region: \"ca-central-1\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.232},\n\t\t\"c6i.16xlarge\":  {Region: \"ca-central-1\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.976},\n\t\t\"c6i.24xlarge\":  {Region: \"ca-central-1\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"c6i.32xlarge\":  {Region: \"ca-central-1\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"c6i.metal\":     {Region: \"ca-central-1\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"d2.xlarge\":     {Region: \"ca-central-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.759},\n\t\t\"d2.2xlarge\":    {Region: \"ca-central-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.518},\n\t\t\"d2.4xlarge\":    {Region: \"ca-central-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.036},\n\t\t\"d2.8xlarge\":    {Region: \"ca-central-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.072},\n\t\t\"g3.4xlarge\":    {Region: \"ca-central-1\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.416},\n\t\t\"g3.8xlarge\":    {Region: \"ca-central-1\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.832},\n\t\t\"g3.16xlarge\":   {Region: \"ca-central-1\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 5.664},\n\t\t\"g4ad.xlarge\":   {Region: \"ca-central-1\", Type: \"g4ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.42263},\n\t\t\"g4ad.2xlarge\":  {Region: \"ca-central-1\", Type: \"g4ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.60421},\n\t\t\"g4ad.4xlarge\":  {Region: \"ca-central-1\", Type: \"g4ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 0.968},\n\t\t\"g4ad.8xlarge\":  {Region: \"ca-central-1\", Type: \"g4ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 1.936},\n\t\t\"g4ad.16xlarge\": {Region: \"ca-central-1\", Type: \"g4ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 3.872},\n\t\t\"g4dn.xlarge\":   {Region: \"ca-central-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.584},\n\t\t\"g4dn.2xlarge\":  {Region: \"ca-central-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.835},\n\t\t\"g4dn.4xlarge\":  {Region: \"ca-central-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.337},\n\t\t\"g4dn.8xlarge\":  {Region: \"ca-central-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.416},\n\t\t\"g4dn.12xlarge\": {Region: \"ca-central-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.343},\n\t\t\"g4dn.16xlarge\": {Region: \"ca-central-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.832},\n\t\t\"g4dn.metal\":    {Region: \"ca-central-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 8.687},\n\t\t\"i3.large\":      {Region: \"ca-central-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"i3.xlarge\":     {Region: \"ca-central-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"i3.2xlarge\":    {Region: \"ca-central-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"i3.4xlarge\":    {Region: \"ca-central-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"i3.8xlarge\":    {Region: \"ca-central-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"i3.16xlarge\":   {Region: \"ca-central-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.504},\n\t\t\"i3.metal\":      {Region: \"ca-central-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.504},\n\t\t\"i3en.large\":    {Region: \"ca-central-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.25},\n\t\t\"i3en.xlarge\":   {Region: \"ca-central-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.5},\n\t\t\"i3en.2xlarge\":  {Region: \"ca-central-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"i3en.3xlarge\":  {Region: \"ca-central-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.5},\n\t\t\"i3en.6xlarge\":  {Region: \"ca-central-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.0},\n\t\t\"i3en.12xlarge\": {Region: \"ca-central-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.0},\n\t\t\"i3en.24xlarge\": {Region: \"ca-central-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.0},\n\t\t\"i3en.metal\":    {Region: \"ca-central-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.0},\n\t\t\"inf1.xlarge\":   {Region: \"ca-central-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.254},\n\t\t\"inf1.2xlarge\":  {Region: \"ca-central-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.403},\n\t\t\"inf1.6xlarge\":  {Region: \"ca-central-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.315},\n\t\t\"inf1.24xlarge\": {Region: \"ca-central-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.26},\n\t\t\"m4.large\":      {Region: \"ca-central-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.111},\n\t\t\"m4.xlarge\":     {Region: \"ca-central-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.222},\n\t\t\"m4.2xlarge\":    {Region: \"ca-central-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.444},\n\t\t\"m4.4xlarge\":    {Region: \"ca-central-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.888},\n\t\t\"m4.10xlarge\":   {Region: \"ca-central-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.22},\n\t\t\"m4.16xlarge\":   {Region: \"ca-central-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"m5.large\":      {Region: \"ca-central-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.107},\n\t\t\"m5.xlarge\":     {Region: \"ca-central-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.214},\n\t\t\"m5.2xlarge\":    {Region: \"ca-central-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.428},\n\t\t\"m5.4xlarge\":    {Region: \"ca-central-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.856},\n\t\t\"m5.8xlarge\":    {Region: \"ca-central-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.712},\n\t\t\"m5.12xlarge\":   {Region: \"ca-central-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.568},\n\t\t\"m5.16xlarge\":   {Region: \"ca-central-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.424},\n\t\t\"m5.24xlarge\":   {Region: \"ca-central-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"m5.metal\":      {Region: \"ca-central-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"m5a.large\":     {Region: \"ca-central-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m5a.xlarge\":    {Region: \"ca-central-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m5a.2xlarge\":   {Region: \"ca-central-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m5a.4xlarge\":   {Region: \"ca-central-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m5a.8xlarge\":   {Region: \"ca-central-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m5a.12xlarge\":  {Region: \"ca-central-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m5a.16xlarge\":  {Region: \"ca-central-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m5a.24xlarge\":  {Region: \"ca-central-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m5ad.large\":    {Region: \"ca-central-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.115},\n\t\t\"m5ad.xlarge\":   {Region: \"ca-central-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.23},\n\t\t\"m5ad.2xlarge\":  {Region: \"ca-central-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.46},\n\t\t\"m5ad.4xlarge\":  {Region: \"ca-central-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.92},\n\t\t\"m5ad.8xlarge\":  {Region: \"ca-central-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.84},\n\t\t\"m5ad.12xlarge\": {Region: \"ca-central-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.76},\n\t\t\"m5ad.16xlarge\": {Region: \"ca-central-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.68},\n\t\t\"m5ad.24xlarge\": {Region: \"ca-central-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"m5d.large\":     {Region: \"ca-central-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"m5d.xlarge\":    {Region: \"ca-central-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"m5d.2xlarge\":   {Region: \"ca-central-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"m5d.4xlarge\":   {Region: \"ca-central-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"m5d.8xlarge\":   {Region: \"ca-central-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"m5d.12xlarge\":  {Region: \"ca-central-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"m5d.16xlarge\":  {Region: \"ca-central-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"m5d.24xlarge\":  {Region: \"ca-central-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"m5d.metal\":     {Region: \"ca-central-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"m6g.medium\":    {Region: \"ca-central-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0428},\n\t\t\"m6g.large\":     {Region: \"ca-central-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0856},\n\t\t\"m6g.xlarge\":    {Region: \"ca-central-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1712},\n\t\t\"m6g.2xlarge\":   {Region: \"ca-central-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3424},\n\t\t\"m6g.4xlarge\":   {Region: \"ca-central-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6848},\n\t\t\"m6g.8xlarge\":   {Region: \"ca-central-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3696},\n\t\t\"m6g.12xlarge\":  {Region: \"ca-central-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0544},\n\t\t\"m6g.16xlarge\":  {Region: \"ca-central-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7392},\n\t\t\"m6g.metal\":     {Region: \"ca-central-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7392},\n\t\t\"m6i.large\":     {Region: \"ca-central-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.107},\n\t\t\"m6i.xlarge\":    {Region: \"ca-central-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.214},\n\t\t\"m6i.2xlarge\":   {Region: \"ca-central-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.428},\n\t\t\"m6i.4xlarge\":   {Region: \"ca-central-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.856},\n\t\t\"m6i.8xlarge\":   {Region: \"ca-central-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.712},\n\t\t\"m6i.12xlarge\":  {Region: \"ca-central-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.568},\n\t\t\"m6i.16xlarge\":  {Region: \"ca-central-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.424},\n\t\t\"m6i.24xlarge\":  {Region: \"ca-central-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"m6i.32xlarge\":  {Region: \"ca-central-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.848},\n\t\t\"m6i.metal\":     {Region: \"ca-central-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.848},\n\t\t\"p3.2xlarge\":    {Region: \"ca-central-1\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 3.366},\n\t\t\"p3.8xlarge\":    {Region: \"ca-central-1\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 13.464},\n\t\t\"p3.16xlarge\":   {Region: \"ca-central-1\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 26.928},\n\t\t\"r4.large\":      {Region: \"ca-central-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.146},\n\t\t\"r4.xlarge\":     {Region: \"ca-central-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.292},\n\t\t\"r4.2xlarge\":    {Region: \"ca-central-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.584},\n\t\t\"r4.4xlarge\":    {Region: \"ca-central-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.168},\n\t\t\"r4.8xlarge\":    {Region: \"ca-central-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.336},\n\t\t\"r4.16xlarge\":   {Region: \"ca-central-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.672},\n\t\t\"r5.large\":      {Region: \"ca-central-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.138},\n\t\t\"r5.xlarge\":     {Region: \"ca-central-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.276},\n\t\t\"r5.2xlarge\":    {Region: \"ca-central-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.552},\n\t\t\"r5.4xlarge\":    {Region: \"ca-central-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.104},\n\t\t\"r5.8xlarge\":    {Region: \"ca-central-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.208},\n\t\t\"r5.12xlarge\":   {Region: \"ca-central-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.312},\n\t\t\"r5.16xlarge\":   {Region: \"ca-central-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.416},\n\t\t\"r5.24xlarge\":   {Region: \"ca-central-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.624},\n\t\t\"r5.metal\":      {Region: \"ca-central-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.624},\n\t\t\"r5a.large\":     {Region: \"ca-central-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.124},\n\t\t\"r5a.xlarge\":    {Region: \"ca-central-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.248},\n\t\t\"r5a.2xlarge\":   {Region: \"ca-central-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.496},\n\t\t\"r5a.4xlarge\":   {Region: \"ca-central-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.992},\n\t\t\"r5a.8xlarge\":   {Region: \"ca-central-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.984},\n\t\t\"r5a.12xlarge\":  {Region: \"ca-central-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.976},\n\t\t\"r5a.16xlarge\":  {Region: \"ca-central-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.968},\n\t\t\"r5a.24xlarge\":  {Region: \"ca-central-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"r5ad.large\":    {Region: \"ca-central-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.144},\n\t\t\"r5ad.xlarge\":   {Region: \"ca-central-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.288},\n\t\t\"r5ad.2xlarge\":  {Region: \"ca-central-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.576},\n\t\t\"r5ad.4xlarge\":  {Region: \"ca-central-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.152},\n\t\t\"r5ad.8xlarge\":  {Region: \"ca-central-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"r5ad.12xlarge\": {Region: \"ca-central-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"r5ad.16xlarge\": {Region: \"ca-central-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"r5ad.24xlarge\": {Region: \"ca-central-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.912},\n\t\t\"r5d.large\":     {Region: \"ca-central-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.158},\n\t\t\"r5d.xlarge\":    {Region: \"ca-central-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.316},\n\t\t\"r5d.2xlarge\":   {Region: \"ca-central-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.632},\n\t\t\"r5d.4xlarge\":   {Region: \"ca-central-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.264},\n\t\t\"r5d.8xlarge\":   {Region: \"ca-central-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.528},\n\t\t\"r5d.12xlarge\":  {Region: \"ca-central-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.792},\n\t\t\"r5d.16xlarge\":  {Region: \"ca-central-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.056},\n\t\t\"r5d.24xlarge\":  {Region: \"ca-central-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.584},\n\t\t\"r5d.metal\":     {Region: \"ca-central-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.584},\n\t\t\"r5n.large\":     {Region: \"ca-central-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.163},\n\t\t\"r5n.xlarge\":    {Region: \"ca-central-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.326},\n\t\t\"r5n.2xlarge\":   {Region: \"ca-central-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.652},\n\t\t\"r5n.4xlarge\":   {Region: \"ca-central-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.304},\n\t\t\"r5n.8xlarge\":   {Region: \"ca-central-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.608},\n\t\t\"r5n.12xlarge\":  {Region: \"ca-central-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.912},\n\t\t\"r5n.16xlarge\":  {Region: \"ca-central-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.216},\n\t\t\"r5n.24xlarge\":  {Region: \"ca-central-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.824},\n\t\t\"r5n.metal\":     {Region: \"ca-central-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.824},\n\t\t\"r6g.medium\":    {Region: \"ca-central-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0552},\n\t\t\"r6g.large\":     {Region: \"ca-central-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1104},\n\t\t\"r6g.xlarge\":    {Region: \"ca-central-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2208},\n\t\t\"r6g.2xlarge\":   {Region: \"ca-central-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4416},\n\t\t\"r6g.4xlarge\":   {Region: \"ca-central-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8832},\n\t\t\"r6g.8xlarge\":   {Region: \"ca-central-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.7664},\n\t\t\"r6g.12xlarge\":  {Region: \"ca-central-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.6496},\n\t\t\"r6g.16xlarge\":  {Region: \"ca-central-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.5328},\n\t\t\"r6g.metal\":     {Region: \"ca-central-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.5328},\n\t\t\"r6gd.medium\":   {Region: \"ca-central-1\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0632},\n\t\t\"r6gd.large\":    {Region: \"ca-central-1\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1264},\n\t\t\"r6gd.xlarge\":   {Region: \"ca-central-1\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2528},\n\t\t\"r6gd.2xlarge\":  {Region: \"ca-central-1\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5056},\n\t\t\"r6gd.4xlarge\":  {Region: \"ca-central-1\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.0112},\n\t\t\"r6gd.8xlarge\":  {Region: \"ca-central-1\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.0224},\n\t\t\"r6gd.12xlarge\": {Region: \"ca-central-1\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.0336},\n\t\t\"r6gd.16xlarge\": {Region: \"ca-central-1\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.0448},\n\t\t\"r6gd.metal\":    {Region: \"ca-central-1\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.0448},\n\t\t\"r6i.large\":     {Region: \"ca-central-1\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.138},\n\t\t\"r6i.xlarge\":    {Region: \"ca-central-1\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.276},\n\t\t\"r6i.2xlarge\":   {Region: \"ca-central-1\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.552},\n\t\t\"r6i.4xlarge\":   {Region: \"ca-central-1\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.104},\n\t\t\"r6i.8xlarge\":   {Region: \"ca-central-1\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.208},\n\t\t\"r6i.12xlarge\":  {Region: \"ca-central-1\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.312},\n\t\t\"r6i.16xlarge\":  {Region: \"ca-central-1\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.416},\n\t\t\"r6i.24xlarge\":  {Region: \"ca-central-1\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.624},\n\t\t\"r6i.32xlarge\":  {Region: \"ca-central-1\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.832},\n\t\t\"r6i.metal\":     {Region: \"ca-central-1\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.832},\n\t\t\"t2.nano\":       {Region: \"ca-central-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0064},\n\t\t\"t2.micro\":      {Region: \"ca-central-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0128},\n\t\t\"t2.small\":      {Region: \"ca-central-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0256},\n\t\t\"t2.medium\":     {Region: \"ca-central-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0512},\n\t\t\"t2.large\":      {Region: \"ca-central-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1024},\n\t\t\"t2.xlarge\":     {Region: \"ca-central-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2048},\n\t\t\"t2.2xlarge\":    {Region: \"ca-central-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4096},\n\t\t\"t3.nano\":       {Region: \"ca-central-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0058},\n\t\t\"t3.micro\":      {Region: \"ca-central-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0116},\n\t\t\"t3.small\":      {Region: \"ca-central-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0232},\n\t\t\"t3.medium\":     {Region: \"ca-central-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0464},\n\t\t\"t3.large\":      {Region: \"ca-central-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0928},\n\t\t\"t3.xlarge\":     {Region: \"ca-central-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1856},\n\t\t\"t3.2xlarge\":    {Region: \"ca-central-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3712},\n\t\t\"t3a.nano\":      {Region: \"ca-central-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0052},\n\t\t\"t3a.micro\":     {Region: \"ca-central-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0104},\n\t\t\"t3a.small\":     {Region: \"ca-central-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0209},\n\t\t\"t3a.medium\":    {Region: \"ca-central-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0418},\n\t\t\"t3a.large\":     {Region: \"ca-central-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0835},\n\t\t\"t3a.xlarge\":    {Region: \"ca-central-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"t3a.2xlarge\":   {Region: \"ca-central-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3341},\n\t\t\"t4g.nano\":      {Region: \"ca-central-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0046},\n\t\t\"t4g.micro\":     {Region: \"ca-central-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0092},\n\t\t\"t4g.small\":     {Region: \"ca-central-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0184},\n\t\t\"t4g.medium\":    {Region: \"ca-central-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0368},\n\t\t\"t4g.large\":     {Region: \"ca-central-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0736},\n\t\t\"t4g.xlarge\":    {Region: \"ca-central-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1472},\n\t\t\"t4g.2xlarge\":   {Region: \"ca-central-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2944},\n\t\t\"x1.16xlarge\":   {Region: \"ca-central-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 7.336},\n\t\t\"x1.32xlarge\":   {Region: \"ca-central-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 14.672},\n\t\t\"x1e.xlarge\":    {Region: \"ca-central-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.917},\n\t\t\"x1e.2xlarge\":   {Region: \"ca-central-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.834},\n\t\t\"x1e.4xlarge\":   {Region: \"ca-central-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.667},\n\t\t\"x1e.8xlarge\":   {Region: \"ca-central-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 7.334},\n\t\t\"x1e.16xlarge\":  {Region: \"ca-central-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 14.669},\n\t\t\"x1e.32xlarge\":  {Region: \"ca-central-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 29.338},\n\t},\n\t\"eu-central-1\": {\n\t\t\"a1.medium\":         {Region: \"eu-central-1\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0291},\n\t\t\"a1.large\":          {Region: \"eu-central-1\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0582},\n\t\t\"a1.xlarge\":         {Region: \"eu-central-1\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1164},\n\t\t\"a1.2xlarge\":        {Region: \"eu-central-1\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2328},\n\t\t\"a1.4xlarge\":        {Region: \"eu-central-1\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.4656},\n\t\t\"a1.metal\":          {Region: \"eu-central-1\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.466},\n\t\t\"c3.large\":          {Region: \"eu-central-1\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.129},\n\t\t\"c3.xlarge\":         {Region: \"eu-central-1\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.258},\n\t\t\"c3.2xlarge\":        {Region: \"eu-central-1\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.516},\n\t\t\"c3.4xlarge\":        {Region: \"eu-central-1\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.032},\n\t\t\"c3.8xlarge\":        {Region: \"eu-central-1\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"c4.large\":          {Region: \"eu-central-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.114},\n\t\t\"c4.xlarge\":         {Region: \"eu-central-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.227},\n\t\t\"c4.2xlarge\":        {Region: \"eu-central-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.454},\n\t\t\"c4.4xlarge\":        {Region: \"eu-central-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.909},\n\t\t\"c4.8xlarge\":        {Region: \"eu-central-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.817},\n\t\t\"c5.large\":          {Region: \"eu-central-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.097},\n\t\t\"c5.xlarge\":         {Region: \"eu-central-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.194},\n\t\t\"c5.2xlarge\":        {Region: \"eu-central-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.388},\n\t\t\"c5.4xlarge\":        {Region: \"eu-central-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.776},\n\t\t\"c5.9xlarge\":        {Region: \"eu-central-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.746},\n\t\t\"c5.12xlarge\":       {Region: \"eu-central-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.328},\n\t\t\"c5.18xlarge\":       {Region: \"eu-central-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.492},\n\t\t\"c5.24xlarge\":       {Region: \"eu-central-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.656},\n\t\t\"c5.metal\":          {Region: \"eu-central-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.656},\n\t\t\"c5a.large\":         {Region: \"eu-central-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.087},\n\t\t\"c5a.xlarge\":        {Region: \"eu-central-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.174},\n\t\t\"c5a.2xlarge\":       {Region: \"eu-central-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.348},\n\t\t\"c5a.4xlarge\":       {Region: \"eu-central-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.696},\n\t\t\"c5a.8xlarge\":       {Region: \"eu-central-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.392},\n\t\t\"c5a.12xlarge\":      {Region: \"eu-central-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.088},\n\t\t\"c5a.16xlarge\":      {Region: \"eu-central-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.784},\n\t\t\"c5a.24xlarge\":      {Region: \"eu-central-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"c5ad.large\":        {Region: \"eu-central-1\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"c5ad.xlarge\":       {Region: \"eu-central-1\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"c5ad.2xlarge\":      {Region: \"eu-central-1\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"c5ad.4xlarge\":      {Region: \"eu-central-1\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"c5ad.8xlarge\":      {Region: \"eu-central-1\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6},\n\t\t\"c5ad.12xlarge\":     {Region: \"eu-central-1\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.4},\n\t\t\"c5ad.16xlarge\":     {Region: \"eu-central-1\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"c5ad.24xlarge\":     {Region: \"eu-central-1\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.8},\n\t\t\"c5d.large\":         {Region: \"eu-central-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.111},\n\t\t\"c5d.xlarge\":        {Region: \"eu-central-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.222},\n\t\t\"c5d.2xlarge\":       {Region: \"eu-central-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.444},\n\t\t\"c5d.4xlarge\":       {Region: \"eu-central-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.888},\n\t\t\"c5d.9xlarge\":       {Region: \"eu-central-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.998},\n\t\t\"c5d.12xlarge\":      {Region: \"eu-central-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.664},\n\t\t\"c5d.18xlarge\":      {Region: \"eu-central-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.996},\n\t\t\"c5d.24xlarge\":      {Region: \"eu-central-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.328},\n\t\t\"c5d.metal\":         {Region: \"eu-central-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.328},\n\t\t\"c5n.large\":         {Region: \"eu-central-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.123},\n\t\t\"c5n.xlarge\":        {Region: \"eu-central-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.246},\n\t\t\"c5n.2xlarge\":       {Region: \"eu-central-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.492},\n\t\t\"c5n.4xlarge\":       {Region: \"eu-central-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.984},\n\t\t\"c5n.9xlarge\":       {Region: \"eu-central-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.214},\n\t\t\"c5n.18xlarge\":      {Region: \"eu-central-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.428},\n\t\t\"c5n.metal\":         {Region: \"eu-central-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.428},\n\t\t\"c6g.medium\":        {Region: \"eu-central-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0388},\n\t\t\"c6g.large\":         {Region: \"eu-central-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0776},\n\t\t\"c6g.xlarge\":        {Region: \"eu-central-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1552},\n\t\t\"c6g.2xlarge\":       {Region: \"eu-central-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3104},\n\t\t\"c6g.4xlarge\":       {Region: \"eu-central-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6208},\n\t\t\"c6g.8xlarge\":       {Region: \"eu-central-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.2416},\n\t\t\"c6g.12xlarge\":      {Region: \"eu-central-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.8624},\n\t\t\"c6g.16xlarge\":      {Region: \"eu-central-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.4832},\n\t\t\"c6g.metal\":         {Region: \"eu-central-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.4832},\n\t\t\"c6gd.medium\":       {Region: \"eu-central-1\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0445},\n\t\t\"c6gd.large\":        {Region: \"eu-central-1\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.089},\n\t\t\"c6gd.xlarge\":       {Region: \"eu-central-1\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.178},\n\t\t\"c6gd.2xlarge\":      {Region: \"eu-central-1\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.356},\n\t\t\"c6gd.4xlarge\":      {Region: \"eu-central-1\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.712},\n\t\t\"c6gd.8xlarge\":      {Region: \"eu-central-1\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.424},\n\t\t\"c6gd.12xlarge\":     {Region: \"eu-central-1\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.136},\n\t\t\"c6gd.16xlarge\":     {Region: \"eu-central-1\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.848},\n\t\t\"c6gd.metal\":        {Region: \"eu-central-1\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.848},\n\t\t\"c6gn.medium\":       {Region: \"eu-central-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0493},\n\t\t\"c6gn.large\":        {Region: \"eu-central-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0986},\n\t\t\"c6gn.xlarge\":       {Region: \"eu-central-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1972},\n\t\t\"c6gn.2xlarge\":      {Region: \"eu-central-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3944},\n\t\t\"c6gn.4xlarge\":      {Region: \"eu-central-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7888},\n\t\t\"c6gn.8xlarge\":      {Region: \"eu-central-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.5776},\n\t\t\"c6gn.12xlarge\":     {Region: \"eu-central-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.3664},\n\t\t\"c6gn.16xlarge\":     {Region: \"eu-central-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.1552},\n\t\t\"d2.xlarge\":         {Region: \"eu-central-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.794},\n\t\t\"d2.2xlarge\":        {Region: \"eu-central-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.588},\n\t\t\"d2.4xlarge\":        {Region: \"eu-central-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.176},\n\t\t\"d2.8xlarge\":        {Region: \"eu-central-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.352},\n\t\t\"d3.xlarge\":         {Region: \"eu-central-1\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.658},\n\t\t\"d3.2xlarge\":        {Region: \"eu-central-1\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.316},\n\t\t\"d3.4xlarge\":        {Region: \"eu-central-1\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.632},\n\t\t\"d3.8xlarge\":        {Region: \"eu-central-1\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 5.26448},\n\t\t\"f1.2xlarge\":        {Region: \"eu-central-1\", Type: \"f1.2xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.734},\n\t\t\"f1.4xlarge\":        {Region: \"eu-central-1\", Type: \"f1.4xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.468},\n\t\t\"f1.16xlarge\":       {Region: \"eu-central-1\", Type: \"f1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.872},\n\t\t\"g2.2xlarge\":        {Region: \"eu-central-1\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.772},\n\t\t\"g2.8xlarge\":        {Region: \"eu-central-1\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 3.088},\n\t\t\"g3.4xlarge\":        {Region: \"eu-central-1\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.425},\n\t\t\"g3.8xlarge\":        {Region: \"eu-central-1\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.85},\n\t\t\"g3.16xlarge\":       {Region: \"eu-central-1\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 5.7},\n\t\t\"g3s.xlarge\":        {Region: \"eu-central-1\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.938},\n\t\t\"g4ad.xlarge\":       {Region: \"eu-central-1\", Type: \"g4ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.47327},\n\t\t\"g4ad.2xlarge\":      {Region: \"eu-central-1\", Type: \"g4ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.67662},\n\t\t\"g4ad.4xlarge\":      {Region: \"eu-central-1\", Type: \"g4ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.084},\n\t\t\"g4ad.8xlarge\":      {Region: \"eu-central-1\", Type: \"g4ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.168},\n\t\t\"g4ad.16xlarge\":     {Region: \"eu-central-1\", Type: \"g4ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 4.336},\n\t\t\"g4dn.xlarge\":       {Region: \"eu-central-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.658},\n\t\t\"g4dn.2xlarge\":      {Region: \"eu-central-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.94},\n\t\t\"g4dn.4xlarge\":      {Region: \"eu-central-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.505},\n\t\t\"g4dn.8xlarge\":      {Region: \"eu-central-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.72},\n\t\t\"g4dn.12xlarge\":     {Region: \"eu-central-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.89},\n\t\t\"g4dn.16xlarge\":     {Region: \"eu-central-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.44},\n\t\t\"g4dn.metal\":        {Region: \"eu-central-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 9.78},\n\t\t\"i2.xlarge\":         {Region: \"eu-central-1\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.013},\n\t\t\"i2.2xlarge\":        {Region: \"eu-central-1\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.026},\n\t\t\"i2.4xlarge\":        {Region: \"eu-central-1\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.051},\n\t\t\"i2.8xlarge\":        {Region: \"eu-central-1\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.102},\n\t\t\"i3.large\":          {Region: \"eu-central-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"i3.xlarge\":         {Region: \"eu-central-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"i3.2xlarge\":        {Region: \"eu-central-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.744},\n\t\t\"i3.4xlarge\":        {Region: \"eu-central-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.488},\n\t\t\"i3.8xlarge\":        {Region: \"eu-central-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.976},\n\t\t\"i3.16xlarge\":       {Region: \"eu-central-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"i3.metal\":          {Region: \"eu-central-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"i3en.large\":        {Region: \"eu-central-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.27},\n\t\t\"i3en.xlarge\":       {Region: \"eu-central-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.54},\n\t\t\"i3en.2xlarge\":      {Region: \"eu-central-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.08},\n\t\t\"i3en.3xlarge\":      {Region: \"eu-central-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.62},\n\t\t\"i3en.6xlarge\":      {Region: \"eu-central-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.24},\n\t\t\"i3en.12xlarge\":     {Region: \"eu-central-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.48},\n\t\t\"i3en.24xlarge\":     {Region: \"eu-central-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.96},\n\t\t\"i3en.metal\":        {Region: \"eu-central-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.96},\n\t\t\"inf1.xlarge\":       {Region: \"eu-central-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.285},\n\t\t\"inf1.2xlarge\":      {Region: \"eu-central-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.453},\n\t\t\"inf1.6xlarge\":      {Region: \"eu-central-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.475},\n\t\t\"inf1.24xlarge\":     {Region: \"eu-central-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.902},\n\t\t\"m3.medium\":         {Region: \"eu-central-1\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.079},\n\t\t\"m3.large\":          {Region: \"eu-central-1\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.158},\n\t\t\"m3.xlarge\":         {Region: \"eu-central-1\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.315},\n\t\t\"m3.2xlarge\":        {Region: \"eu-central-1\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.632},\n\t\t\"m4.large\":          {Region: \"eu-central-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"m4.xlarge\":         {Region: \"eu-central-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"m4.2xlarge\":        {Region: \"eu-central-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"m4.4xlarge\":        {Region: \"eu-central-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.96},\n\t\t\"m4.10xlarge\":       {Region: \"eu-central-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.4},\n\t\t\"m4.16xlarge\":       {Region: \"eu-central-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.84},\n\t\t\"m5.large\":          {Region: \"eu-central-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.115},\n\t\t\"m5.xlarge\":         {Region: \"eu-central-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.23},\n\t\t\"m5.2xlarge\":        {Region: \"eu-central-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.46},\n\t\t\"m5.4xlarge\":        {Region: \"eu-central-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.92},\n\t\t\"m5.8xlarge\":        {Region: \"eu-central-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.84},\n\t\t\"m5.12xlarge\":       {Region: \"eu-central-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.76},\n\t\t\"m5.16xlarge\":       {Region: \"eu-central-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.68},\n\t\t\"m5.24xlarge\":       {Region: \"eu-central-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"m5.metal\":          {Region: \"eu-central-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"m5a.large\":         {Region: \"eu-central-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.104},\n\t\t\"m5a.xlarge\":        {Region: \"eu-central-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.208},\n\t\t\"m5a.2xlarge\":       {Region: \"eu-central-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.416},\n\t\t\"m5a.4xlarge\":       {Region: \"eu-central-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.832},\n\t\t\"m5a.8xlarge\":       {Region: \"eu-central-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.664},\n\t\t\"m5a.12xlarge\":      {Region: \"eu-central-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.496},\n\t\t\"m5a.16xlarge\":      {Region: \"eu-central-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.328},\n\t\t\"m5a.24xlarge\":      {Region: \"eu-central-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"m5ad.large\":        {Region: \"eu-central-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.125},\n\t\t\"m5ad.xlarge\":       {Region: \"eu-central-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.25},\n\t\t\"m5ad.2xlarge\":      {Region: \"eu-central-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5},\n\t\t\"m5ad.4xlarge\":      {Region: \"eu-central-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"m5ad.8xlarge\":      {Region: \"eu-central-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"m5ad.12xlarge\":     {Region: \"eu-central-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.0},\n\t\t\"m5ad.16xlarge\":     {Region: \"eu-central-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.0},\n\t\t\"m5ad.24xlarge\":     {Region: \"eu-central-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.0},\n\t\t\"m5d.large\":         {Region: \"eu-central-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"m5d.xlarge\":        {Region: \"eu-central-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"m5d.2xlarge\":       {Region: \"eu-central-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"m5d.4xlarge\":       {Region: \"eu-central-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"m5d.8xlarge\":       {Region: \"eu-central-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"m5d.12xlarge\":      {Region: \"eu-central-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"m5d.16xlarge\":      {Region: \"eu-central-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"m5d.24xlarge\":      {Region: \"eu-central-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"m5d.metal\":         {Region: \"eu-central-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"m5dn.large\":        {Region: \"eu-central-1\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.162},\n\t\t\"m5dn.xlarge\":       {Region: \"eu-central-1\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.324},\n\t\t\"m5dn.2xlarge\":      {Region: \"eu-central-1\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.648},\n\t\t\"m5dn.4xlarge\":      {Region: \"eu-central-1\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.296},\n\t\t\"m5dn.8xlarge\":      {Region: \"eu-central-1\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"m5dn.12xlarge\":     {Region: \"eu-central-1\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"m5dn.16xlarge\":     {Region: \"eu-central-1\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.184},\n\t\t\"m5dn.24xlarge\":     {Region: \"eu-central-1\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.776},\n\t\t\"m5dn.metal\":        {Region: \"eu-central-1\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.776},\n\t\t\"m5n.large\":         {Region: \"eu-central-1\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.141},\n\t\t\"m5n.xlarge\":        {Region: \"eu-central-1\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.282},\n\t\t\"m5n.2xlarge\":       {Region: \"eu-central-1\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.564},\n\t\t\"m5n.4xlarge\":       {Region: \"eu-central-1\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.128},\n\t\t\"m5n.8xlarge\":       {Region: \"eu-central-1\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.256},\n\t\t\"m5n.12xlarge\":      {Region: \"eu-central-1\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.384},\n\t\t\"m5n.16xlarge\":      {Region: \"eu-central-1\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.512},\n\t\t\"m5n.24xlarge\":      {Region: \"eu-central-1\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.768},\n\t\t\"m5n.metal\":         {Region: \"eu-central-1\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.768},\n\t\t\"m5zn.large\":        {Region: \"eu-central-1\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1979},\n\t\t\"m5zn.xlarge\":       {Region: \"eu-central-1\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3957},\n\t\t\"m5zn.2xlarge\":      {Region: \"eu-central-1\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.7914},\n\t\t\"m5zn.3xlarge\":      {Region: \"eu-central-1\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.1872},\n\t\t\"m5zn.6xlarge\":      {Region: \"eu-central-1\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.3743},\n\t\t\"m5zn.12xlarge\":     {Region: \"eu-central-1\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.7486},\n\t\t\"m5zn.metal\":        {Region: \"eu-central-1\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.7486},\n\t\t\"m6g.medium\":        {Region: \"eu-central-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.046},\n\t\t\"m6g.large\":         {Region: \"eu-central-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.092},\n\t\t\"m6g.xlarge\":        {Region: \"eu-central-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.184},\n\t\t\"m6g.2xlarge\":       {Region: \"eu-central-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.368},\n\t\t\"m6g.4xlarge\":       {Region: \"eu-central-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.736},\n\t\t\"m6g.8xlarge\":       {Region: \"eu-central-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.472},\n\t\t\"m6g.12xlarge\":      {Region: \"eu-central-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.208},\n\t\t\"m6g.16xlarge\":      {Region: \"eu-central-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.944},\n\t\t\"m6g.metal\":         {Region: \"eu-central-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.944},\n\t\t\"m6gd.medium\":       {Region: \"eu-central-1\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0545},\n\t\t\"m6gd.large\":        {Region: \"eu-central-1\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.109},\n\t\t\"m6gd.xlarge\":       {Region: \"eu-central-1\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.218},\n\t\t\"m6gd.2xlarge\":      {Region: \"eu-central-1\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.436},\n\t\t\"m6gd.4xlarge\":      {Region: \"eu-central-1\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.872},\n\t\t\"m6gd.8xlarge\":      {Region: \"eu-central-1\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.744},\n\t\t\"m6gd.12xlarge\":     {Region: \"eu-central-1\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.616},\n\t\t\"m6gd.16xlarge\":     {Region: \"eu-central-1\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.488},\n\t\t\"m6gd.metal\":        {Region: \"eu-central-1\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.488},\n\t\t\"m6i.large\":         {Region: \"eu-central-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.115},\n\t\t\"m6i.xlarge\":        {Region: \"eu-central-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.23},\n\t\t\"m6i.2xlarge\":       {Region: \"eu-central-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.46},\n\t\t\"m6i.4xlarge\":       {Region: \"eu-central-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.92},\n\t\t\"m6i.8xlarge\":       {Region: \"eu-central-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.84},\n\t\t\"m6i.12xlarge\":      {Region: \"eu-central-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.76},\n\t\t\"m6i.16xlarge\":      {Region: \"eu-central-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.68},\n\t\t\"m6i.24xlarge\":      {Region: \"eu-central-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"m6i.32xlarge\":      {Region: \"eu-central-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.36},\n\t\t\"m6i.metal\":         {Region: \"eu-central-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.36},\n\t\t\"p2.xlarge\":         {Region: \"eu-central-1\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.326},\n\t\t\"p2.8xlarge\":        {Region: \"eu-central-1\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 10.608},\n\t\t\"p2.16xlarge\":       {Region: \"eu-central-1\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 21.216},\n\t\t\"p3.2xlarge\":        {Region: \"eu-central-1\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 3.823},\n\t\t\"p3.8xlarge\":        {Region: \"eu-central-1\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 15.292},\n\t\t\"p3.16xlarge\":       {Region: \"eu-central-1\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 30.584},\n\t\t\"p4d.24xlarge\":      {Region: \"eu-central-1\", Type: \"p4d.24xlarge\", Memory: kresource.MustParse(\"1179648Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 40.94475},\n\t\t\"r3.large\":          {Region: \"eu-central-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"r3.xlarge\":         {Region: \"eu-central-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"r3.2xlarge\":        {Region: \"eu-central-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"r3.4xlarge\":        {Region: \"eu-central-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.6},\n\t\t\"r3.8xlarge\":        {Region: \"eu-central-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.201},\n\t\t\"r4.large\":          {Region: \"eu-central-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.16005},\n\t\t\"r4.xlarge\":         {Region: \"eu-central-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3201},\n\t\t\"r4.2xlarge\":        {Region: \"eu-central-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.6402},\n\t\t\"r4.4xlarge\":        {Region: \"eu-central-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.2804},\n\t\t\"r4.8xlarge\":        {Region: \"eu-central-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.5608},\n\t\t\"r4.16xlarge\":       {Region: \"eu-central-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.1216},\n\t\t\"r5.large\":          {Region: \"eu-central-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r5.xlarge\":         {Region: \"eu-central-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r5.2xlarge\":        {Region: \"eu-central-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r5.4xlarge\":        {Region: \"eu-central-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r5.8xlarge\":        {Region: \"eu-central-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r5.12xlarge\":       {Region: \"eu-central-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r5.16xlarge\":       {Region: \"eu-central-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r5.24xlarge\":       {Region: \"eu-central-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5.metal\":          {Region: \"eu-central-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5a.large\":         {Region: \"eu-central-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.137},\n\t\t\"r5a.xlarge\":        {Region: \"eu-central-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.274},\n\t\t\"r5a.2xlarge\":       {Region: \"eu-central-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.548},\n\t\t\"r5a.4xlarge\":       {Region: \"eu-central-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.096},\n\t\t\"r5a.8xlarge\":       {Region: \"eu-central-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.192},\n\t\t\"r5a.12xlarge\":      {Region: \"eu-central-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.288},\n\t\t\"r5a.16xlarge\":      {Region: \"eu-central-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.384},\n\t\t\"r5a.24xlarge\":      {Region: \"eu-central-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.576},\n\t\t\"r5ad.large\":        {Region: \"eu-central-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.158},\n\t\t\"r5ad.xlarge\":       {Region: \"eu-central-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.316},\n\t\t\"r5ad.2xlarge\":      {Region: \"eu-central-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.632},\n\t\t\"r5ad.4xlarge\":      {Region: \"eu-central-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.264},\n\t\t\"r5ad.8xlarge\":      {Region: \"eu-central-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.528},\n\t\t\"r5ad.12xlarge\":     {Region: \"eu-central-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.792},\n\t\t\"r5ad.16xlarge\":     {Region: \"eu-central-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.056},\n\t\t\"r5ad.24xlarge\":     {Region: \"eu-central-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.584},\n\t\t\"r5b.large\":         {Region: \"eu-central-1\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.178},\n\t\t\"r5b.xlarge\":        {Region: \"eu-central-1\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.356},\n\t\t\"r5b.2xlarge\":       {Region: \"eu-central-1\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.712},\n\t\t\"r5b.4xlarge\":       {Region: \"eu-central-1\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.424},\n\t\t\"r5b.8xlarge\":       {Region: \"eu-central-1\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.848},\n\t\t\"r5b.12xlarge\":      {Region: \"eu-central-1\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.272},\n\t\t\"r5b.16xlarge\":      {Region: \"eu-central-1\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.696},\n\t\t\"r5b.24xlarge\":      {Region: \"eu-central-1\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5b.metal\":         {Region: \"eu-central-1\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5d.large\":         {Region: \"eu-central-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.173},\n\t\t\"r5d.xlarge\":        {Region: \"eu-central-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.346},\n\t\t\"r5d.2xlarge\":       {Region: \"eu-central-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.692},\n\t\t\"r5d.4xlarge\":       {Region: \"eu-central-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.384},\n\t\t\"r5d.8xlarge\":       {Region: \"eu-central-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.768},\n\t\t\"r5d.12xlarge\":      {Region: \"eu-central-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.152},\n\t\t\"r5d.16xlarge\":      {Region: \"eu-central-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.536},\n\t\t\"r5d.24xlarge\":      {Region: \"eu-central-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.304},\n\t\t\"r5d.metal\":         {Region: \"eu-central-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.304},\n\t\t\"r5dn.large\":        {Region: \"eu-central-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.199},\n\t\t\"r5dn.xlarge\":       {Region: \"eu-central-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.398},\n\t\t\"r5dn.2xlarge\":      {Region: \"eu-central-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.796},\n\t\t\"r5dn.4xlarge\":      {Region: \"eu-central-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.592},\n\t\t\"r5dn.8xlarge\":      {Region: \"eu-central-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.184},\n\t\t\"r5dn.12xlarge\":     {Region: \"eu-central-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.776},\n\t\t\"r5dn.16xlarge\":     {Region: \"eu-central-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.368},\n\t\t\"r5dn.24xlarge\":     {Region: \"eu-central-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.552},\n\t\t\"r5dn.metal\":        {Region: \"eu-central-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.552},\n\t\t\"r5n.large\":         {Region: \"eu-central-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.178},\n\t\t\"r5n.xlarge\":        {Region: \"eu-central-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.356},\n\t\t\"r5n.2xlarge\":       {Region: \"eu-central-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.712},\n\t\t\"r5n.4xlarge\":       {Region: \"eu-central-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.424},\n\t\t\"r5n.8xlarge\":       {Region: \"eu-central-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.848},\n\t\t\"r5n.12xlarge\":      {Region: \"eu-central-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.272},\n\t\t\"r5n.16xlarge\":      {Region: \"eu-central-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.696},\n\t\t\"r5n.24xlarge\":      {Region: \"eu-central-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r5n.metal\":         {Region: \"eu-central-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.544},\n\t\t\"r6g.medium\":        {Region: \"eu-central-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0608},\n\t\t\"r6g.large\":         {Region: \"eu-central-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1216},\n\t\t\"r6g.xlarge\":        {Region: \"eu-central-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2432},\n\t\t\"r6g.2xlarge\":       {Region: \"eu-central-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4864},\n\t\t\"r6g.4xlarge\":       {Region: \"eu-central-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9728},\n\t\t\"r6g.8xlarge\":       {Region: \"eu-central-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.9456},\n\t\t\"r6g.12xlarge\":      {Region: \"eu-central-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.9184},\n\t\t\"r6g.16xlarge\":      {Region: \"eu-central-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8912},\n\t\t\"r6g.metal\":         {Region: \"eu-central-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8912},\n\t\t\"r6gd.medium\":       {Region: \"eu-central-1\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.069},\n\t\t\"r6gd.large\":        {Region: \"eu-central-1\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.138},\n\t\t\"r6gd.xlarge\":       {Region: \"eu-central-1\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.276},\n\t\t\"r6gd.2xlarge\":      {Region: \"eu-central-1\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.552},\n\t\t\"r6gd.4xlarge\":      {Region: \"eu-central-1\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.104},\n\t\t\"r6gd.8xlarge\":      {Region: \"eu-central-1\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.208},\n\t\t\"r6gd.12xlarge\":     {Region: \"eu-central-1\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.312},\n\t\t\"r6gd.16xlarge\":     {Region: \"eu-central-1\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.416},\n\t\t\"r6gd.metal\":        {Region: \"eu-central-1\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.416},\n\t\t\"t2.nano\":           {Region: \"eu-central-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0067},\n\t\t\"t2.micro\":          {Region: \"eu-central-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0134},\n\t\t\"t2.small\":          {Region: \"eu-central-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0268},\n\t\t\"t2.medium\":         {Region: \"eu-central-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0536},\n\t\t\"t2.large\":          {Region: \"eu-central-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1072},\n\t\t\"t2.xlarge\":         {Region: \"eu-central-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2144},\n\t\t\"t2.2xlarge\":        {Region: \"eu-central-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4288},\n\t\t\"t3.nano\":           {Region: \"eu-central-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.006},\n\t\t\"t3.micro\":          {Region: \"eu-central-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.012},\n\t\t\"t3.small\":          {Region: \"eu-central-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.024},\n\t\t\"t3.medium\":         {Region: \"eu-central-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.048},\n\t\t\"t3.large\":          {Region: \"eu-central-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"t3.xlarge\":         {Region: \"eu-central-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"t3.2xlarge\":        {Region: \"eu-central-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"t3a.nano\":          {Region: \"eu-central-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0054},\n\t\t\"t3a.micro\":         {Region: \"eu-central-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0108},\n\t\t\"t3a.small\":         {Region: \"eu-central-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0216},\n\t\t\"t3a.medium\":        {Region: \"eu-central-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0432},\n\t\t\"t3a.large\":         {Region: \"eu-central-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0864},\n\t\t\"t3a.xlarge\":        {Region: \"eu-central-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1728},\n\t\t\"t3a.2xlarge\":       {Region: \"eu-central-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3456},\n\t\t\"t4g.nano\":          {Region: \"eu-central-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0048},\n\t\t\"t4g.micro\":         {Region: \"eu-central-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0096},\n\t\t\"t4g.small\":         {Region: \"eu-central-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0192},\n\t\t\"t4g.medium\":        {Region: \"eu-central-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0384},\n\t\t\"t4g.large\":         {Region: \"eu-central-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0768},\n\t\t\"t4g.xlarge\":        {Region: \"eu-central-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1536},\n\t\t\"t4g.2xlarge\":       {Region: \"eu-central-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3072},\n\t\t\"u-12tb1.112xlarge\": {Region: \"eu-central-1\", Type: \"u-12tb1.112xlarge\", Memory: kresource.MustParse(\"12582912Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 131.733},\n\t\t\"u-3tb1.56xlarge\":   {Region: \"eu-central-1\", Type: \"u-3tb1.56xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 32.9335},\n\t\t\"u-6tb1.56xlarge\":   {Region: \"eu-central-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 55.9796},\n\t\t\"u-6tb1.112xlarge\":  {Region: \"eu-central-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 65.867},\n\t\t\"u-9tb1.112xlarge\":  {Region: \"eu-central-1\", Type: \"u-9tb1.112xlarge\", Memory: kresource.MustParse(\"9437184Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 98.8},\n\t\t\"x1.16xlarge\":       {Region: \"eu-central-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.337},\n\t\t\"x1.32xlarge\":       {Region: \"eu-central-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 18.674},\n\t\t\"x1e.xlarge\":        {Region: \"eu-central-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.167},\n\t\t\"x1e.2xlarge\":       {Region: \"eu-central-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.334},\n\t\t\"x1e.4xlarge\":       {Region: \"eu-central-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.668},\n\t\t\"x1e.8xlarge\":       {Region: \"eu-central-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.336},\n\t\t\"x1e.16xlarge\":      {Region: \"eu-central-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 18.672},\n\t\t\"x1e.32xlarge\":      {Region: \"eu-central-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 37.344},\n\t\t\"x2idn.16xlarge\":    {Region: \"eu-central-1\", Type: \"x2idn.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 9.337},\n\t\t\"x2idn.24xlarge\":    {Region: \"eu-central-1\", Type: \"x2idn.24xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 14.0055},\n\t\t\"x2idn.32xlarge\":    {Region: \"eu-central-1\", Type: \"x2idn.32xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 18.674},\n\t\t\"x2iedn.xlarge\":     {Region: \"eu-central-1\", Type: \"x2iedn.xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.16713},\n\t\t\"x2iedn.2xlarge\":    {Region: \"eu-central-1\", Type: \"x2iedn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.33425},\n\t\t\"x2iedn.4xlarge\":    {Region: \"eu-central-1\", Type: \"x2iedn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.6685},\n\t\t\"x2iedn.8xlarge\":    {Region: \"eu-central-1\", Type: \"x2iedn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 9.337},\n\t\t\"x2iedn.16xlarge\":   {Region: \"eu-central-1\", Type: \"x2iedn.16xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 18.674},\n\t\t\"x2iedn.24xlarge\":   {Region: \"eu-central-1\", Type: \"x2iedn.24xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 28.011},\n\t\t\"x2iedn.32xlarge\":   {Region: \"eu-central-1\", Type: \"x2iedn.32xlarge\", Memory: kresource.MustParse(\"4194304Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 37.348},\n\t\t\"z1d.large\":         {Region: \"eu-central-1\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.225},\n\t\t\"z1d.xlarge\":        {Region: \"eu-central-1\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.45},\n\t\t\"z1d.2xlarge\":       {Region: \"eu-central-1\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.9},\n\t\t\"z1d.3xlarge\":       {Region: \"eu-central-1\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.35},\n\t\t\"z1d.6xlarge\":       {Region: \"eu-central-1\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.7},\n\t\t\"z1d.12xlarge\":      {Region: \"eu-central-1\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.4},\n\t\t\"z1d.metal\":         {Region: \"eu-central-1\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.4},\n\t},\n\t\"eu-north-1\": {\n\t\t\"c5.large\":         {Region: \"eu-north-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.091},\n\t\t\"c5.xlarge\":        {Region: \"eu-north-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.182},\n\t\t\"c5.2xlarge\":       {Region: \"eu-north-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.364},\n\t\t\"c5.4xlarge\":       {Region: \"eu-north-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.728},\n\t\t\"c5.9xlarge\":       {Region: \"eu-north-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.638},\n\t\t\"c5.12xlarge\":      {Region: \"eu-north-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.184},\n\t\t\"c5.18xlarge\":      {Region: \"eu-north-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.276},\n\t\t\"c5.24xlarge\":      {Region: \"eu-north-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.368},\n\t\t\"c5.metal\":         {Region: \"eu-north-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.368},\n\t\t\"c5a.large\":        {Region: \"eu-north-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.082},\n\t\t\"c5a.xlarge\":       {Region: \"eu-north-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.164},\n\t\t\"c5a.2xlarge\":      {Region: \"eu-north-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.328},\n\t\t\"c5a.4xlarge\":      {Region: \"eu-north-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.656},\n\t\t\"c5a.8xlarge\":      {Region: \"eu-north-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.312},\n\t\t\"c5a.12xlarge\":     {Region: \"eu-north-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.968},\n\t\t\"c5a.16xlarge\":     {Region: \"eu-north-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.624},\n\t\t\"c5a.24xlarge\":     {Region: \"eu-north-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.936},\n\t\t\"c5d.large\":        {Region: \"eu-north-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.104},\n\t\t\"c5d.xlarge\":       {Region: \"eu-north-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.208},\n\t\t\"c5d.2xlarge\":      {Region: \"eu-north-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.416},\n\t\t\"c5d.4xlarge\":      {Region: \"eu-north-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.832},\n\t\t\"c5d.9xlarge\":      {Region: \"eu-north-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.872},\n\t\t\"c5d.12xlarge\":     {Region: \"eu-north-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.496},\n\t\t\"c5d.18xlarge\":     {Region: \"eu-north-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.744},\n\t\t\"c5d.24xlarge\":     {Region: \"eu-north-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"c5d.metal\":        {Region: \"eu-north-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"c5n.large\":        {Region: \"eu-north-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.116},\n\t\t\"c5n.xlarge\":       {Region: \"eu-north-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.232},\n\t\t\"c5n.2xlarge\":      {Region: \"eu-north-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.464},\n\t\t\"c5n.4xlarge\":      {Region: \"eu-north-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.928},\n\t\t\"c5n.9xlarge\":      {Region: \"eu-north-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.088},\n\t\t\"c5n.18xlarge\":     {Region: \"eu-north-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"c5n.metal\":        {Region: \"eu-north-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"c6g.medium\":       {Region: \"eu-north-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0365},\n\t\t\"c6g.large\":        {Region: \"eu-north-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.073},\n\t\t\"c6g.xlarge\":       {Region: \"eu-north-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.146},\n\t\t\"c6g.2xlarge\":      {Region: \"eu-north-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.292},\n\t\t\"c6g.4xlarge\":      {Region: \"eu-north-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.584},\n\t\t\"c6g.8xlarge\":      {Region: \"eu-north-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.168},\n\t\t\"c6g.12xlarge\":     {Region: \"eu-north-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.752},\n\t\t\"c6g.16xlarge\":     {Region: \"eu-north-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.336},\n\t\t\"c6g.metal\":        {Region: \"eu-north-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.336},\n\t\t\"c6gn.medium\":      {Region: \"eu-north-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0465},\n\t\t\"c6gn.large\":       {Region: \"eu-north-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.093},\n\t\t\"c6gn.xlarge\":      {Region: \"eu-north-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"c6gn.2xlarge\":     {Region: \"eu-north-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"c6gn.4xlarge\":     {Region: \"eu-north-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.744},\n\t\t\"c6gn.8xlarge\":     {Region: \"eu-north-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.488},\n\t\t\"c6gn.12xlarge\":    {Region: \"eu-north-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.232},\n\t\t\"c6gn.16xlarge\":    {Region: \"eu-north-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.976},\n\t\t\"d2.xlarge\":        {Region: \"eu-north-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.698},\n\t\t\"d2.2xlarge\":       {Region: \"eu-north-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.396},\n\t\t\"d2.4xlarge\":       {Region: \"eu-north-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.792},\n\t\t\"d2.8xlarge\":       {Region: \"eu-north-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 5.584},\n\t\t\"g4dn.xlarge\":      {Region: \"eu-north-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.558},\n\t\t\"g4dn.2xlarge\":     {Region: \"eu-north-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.798},\n\t\t\"g4dn.4xlarge\":     {Region: \"eu-north-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.277},\n\t\t\"g4dn.8xlarge\":     {Region: \"eu-north-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.308},\n\t\t\"g4dn.12xlarge\":    {Region: \"eu-north-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.15},\n\t\t\"g4dn.16xlarge\":    {Region: \"eu-north-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.617},\n\t\t\"g4dn.metal\":       {Region: \"eu-north-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 8.3},\n\t\t\"i3.large\":         {Region: \"eu-north-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.163},\n\t\t\"i3.xlarge\":        {Region: \"eu-north-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.326},\n\t\t\"i3.2xlarge\":       {Region: \"eu-north-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.652},\n\t\t\"i3.4xlarge\":       {Region: \"eu-north-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.304},\n\t\t\"i3.8xlarge\":       {Region: \"eu-north-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.608},\n\t\t\"i3.16xlarge\":      {Region: \"eu-north-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.216},\n\t\t\"i3.metal\":         {Region: \"eu-north-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.216},\n\t\t\"i3en.large\":       {Region: \"eu-north-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.237},\n\t\t\"i3en.xlarge\":      {Region: \"eu-north-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.474},\n\t\t\"i3en.2xlarge\":     {Region: \"eu-north-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.948},\n\t\t\"i3en.3xlarge\":     {Region: \"eu-north-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.422},\n\t\t\"i3en.6xlarge\":     {Region: \"eu-north-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.844},\n\t\t\"i3en.12xlarge\":    {Region: \"eu-north-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.688},\n\t\t\"i3en.24xlarge\":    {Region: \"eu-north-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 11.376},\n\t\t\"i3en.metal\":       {Region: \"eu-north-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 11.376},\n\t\t\"inf1.xlarge\":      {Region: \"eu-north-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.242},\n\t\t\"inf1.2xlarge\":     {Region: \"eu-north-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.385},\n\t\t\"inf1.6xlarge\":     {Region: \"eu-north-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.254},\n\t\t\"inf1.24xlarge\":    {Region: \"eu-north-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.016},\n\t\t\"m5.large\":         {Region: \"eu-north-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.102},\n\t\t\"m5.xlarge\":        {Region: \"eu-north-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.204},\n\t\t\"m5.2xlarge\":       {Region: \"eu-north-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"m5.4xlarge\":       {Region: \"eu-north-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.816},\n\t\t\"m5.8xlarge\":       {Region: \"eu-north-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.632},\n\t\t\"m5.12xlarge\":      {Region: \"eu-north-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"m5.16xlarge\":      {Region: \"eu-north-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"m5.24xlarge\":      {Region: \"eu-north-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"m5.metal\":         {Region: \"eu-north-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"m5d.large\":        {Region: \"eu-north-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"m5d.xlarge\":       {Region: \"eu-north-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"m5d.2xlarge\":      {Region: \"eu-north-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"m5d.4xlarge\":      {Region: \"eu-north-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.96},\n\t\t\"m5d.8xlarge\":      {Region: \"eu-north-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.92},\n\t\t\"m5d.12xlarge\":     {Region: \"eu-north-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m5d.16xlarge\":     {Region: \"eu-north-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.84},\n\t\t\"m5d.24xlarge\":     {Region: \"eu-north-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m5d.metal\":        {Region: \"eu-north-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m6g.medium\":       {Region: \"eu-north-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.041},\n\t\t\"m6g.large\":        {Region: \"eu-north-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.082},\n\t\t\"m6g.xlarge\":       {Region: \"eu-north-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.164},\n\t\t\"m6g.2xlarge\":      {Region: \"eu-north-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.328},\n\t\t\"m6g.4xlarge\":      {Region: \"eu-north-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.656},\n\t\t\"m6g.8xlarge\":      {Region: \"eu-north-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.312},\n\t\t\"m6g.12xlarge\":     {Region: \"eu-north-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.968},\n\t\t\"m6g.16xlarge\":     {Region: \"eu-north-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.624},\n\t\t\"m6g.metal\":        {Region: \"eu-north-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.624},\n\t\t\"r5.large\":         {Region: \"eu-north-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.134},\n\t\t\"r5.xlarge\":        {Region: \"eu-north-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.268},\n\t\t\"r5.2xlarge\":       {Region: \"eu-north-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.536},\n\t\t\"r5.4xlarge\":       {Region: \"eu-north-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.072},\n\t\t\"r5.8xlarge\":       {Region: \"eu-north-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.144},\n\t\t\"r5.12xlarge\":      {Region: \"eu-north-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.216},\n\t\t\"r5.16xlarge\":      {Region: \"eu-north-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.288},\n\t\t\"r5.24xlarge\":      {Region: \"eu-north-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.432},\n\t\t\"r5.metal\":         {Region: \"eu-north-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.432},\n\t\t\"r5d.large\":        {Region: \"eu-north-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"r5d.xlarge\":       {Region: \"eu-north-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"r5d.2xlarge\":      {Region: \"eu-north-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"r5d.4xlarge\":      {Region: \"eu-north-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"r5d.8xlarge\":      {Region: \"eu-north-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"r5d.12xlarge\":     {Region: \"eu-north-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r5d.16xlarge\":     {Region: \"eu-north-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"r5d.24xlarge\":     {Region: \"eu-north-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5d.metal\":        {Region: \"eu-north-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5dn.large\":       {Region: \"eu-north-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.177},\n\t\t\"r5dn.xlarge\":      {Region: \"eu-north-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.354},\n\t\t\"r5dn.2xlarge\":     {Region: \"eu-north-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.708},\n\t\t\"r5dn.4xlarge\":     {Region: \"eu-north-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.416},\n\t\t\"r5dn.8xlarge\":     {Region: \"eu-north-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.832},\n\t\t\"r5dn.12xlarge\":    {Region: \"eu-north-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.248},\n\t\t\"r5dn.16xlarge\":    {Region: \"eu-north-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.664},\n\t\t\"r5dn.24xlarge\":    {Region: \"eu-north-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.496},\n\t\t\"r5dn.metal\":       {Region: \"eu-north-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.496},\n\t\t\"r5n.large\":        {Region: \"eu-north-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.159},\n\t\t\"r5n.xlarge\":       {Region: \"eu-north-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.318},\n\t\t\"r5n.2xlarge\":      {Region: \"eu-north-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.636},\n\t\t\"r5n.4xlarge\":      {Region: \"eu-north-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.272},\n\t\t\"r5n.8xlarge\":      {Region: \"eu-north-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"r5n.12xlarge\":     {Region: \"eu-north-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.816},\n\t\t\"r5n.16xlarge\":     {Region: \"eu-north-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"r5n.24xlarge\":     {Region: \"eu-north-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.632},\n\t\t\"r5n.metal\":        {Region: \"eu-north-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.632},\n\t\t\"r6g.medium\":       {Region: \"eu-north-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0535},\n\t\t\"r6g.large\":        {Region: \"eu-north-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.107},\n\t\t\"r6g.xlarge\":       {Region: \"eu-north-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.214},\n\t\t\"r6g.2xlarge\":      {Region: \"eu-north-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.428},\n\t\t\"r6g.4xlarge\":      {Region: \"eu-north-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.856},\n\t\t\"r6g.8xlarge\":      {Region: \"eu-north-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.712},\n\t\t\"r6g.12xlarge\":     {Region: \"eu-north-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.568},\n\t\t\"r6g.16xlarge\":     {Region: \"eu-north-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.424},\n\t\t\"r6g.metal\":        {Region: \"eu-north-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.424},\n\t\t\"t3.nano\":          {Region: \"eu-north-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0054},\n\t\t\"t3.micro\":         {Region: \"eu-north-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0108},\n\t\t\"t3.small\":         {Region: \"eu-north-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0216},\n\t\t\"t3.medium\":        {Region: \"eu-north-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0432},\n\t\t\"t3.large\":         {Region: \"eu-north-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0864},\n\t\t\"t3.xlarge\":        {Region: \"eu-north-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1728},\n\t\t\"t3.2xlarge\":       {Region: \"eu-north-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3456},\n\t\t\"t4g.nano\":         {Region: \"eu-north-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0043},\n\t\t\"t4g.micro\":        {Region: \"eu-north-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0086},\n\t\t\"t4g.small\":        {Region: \"eu-north-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0172},\n\t\t\"t4g.medium\":       {Region: \"eu-north-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0344},\n\t\t\"t4g.large\":        {Region: \"eu-north-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0688},\n\t\t\"t4g.xlarge\":       {Region: \"eu-north-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1376},\n\t\t\"t4g.2xlarge\":      {Region: \"eu-north-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2752},\n\t\t\"u-6tb1.56xlarge\":  {Region: \"eu-north-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 49.33177},\n\t\t\"u-6tb1.112xlarge\": {Region: \"eu-north-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 58.045},\n\t},\n\t\"eu-south-1\": {\n\t\t\"c5.large\":         {Region: \"eu-south-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"c5.xlarge\":        {Region: \"eu-south-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"c5.2xlarge\":       {Region: \"eu-south-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"c5.4xlarge\":       {Region: \"eu-south-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"c5.9xlarge\":       {Region: \"eu-south-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.818},\n\t\t\"c5.12xlarge\":      {Region: \"eu-south-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"c5.18xlarge\":      {Region: \"eu-south-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.636},\n\t\t\"c5.24xlarge\":      {Region: \"eu-south-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c5.metal\":         {Region: \"eu-south-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c5a.large\":        {Region: \"eu-south-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.091},\n\t\t\"c5a.xlarge\":       {Region: \"eu-south-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.182},\n\t\t\"c5a.2xlarge\":      {Region: \"eu-south-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.364},\n\t\t\"c5a.4xlarge\":      {Region: \"eu-south-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.728},\n\t\t\"c5a.8xlarge\":      {Region: \"eu-south-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.456},\n\t\t\"c5a.12xlarge\":     {Region: \"eu-south-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.184},\n\t\t\"c5a.16xlarge\":     {Region: \"eu-south-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.912},\n\t\t\"c5a.24xlarge\":     {Region: \"eu-south-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.368},\n\t\t\"c5ad.large\":       {Region: \"eu-south-1\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.103},\n\t\t\"c5ad.xlarge\":      {Region: \"eu-south-1\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.206},\n\t\t\"c5ad.2xlarge\":     {Region: \"eu-south-1\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.412},\n\t\t\"c5ad.4xlarge\":     {Region: \"eu-south-1\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.824},\n\t\t\"c5ad.8xlarge\":     {Region: \"eu-south-1\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.648},\n\t\t\"c5ad.12xlarge\":    {Region: \"eu-south-1\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.472},\n\t\t\"c5ad.16xlarge\":    {Region: \"eu-south-1\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.296},\n\t\t\"c5ad.24xlarge\":    {Region: \"eu-south-1\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.944},\n\t\t\"c5d.large\":        {Region: \"eu-south-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.114},\n\t\t\"c5d.xlarge\":       {Region: \"eu-south-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.228},\n\t\t\"c5d.2xlarge\":      {Region: \"eu-south-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.456},\n\t\t\"c5d.4xlarge\":      {Region: \"eu-south-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.912},\n\t\t\"c5d.9xlarge\":      {Region: \"eu-south-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.052},\n\t\t\"c5d.12xlarge\":     {Region: \"eu-south-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.736},\n\t\t\"c5d.18xlarge\":     {Region: \"eu-south-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.104},\n\t\t\"c5d.24xlarge\":     {Region: \"eu-south-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.472},\n\t\t\"c5d.metal\":        {Region: \"eu-south-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.472},\n\t\t\"c5n.large\":        {Region: \"eu-south-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.129},\n\t\t\"c5n.xlarge\":       {Region: \"eu-south-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.258},\n\t\t\"c5n.2xlarge\":      {Region: \"eu-south-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.516},\n\t\t\"c5n.4xlarge\":      {Region: \"eu-south-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.032},\n\t\t\"c5n.9xlarge\":      {Region: \"eu-south-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.322},\n\t\t\"c5n.18xlarge\":     {Region: \"eu-south-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.644},\n\t\t\"c5n.metal\":        {Region: \"eu-south-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.644},\n\t\t\"c6g.medium\":       {Region: \"eu-south-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0404},\n\t\t\"c6g.large\":        {Region: \"eu-south-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0808},\n\t\t\"c6g.xlarge\":       {Region: \"eu-south-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1616},\n\t\t\"c6g.2xlarge\":      {Region: \"eu-south-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3232},\n\t\t\"c6g.4xlarge\":      {Region: \"eu-south-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6464},\n\t\t\"c6g.8xlarge\":      {Region: \"eu-south-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.2928},\n\t\t\"c6g.12xlarge\":     {Region: \"eu-south-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.9392},\n\t\t\"c6g.16xlarge\":     {Region: \"eu-south-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.5856},\n\t\t\"c6g.metal\":        {Region: \"eu-south-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.5856},\n\t\t\"d2.xlarge\":        {Region: \"eu-south-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.772},\n\t\t\"d2.2xlarge\":       {Region: \"eu-south-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.544},\n\t\t\"d2.4xlarge\":       {Region: \"eu-south-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.088},\n\t\t\"d2.8xlarge\":       {Region: \"eu-south-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.176},\n\t\t\"g4dn.xlarge\":      {Region: \"eu-south-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.616},\n\t\t\"g4dn.2xlarge\":     {Region: \"eu-south-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.88},\n\t\t\"g4dn.4xlarge\":     {Region: \"eu-south-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.41},\n\t\t\"g4dn.8xlarge\":     {Region: \"eu-south-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.547},\n\t\t\"g4dn.12xlarge\":    {Region: \"eu-south-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.58},\n\t\t\"g4dn.16xlarge\":    {Region: \"eu-south-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.095},\n\t\t\"g4dn.metal\":       {Region: \"eu-south-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 9.16},\n\t\t\"i3.large\":         {Region: \"eu-south-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.181},\n\t\t\"i3.xlarge\":        {Region: \"eu-south-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.362},\n\t\t\"i3.2xlarge\":       {Region: \"eu-south-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.724},\n\t\t\"i3.4xlarge\":       {Region: \"eu-south-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.448},\n\t\t\"i3.8xlarge\":       {Region: \"eu-south-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.896},\n\t\t\"i3.16xlarge\":      {Region: \"eu-south-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"i3.metal\":         {Region: \"eu-south-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"i3en.large\":       {Region: \"eu-south-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.262},\n\t\t\"i3en.xlarge\":      {Region: \"eu-south-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"i3en.2xlarge\":     {Region: \"eu-south-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.049},\n\t\t\"i3en.3xlarge\":     {Region: \"eu-south-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.573},\n\t\t\"i3en.6xlarge\":     {Region: \"eu-south-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.146},\n\t\t\"i3en.12xlarge\":    {Region: \"eu-south-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.293},\n\t\t\"i3en.24xlarge\":    {Region: \"eu-south-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.586},\n\t\t\"i3en.metal\":       {Region: \"eu-south-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.586},\n\t\t\"inf1.xlarge\":      {Region: \"eu-south-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.267},\n\t\t\"inf1.2xlarge\":     {Region: \"eu-south-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.424},\n\t\t\"inf1.6xlarge\":     {Region: \"eu-south-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.382},\n\t\t\"inf1.24xlarge\":    {Region: \"eu-south-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.53},\n\t\t\"m5.large\":         {Region: \"eu-south-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.112},\n\t\t\"m5.xlarge\":        {Region: \"eu-south-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.224},\n\t\t\"m5.2xlarge\":       {Region: \"eu-south-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.448},\n\t\t\"m5.4xlarge\":       {Region: \"eu-south-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.896},\n\t\t\"m5.8xlarge\":       {Region: \"eu-south-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.792},\n\t\t\"m5.12xlarge\":      {Region: \"eu-south-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"m5.16xlarge\":      {Region: \"eu-south-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.584},\n\t\t\"m5.24xlarge\":      {Region: \"eu-south-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m5.metal\":         {Region: \"eu-south-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m5a.large\":        {Region: \"eu-south-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"m5a.xlarge\":       {Region: \"eu-south-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"m5a.2xlarge\":      {Region: \"eu-south-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"m5a.4xlarge\":      {Region: \"eu-south-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"m5a.8xlarge\":      {Region: \"eu-south-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"m5a.12xlarge\":     {Region: \"eu-south-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"m5a.16xlarge\":     {Region: \"eu-south-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"m5a.24xlarge\":     {Region: \"eu-south-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"m5d.large\":        {Region: \"eu-south-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.132},\n\t\t\"m5d.xlarge\":       {Region: \"eu-south-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.264},\n\t\t\"m5d.2xlarge\":      {Region: \"eu-south-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.528},\n\t\t\"m5d.4xlarge\":      {Region: \"eu-south-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.056},\n\t\t\"m5d.8xlarge\":      {Region: \"eu-south-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.112},\n\t\t\"m5d.12xlarge\":     {Region: \"eu-south-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.168},\n\t\t\"m5d.16xlarge\":     {Region: \"eu-south-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.224},\n\t\t\"m5d.24xlarge\":     {Region: \"eu-south-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.336},\n\t\t\"m5d.metal\":        {Region: \"eu-south-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.336},\n\t\t\"m6g.medium\":       {Region: \"eu-south-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0448},\n\t\t\"m6g.large\":        {Region: \"eu-south-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0896},\n\t\t\"m6g.xlarge\":       {Region: \"eu-south-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1792},\n\t\t\"m6g.2xlarge\":      {Region: \"eu-south-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3584},\n\t\t\"m6g.4xlarge\":      {Region: \"eu-south-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7168},\n\t\t\"m6g.8xlarge\":      {Region: \"eu-south-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.4336},\n\t\t\"m6g.12xlarge\":     {Region: \"eu-south-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.1504},\n\t\t\"m6g.16xlarge\":     {Region: \"eu-south-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8672},\n\t\t\"m6g.metal\":        {Region: \"eu-south-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8672},\n\t\t\"r5.large\":         {Region: \"eu-south-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.148},\n\t\t\"r5.xlarge\":        {Region: \"eu-south-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.296},\n\t\t\"r5.2xlarge\":       {Region: \"eu-south-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"r5.4xlarge\":       {Region: \"eu-south-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.184},\n\t\t\"r5.8xlarge\":       {Region: \"eu-south-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.368},\n\t\t\"r5.12xlarge\":      {Region: \"eu-south-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"r5.16xlarge\":      {Region: \"eu-south-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.736},\n\t\t\"r5.24xlarge\":      {Region: \"eu-south-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r5.metal\":         {Region: \"eu-south-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r5a.large\":        {Region: \"eu-south-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"r5a.xlarge\":       {Region: \"eu-south-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"r5a.2xlarge\":      {Region: \"eu-south-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"r5a.4xlarge\":      {Region: \"eu-south-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"r5a.8xlarge\":      {Region: \"eu-south-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"r5a.12xlarge\":     {Region: \"eu-south-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"r5a.16xlarge\":     {Region: \"eu-south-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"r5a.24xlarge\":     {Region: \"eu-south-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"r5d.large\":        {Region: \"eu-south-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.168},\n\t\t\"r5d.xlarge\":       {Region: \"eu-south-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.336},\n\t\t\"r5d.2xlarge\":      {Region: \"eu-south-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.672},\n\t\t\"r5d.4xlarge\":      {Region: \"eu-south-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.344},\n\t\t\"r5d.8xlarge\":      {Region: \"eu-south-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"r5d.12xlarge\":     {Region: \"eu-south-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r5d.16xlarge\":     {Region: \"eu-south-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"r5d.24xlarge\":     {Region: \"eu-south-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"r5d.metal\":        {Region: \"eu-south-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"r5dn.large\":       {Region: \"eu-south-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"r5dn.xlarge\":      {Region: \"eu-south-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"r5dn.2xlarge\":     {Region: \"eu-south-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"r5dn.4xlarge\":     {Region: \"eu-south-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.568},\n\t\t\"r5dn.8xlarge\":     {Region: \"eu-south-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.136},\n\t\t\"r5dn.12xlarge\":    {Region: \"eu-south-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.704},\n\t\t\"r5dn.16xlarge\":    {Region: \"eu-south-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.272},\n\t\t\"r5dn.24xlarge\":    {Region: \"eu-south-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.408},\n\t\t\"r5dn.metal\":       {Region: \"eu-south-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.408},\n\t\t\"r6g.medium\":       {Region: \"eu-south-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0592},\n\t\t\"r6g.large\":        {Region: \"eu-south-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1184},\n\t\t\"r6g.xlarge\":       {Region: \"eu-south-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2368},\n\t\t\"r6g.2xlarge\":      {Region: \"eu-south-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4736},\n\t\t\"r6g.4xlarge\":      {Region: \"eu-south-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9472},\n\t\t\"r6g.8xlarge\":      {Region: \"eu-south-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.8944},\n\t\t\"r6g.12xlarge\":     {Region: \"eu-south-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.8416},\n\t\t\"r6g.16xlarge\":     {Region: \"eu-south-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.7888},\n\t\t\"r6g.metal\":        {Region: \"eu-south-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.7888},\n\t\t\"t3.nano\":          {Region: \"eu-south-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.006},\n\t\t\"t3.micro\":         {Region: \"eu-south-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.012},\n\t\t\"t3.small\":         {Region: \"eu-south-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.024},\n\t\t\"t3.medium\":        {Region: \"eu-south-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0479},\n\t\t\"t3.large\":         {Region: \"eu-south-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0958},\n\t\t\"t3.xlarge\":        {Region: \"eu-south-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1917},\n\t\t\"t3.2xlarge\":       {Region: \"eu-south-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3834},\n\t\t\"t3a.nano\":         {Region: \"eu-south-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0054},\n\t\t\"t3a.micro\":        {Region: \"eu-south-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0108},\n\t\t\"t3a.small\":        {Region: \"eu-south-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0216},\n\t\t\"t3a.medium\":       {Region: \"eu-south-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0431},\n\t\t\"t3a.large\":        {Region: \"eu-south-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0862},\n\t\t\"t3a.xlarge\":       {Region: \"eu-south-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1725},\n\t\t\"t3a.2xlarge\":      {Region: \"eu-south-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.345},\n\t\t\"t4g.nano\":         {Region: \"eu-south-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0048},\n\t\t\"t4g.micro\":        {Region: \"eu-south-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0096},\n\t\t\"t4g.small\":        {Region: \"eu-south-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0192},\n\t\t\"t4g.medium\":       {Region: \"eu-south-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0384},\n\t\t\"t4g.large\":        {Region: \"eu-south-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0768},\n\t\t\"t4g.xlarge\":       {Region: \"eu-south-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1536},\n\t\t\"t4g.2xlarge\":      {Region: \"eu-south-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3072},\n\t\t\"u-3tb1.56xlarge\":  {Region: \"eu-south-1\", Type: \"u-3tb1.56xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 32.0775},\n\t\t\"u-6tb1.56xlarge\":  {Region: \"eu-south-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 54.52459},\n\t\t\"u-6tb1.112xlarge\": {Region: \"eu-south-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 64.155},\n\t},\n\t\"eu-west-1\": {\n\t\t\"a1.medium\":         {Region: \"eu-west-1\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0288},\n\t\t\"a1.large\":          {Region: \"eu-west-1\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0576},\n\t\t\"a1.xlarge\":         {Region: \"eu-west-1\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1152},\n\t\t\"a1.2xlarge\":        {Region: \"eu-west-1\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2304},\n\t\t\"a1.4xlarge\":        {Region: \"eu-west-1\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.4608},\n\t\t\"a1.metal\":          {Region: \"eu-west-1\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.461},\n\t\t\"c1.medium\":         {Region: \"eu-west-1\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.148},\n\t\t\"c1.xlarge\":         {Region: \"eu-west-1\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"c3.large\":          {Region: \"eu-west-1\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"c3.xlarge\":         {Region: \"eu-west-1\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.239},\n\t\t\"c3.2xlarge\":        {Region: \"eu-west-1\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.478},\n\t\t\"c3.4xlarge\":        {Region: \"eu-west-1\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.956},\n\t\t\"c3.8xlarge\":        {Region: \"eu-west-1\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.912},\n\t\t\"c4.large\":          {Region: \"eu-west-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"c4.xlarge\":         {Region: \"eu-west-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"c4.2xlarge\":        {Region: \"eu-west-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.453},\n\t\t\"c4.4xlarge\":        {Region: \"eu-west-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.905},\n\t\t\"c4.8xlarge\":        {Region: \"eu-west-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.811},\n\t\t\"c5.large\":          {Region: \"eu-west-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"c5.xlarge\":         {Region: \"eu-west-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"c5.2xlarge\":        {Region: \"eu-west-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"c5.4xlarge\":        {Region: \"eu-west-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"c5.9xlarge\":        {Region: \"eu-west-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"c5.12xlarge\":       {Region: \"eu-west-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c5.18xlarge\":       {Region: \"eu-west-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"c5.24xlarge\":       {Region: \"eu-west-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5.metal\":          {Region: \"eu-west-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5a.large\":         {Region: \"eu-west-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"c5a.xlarge\":        {Region: \"eu-west-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"c5a.2xlarge\":       {Region: \"eu-west-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"c5a.4xlarge\":       {Region: \"eu-west-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"c5a.8xlarge\":       {Region: \"eu-west-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"c5a.12xlarge\":      {Region: \"eu-west-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"c5a.16xlarge\":      {Region: \"eu-west-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"c5a.24xlarge\":      {Region: \"eu-west-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"c5ad.large\":        {Region: \"eu-west-1\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.098},\n\t\t\"c5ad.xlarge\":       {Region: \"eu-west-1\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"c5ad.2xlarge\":      {Region: \"eu-west-1\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"c5ad.4xlarge\":      {Region: \"eu-west-1\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"c5ad.8xlarge\":      {Region: \"eu-west-1\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.568},\n\t\t\"c5ad.12xlarge\":     {Region: \"eu-west-1\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.352},\n\t\t\"c5ad.16xlarge\":     {Region: \"eu-west-1\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.136},\n\t\t\"c5ad.24xlarge\":     {Region: \"eu-west-1\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.704},\n\t\t\"c5d.large\":         {Region: \"eu-west-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.109},\n\t\t\"c5d.xlarge\":        {Region: \"eu-west-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.218},\n\t\t\"c5d.2xlarge\":       {Region: \"eu-west-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.436},\n\t\t\"c5d.4xlarge\":       {Region: \"eu-west-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.872},\n\t\t\"c5d.9xlarge\":       {Region: \"eu-west-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.962},\n\t\t\"c5d.12xlarge\":      {Region: \"eu-west-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.616},\n\t\t\"c5d.18xlarge\":      {Region: \"eu-west-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.924},\n\t\t\"c5d.24xlarge\":      {Region: \"eu-west-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.232},\n\t\t\"c5d.metal\":         {Region: \"eu-west-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.232},\n\t\t\"c5n.large\":         {Region: \"eu-west-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.122},\n\t\t\"c5n.xlarge\":        {Region: \"eu-west-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.244},\n\t\t\"c5n.2xlarge\":       {Region: \"eu-west-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.488},\n\t\t\"c5n.4xlarge\":       {Region: \"eu-west-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.976},\n\t\t\"c5n.9xlarge\":       {Region: \"eu-west-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.196},\n\t\t\"c5n.18xlarge\":      {Region: \"eu-west-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.392},\n\t\t\"c5n.metal\":         {Region: \"eu-west-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.392},\n\t\t\"c6a.large\":         {Region: \"eu-west-1\", Type: \"c6a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.08208},\n\t\t\"c6a.xlarge\":        {Region: \"eu-west-1\", Type: \"c6a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.16416},\n\t\t\"c6a.2xlarge\":       {Region: \"eu-west-1\", Type: \"c6a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.32832},\n\t\t\"c6a.4xlarge\":       {Region: \"eu-west-1\", Type: \"c6a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.65664},\n\t\t\"c6a.8xlarge\":       {Region: \"eu-west-1\", Type: \"c6a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.31328},\n\t\t\"c6a.12xlarge\":      {Region: \"eu-west-1\", Type: \"c6a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.96992},\n\t\t\"c6a.16xlarge\":      {Region: \"eu-west-1\", Type: \"c6a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.62656},\n\t\t\"c6a.24xlarge\":      {Region: \"eu-west-1\", Type: \"c6a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.93984},\n\t\t\"c6a.32xlarge\":      {Region: \"eu-west-1\", Type: \"c6a.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.25312},\n\t\t\"c6a.48xlarge\":      {Region: \"eu-west-1\", Type: \"c6a.48xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 7.87968},\n\t\t\"c6a.metal\":         {Region: \"eu-west-1\", Type: \"c6a.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 7.87968},\n\t\t\"c6g.medium\":        {Region: \"eu-west-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0365},\n\t\t\"c6g.large\":         {Region: \"eu-west-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.073},\n\t\t\"c6g.xlarge\":        {Region: \"eu-west-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1459},\n\t\t\"c6g.2xlarge\":       {Region: \"eu-west-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2918},\n\t\t\"c6g.4xlarge\":       {Region: \"eu-west-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.5837},\n\t\t\"c6g.8xlarge\":       {Region: \"eu-west-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.1674},\n\t\t\"c6g.12xlarge\":      {Region: \"eu-west-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.751},\n\t\t\"c6g.16xlarge\":      {Region: \"eu-west-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.3347},\n\t\t\"c6g.metal\":         {Region: \"eu-west-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.3347},\n\t\t\"c6gd.medium\":       {Region: \"eu-west-1\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0436},\n\t\t\"c6gd.large\":        {Region: \"eu-west-1\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0872},\n\t\t\"c6gd.xlarge\":       {Region: \"eu-west-1\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1744},\n\t\t\"c6gd.2xlarge\":      {Region: \"eu-west-1\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3488},\n\t\t\"c6gd.4xlarge\":      {Region: \"eu-west-1\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6976},\n\t\t\"c6gd.8xlarge\":      {Region: \"eu-west-1\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3952},\n\t\t\"c6gd.12xlarge\":     {Region: \"eu-west-1\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0928},\n\t\t\"c6gd.16xlarge\":     {Region: \"eu-west-1\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7904},\n\t\t\"c6gd.metal\":        {Region: \"eu-west-1\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7904},\n\t\t\"c6gn.medium\":       {Region: \"eu-west-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0488},\n\t\t\"c6gn.large\":        {Region: \"eu-west-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0976},\n\t\t\"c6gn.xlarge\":       {Region: \"eu-west-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1952},\n\t\t\"c6gn.2xlarge\":      {Region: \"eu-west-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3904},\n\t\t\"c6gn.4xlarge\":      {Region: \"eu-west-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7808},\n\t\t\"c6gn.8xlarge\":      {Region: \"eu-west-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.5616},\n\t\t\"c6gn.12xlarge\":     {Region: \"eu-west-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.3424},\n\t\t\"c6gn.16xlarge\":     {Region: \"eu-west-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.1232},\n\t\t\"c6i.large\":         {Region: \"eu-west-1\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0912},\n\t\t\"c6i.xlarge\":        {Region: \"eu-west-1\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1824},\n\t\t\"c6i.2xlarge\":       {Region: \"eu-west-1\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3648},\n\t\t\"c6i.4xlarge\":       {Region: \"eu-west-1\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7296},\n\t\t\"c6i.8xlarge\":       {Region: \"eu-west-1\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.4592},\n\t\t\"c6i.12xlarge\":      {Region: \"eu-west-1\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.1888},\n\t\t\"c6i.16xlarge\":      {Region: \"eu-west-1\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.9184},\n\t\t\"c6i.24xlarge\":      {Region: \"eu-west-1\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.3776},\n\t\t\"c6i.32xlarge\":      {Region: \"eu-west-1\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.8368},\n\t\t\"c6i.metal\":         {Region: \"eu-west-1\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.8368},\n\t\t\"cc2.8xlarge\":       {Region: \"eu-west-1\", Type: \"cc2.8xlarge\", Memory: kresource.MustParse(\"61952Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.25},\n\t\t\"cr1.8xlarge\":       {Region: \"eu-west-1\", Type: \"cr1.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.75},\n\t\t\"d2.xlarge\":         {Region: \"eu-west-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.735},\n\t\t\"d2.2xlarge\":        {Region: \"eu-west-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.47},\n\t\t\"d2.4xlarge\":        {Region: \"eu-west-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.94},\n\t\t\"d2.8xlarge\":        {Region: \"eu-west-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 5.88},\n\t\t\"d3.xlarge\":         {Region: \"eu-west-1\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.609},\n\t\t\"d3.2xlarge\":        {Region: \"eu-west-1\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.219},\n\t\t\"d3.4xlarge\":        {Region: \"eu-west-1\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.437},\n\t\t\"d3.8xlarge\":        {Region: \"eu-west-1\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.87448},\n\t\t\"d3en.xlarge\":       {Region: \"eu-west-1\", Type: \"d3en.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.641},\n\t\t\"d3en.2xlarge\":      {Region: \"eu-west-1\", Type: \"d3en.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.283},\n\t\t\"d3en.4xlarge\":      {Region: \"eu-west-1\", Type: \"d3en.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.566},\n\t\t\"d3en.6xlarge\":      {Region: \"eu-west-1\", Type: \"d3en.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.848},\n\t\t\"d3en.8xlarge\":      {Region: \"eu-west-1\", Type: \"d3en.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 5.13104},\n\t\t\"d3en.12xlarge\":     {Region: \"eu-west-1\", Type: \"d3en.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 7.69656},\n\t\t\"f1.2xlarge\":        {Region: \"eu-west-1\", Type: \"f1.2xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.815},\n\t\t\"f1.4xlarge\":        {Region: \"eu-west-1\", Type: \"f1.4xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.63},\n\t\t\"f1.16xlarge\":       {Region: \"eu-west-1\", Type: \"f1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 14.52},\n\t\t\"g2.2xlarge\":        {Region: \"eu-west-1\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.702},\n\t\t\"g2.8xlarge\":        {Region: \"eu-west-1\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 2.808},\n\t\t\"g3.4xlarge\":        {Region: \"eu-west-1\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.21},\n\t\t\"g3.8xlarge\":        {Region: \"eu-west-1\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.42},\n\t\t\"g3.16xlarge\":       {Region: \"eu-west-1\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 4.84},\n\t\t\"g3s.xlarge\":        {Region: \"eu-west-1\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.796},\n\t\t\"g4ad.xlarge\":       {Region: \"eu-west-1\", Type: \"g4ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.42263},\n\t\t\"g4ad.2xlarge\":      {Region: \"eu-west-1\", Type: \"g4ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.60421},\n\t\t\"g4ad.4xlarge\":      {Region: \"eu-west-1\", Type: \"g4ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 0.968},\n\t\t\"g4ad.8xlarge\":      {Region: \"eu-west-1\", Type: \"g4ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 1.936},\n\t\t\"g4ad.16xlarge\":     {Region: \"eu-west-1\", Type: \"g4ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 3.872},\n\t\t\"g4dn.xlarge\":       {Region: \"eu-west-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.587},\n\t\t\"g4dn.2xlarge\":      {Region: \"eu-west-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.838},\n\t\t\"g4dn.4xlarge\":      {Region: \"eu-west-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.342},\n\t\t\"g4dn.8xlarge\":      {Region: \"eu-west-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.426},\n\t\t\"g4dn.12xlarge\":     {Region: \"eu-west-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.362},\n\t\t\"g4dn.16xlarge\":     {Region: \"eu-west-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.853},\n\t\t\"g4dn.metal\":        {Region: \"eu-west-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 8.724},\n\t\t\"g5.xlarge\":         {Region: \"eu-west-1\", Type: \"g5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.123},\n\t\t\"g5.2xlarge\":        {Region: \"eu-west-1\", Type: \"g5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.35296},\n\t\t\"g5.4xlarge\":        {Region: \"eu-west-1\", Type: \"g5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.81287},\n\t\t\"g5.8xlarge\":        {Region: \"eu-west-1\", Type: \"g5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.73271},\n\t\t\"g5.12xlarge\":       {Region: \"eu-west-1\", Type: \"g5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 6.33167},\n\t\t\"g5.16xlarge\":       {Region: \"eu-west-1\", Type: \"g5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.57237},\n\t\t\"g5.24xlarge\":       {Region: \"eu-west-1\", Type: \"g5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 4, Inf: 0, Price: 9.09117},\n\t\t\"g5.48xlarge\":       {Region: \"eu-west-1\", Type: \"g5.48xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 8, Inf: 0, Price: 18.18233},\n\t\t\"h1.2xlarge\":        {Region: \"eu-west-1\", Type: \"h1.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.519},\n\t\t\"h1.4xlarge\":        {Region: \"eu-west-1\", Type: \"h1.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.038},\n\t\t\"h1.8xlarge\":        {Region: \"eu-west-1\", Type: \"h1.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.076},\n\t\t\"h1.16xlarge\":       {Region: \"eu-west-1\", Type: \"h1.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.152},\n\t\t\"hs1.8xlarge\":       {Region: \"eu-west-1\", Type: \"hs1.8xlarge\", Memory: kresource.MustParse(\"119808Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.9},\n\t\t\"i2.xlarge\":         {Region: \"eu-west-1\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.938},\n\t\t\"i2.2xlarge\":        {Region: \"eu-west-1\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.876},\n\t\t\"i2.4xlarge\":        {Region: \"eu-west-1\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.751},\n\t\t\"i2.8xlarge\":        {Region: \"eu-west-1\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 7.502},\n\t\t\"i3.large\":          {Region: \"eu-west-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"i3.xlarge\":         {Region: \"eu-west-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"i3.2xlarge\":        {Region: \"eu-west-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"i3.4xlarge\":        {Region: \"eu-west-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"i3.8xlarge\":        {Region: \"eu-west-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"i3.16xlarge\":       {Region: \"eu-west-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.504},\n\t\t\"i3.metal\":          {Region: \"eu-west-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.504},\n\t\t\"i3en.large\":        {Region: \"eu-west-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.25},\n\t\t\"i3en.xlarge\":       {Region: \"eu-west-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.5},\n\t\t\"i3en.2xlarge\":      {Region: \"eu-west-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"i3en.3xlarge\":      {Region: \"eu-west-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.5},\n\t\t\"i3en.6xlarge\":      {Region: \"eu-west-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.0},\n\t\t\"i3en.12xlarge\":     {Region: \"eu-west-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.0},\n\t\t\"i3en.24xlarge\":     {Region: \"eu-west-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.0},\n\t\t\"i3en.metal\":        {Region: \"eu-west-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.0},\n\t\t\"im4gn.large\":       {Region: \"eu-west-1\", Type: \"im4gn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.20055},\n\t\t\"im4gn.xlarge\":      {Region: \"eu-west-1\", Type: \"im4gn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.4011},\n\t\t\"im4gn.2xlarge\":     {Region: \"eu-west-1\", Type: \"im4gn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.80221},\n\t\t\"im4gn.4xlarge\":     {Region: \"eu-west-1\", Type: \"im4gn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.60442},\n\t\t\"im4gn.8xlarge\":     {Region: \"eu-west-1\", Type: \"im4gn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.20883},\n\t\t\"im4gn.16xlarge\":    {Region: \"eu-west-1\", Type: \"im4gn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.41766},\n\t\t\"inf1.xlarge\":       {Region: \"eu-west-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.254},\n\t\t\"inf1.2xlarge\":      {Region: \"eu-west-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.403},\n\t\t\"inf1.6xlarge\":      {Region: \"eu-west-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.315},\n\t\t\"inf1.24xlarge\":     {Region: \"eu-west-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.26},\n\t\t\"is4gen.medium\":     {Region: \"eu-west-1\", Type: \"is4gen.medium\", Memory: kresource.MustParse(\"6144Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.15938},\n\t\t\"is4gen.large\":      {Region: \"eu-west-1\", Type: \"is4gen.large\", Memory: kresource.MustParse(\"12288Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.31875},\n\t\t\"is4gen.xlarge\":     {Region: \"eu-west-1\", Type: \"is4gen.xlarge\", Memory: kresource.MustParse(\"24576Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.6375},\n\t\t\"is4gen.2xlarge\":    {Region: \"eu-west-1\", Type: \"is4gen.2xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.275},\n\t\t\"is4gen.4xlarge\":    {Region: \"eu-west-1\", Type: \"is4gen.4xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.55},\n\t\t\"is4gen.8xlarge\":    {Region: \"eu-west-1\", Type: \"is4gen.8xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 5.1},\n\t\t\"m1.small\":          {Region: \"eu-west-1\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.047},\n\t\t\"m1.medium\":         {Region: \"eu-west-1\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.095},\n\t\t\"m1.large\":          {Region: \"eu-west-1\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.19},\n\t\t\"m1.xlarge\":         {Region: \"eu-west-1\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.379},\n\t\t\"m2.xlarge\":         {Region: \"eu-west-1\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.275},\n\t\t\"m2.2xlarge\":        {Region: \"eu-west-1\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.55},\n\t\t\"m2.4xlarge\":        {Region: \"eu-west-1\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.1},\n\t\t\"m3.medium\":         {Region: \"eu-west-1\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.073},\n\t\t\"m3.large\":          {Region: \"eu-west-1\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.146},\n\t\t\"m3.xlarge\":         {Region: \"eu-west-1\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.293},\n\t\t\"m3.2xlarge\":        {Region: \"eu-west-1\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.585},\n\t\t\"m4.large\":          {Region: \"eu-west-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.111},\n\t\t\"m4.xlarge\":         {Region: \"eu-west-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.222},\n\t\t\"m4.2xlarge\":        {Region: \"eu-west-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.444},\n\t\t\"m4.4xlarge\":        {Region: \"eu-west-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.888},\n\t\t\"m4.10xlarge\":       {Region: \"eu-west-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.22},\n\t\t\"m4.16xlarge\":       {Region: \"eu-west-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"m5.large\":          {Region: \"eu-west-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.107},\n\t\t\"m5.xlarge\":         {Region: \"eu-west-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.214},\n\t\t\"m5.2xlarge\":        {Region: \"eu-west-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.428},\n\t\t\"m5.4xlarge\":        {Region: \"eu-west-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.856},\n\t\t\"m5.8xlarge\":        {Region: \"eu-west-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.712},\n\t\t\"m5.12xlarge\":       {Region: \"eu-west-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.568},\n\t\t\"m5.16xlarge\":       {Region: \"eu-west-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.424},\n\t\t\"m5.24xlarge\":       {Region: \"eu-west-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"m5.metal\":          {Region: \"eu-west-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"m5a.large\":         {Region: \"eu-west-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m5a.xlarge\":        {Region: \"eu-west-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m5a.2xlarge\":       {Region: \"eu-west-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m5a.4xlarge\":       {Region: \"eu-west-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m5a.8xlarge\":       {Region: \"eu-west-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m5a.12xlarge\":      {Region: \"eu-west-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m5a.16xlarge\":      {Region: \"eu-west-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m5a.24xlarge\":      {Region: \"eu-west-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m5ad.large\":        {Region: \"eu-west-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.115},\n\t\t\"m5ad.xlarge\":       {Region: \"eu-west-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.23},\n\t\t\"m5ad.2xlarge\":      {Region: \"eu-west-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.46},\n\t\t\"m5ad.4xlarge\":      {Region: \"eu-west-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.92},\n\t\t\"m5ad.8xlarge\":      {Region: \"eu-west-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.84},\n\t\t\"m5ad.12xlarge\":     {Region: \"eu-west-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.76},\n\t\t\"m5ad.16xlarge\":     {Region: \"eu-west-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.68},\n\t\t\"m5ad.24xlarge\":     {Region: \"eu-west-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"m5d.large\":         {Region: \"eu-west-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"m5d.xlarge\":        {Region: \"eu-west-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"m5d.2xlarge\":       {Region: \"eu-west-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"m5d.4xlarge\":       {Region: \"eu-west-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"m5d.8xlarge\":       {Region: \"eu-west-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"m5d.12xlarge\":      {Region: \"eu-west-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"m5d.16xlarge\":      {Region: \"eu-west-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"m5d.24xlarge\":      {Region: \"eu-west-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"m5d.metal\":         {Region: \"eu-west-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"m5dn.large\":        {Region: \"eu-west-1\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.152},\n\t\t\"m5dn.xlarge\":       {Region: \"eu-west-1\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.304},\n\t\t\"m5dn.2xlarge\":      {Region: \"eu-west-1\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.608},\n\t\t\"m5dn.4xlarge\":      {Region: \"eu-west-1\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.216},\n\t\t\"m5dn.8xlarge\":      {Region: \"eu-west-1\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.432},\n\t\t\"m5dn.12xlarge\":     {Region: \"eu-west-1\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"m5dn.16xlarge\":     {Region: \"eu-west-1\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.864},\n\t\t\"m5dn.24xlarge\":     {Region: \"eu-west-1\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"m5dn.metal\":        {Region: \"eu-west-1\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"m5n.large\":         {Region: \"eu-west-1\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"m5n.xlarge\":        {Region: \"eu-west-1\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"m5n.2xlarge\":       {Region: \"eu-west-1\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"m5n.4xlarge\":       {Region: \"eu-west-1\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"m5n.8xlarge\":       {Region: \"eu-west-1\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"m5n.12xlarge\":      {Region: \"eu-west-1\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"m5n.16xlarge\":      {Region: \"eu-west-1\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"m5n.24xlarge\":      {Region: \"eu-west-1\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"m5n.metal\":         {Region: \"eu-west-1\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"m5zn.large\":        {Region: \"eu-west-1\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1841},\n\t\t\"m5zn.xlarge\":       {Region: \"eu-west-1\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3682},\n\t\t\"m5zn.2xlarge\":      {Region: \"eu-west-1\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.7364},\n\t\t\"m5zn.3xlarge\":      {Region: \"eu-west-1\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.1046},\n\t\t\"m5zn.6xlarge\":      {Region: \"eu-west-1\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.2092},\n\t\t\"m5zn.12xlarge\":     {Region: \"eu-west-1\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.4184},\n\t\t\"m5zn.metal\":        {Region: \"eu-west-1\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.4184},\n\t\t\"m6a.large\":         {Region: \"eu-west-1\", Type: \"m6a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0963},\n\t\t\"m6a.xlarge\":        {Region: \"eu-west-1\", Type: \"m6a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1926},\n\t\t\"m6a.2xlarge\":       {Region: \"eu-west-1\", Type: \"m6a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3852},\n\t\t\"m6a.4xlarge\":       {Region: \"eu-west-1\", Type: \"m6a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7704},\n\t\t\"m6a.8xlarge\":       {Region: \"eu-west-1\", Type: \"m6a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.5408},\n\t\t\"m6a.12xlarge\":      {Region: \"eu-west-1\", Type: \"m6a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.3112},\n\t\t\"m6a.16xlarge\":      {Region: \"eu-west-1\", Type: \"m6a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.0816},\n\t\t\"m6a.24xlarge\":      {Region: \"eu-west-1\", Type: \"m6a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.6224},\n\t\t\"m6a.32xlarge\":      {Region: \"eu-west-1\", Type: \"m6a.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.1632},\n\t\t\"m6a.48xlarge\":      {Region: \"eu-west-1\", Type: \"m6a.48xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 9.2448},\n\t\t\"m6a.metal\":         {Region: \"eu-west-1\", Type: \"m6a.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 9.2448},\n\t\t\"m6g.medium\":        {Region: \"eu-west-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.043},\n\t\t\"m6g.large\":         {Region: \"eu-west-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"m6g.xlarge\":        {Region: \"eu-west-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"m6g.2xlarge\":       {Region: \"eu-west-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"m6g.4xlarge\":       {Region: \"eu-west-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"m6g.8xlarge\":       {Region: \"eu-west-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"m6g.12xlarge\":      {Region: \"eu-west-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"m6g.16xlarge\":      {Region: \"eu-west-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"m6g.metal\":         {Region: \"eu-west-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"m6gd.medium\":       {Region: \"eu-west-1\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0504},\n\t\t\"m6gd.large\":        {Region: \"eu-west-1\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1008},\n\t\t\"m6gd.xlarge\":       {Region: \"eu-west-1\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2016},\n\t\t\"m6gd.2xlarge\":      {Region: \"eu-west-1\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4032},\n\t\t\"m6gd.4xlarge\":      {Region: \"eu-west-1\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8064},\n\t\t\"m6gd.8xlarge\":      {Region: \"eu-west-1\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6128},\n\t\t\"m6gd.12xlarge\":     {Region: \"eu-west-1\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.4192},\n\t\t\"m6gd.16xlarge\":     {Region: \"eu-west-1\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2256},\n\t\t\"m6gd.metal\":        {Region: \"eu-west-1\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2256},\n\t\t\"m6i.large\":         {Region: \"eu-west-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.107},\n\t\t\"m6i.xlarge\":        {Region: \"eu-west-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.214},\n\t\t\"m6i.2xlarge\":       {Region: \"eu-west-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.428},\n\t\t\"m6i.4xlarge\":       {Region: \"eu-west-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.856},\n\t\t\"m6i.8xlarge\":       {Region: \"eu-west-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.712},\n\t\t\"m6i.12xlarge\":      {Region: \"eu-west-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.568},\n\t\t\"m6i.16xlarge\":      {Region: \"eu-west-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.424},\n\t\t\"m6i.24xlarge\":      {Region: \"eu-west-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.136},\n\t\t\"m6i.32xlarge\":      {Region: \"eu-west-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.848},\n\t\t\"m6i.metal\":         {Region: \"eu-west-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.848},\n\t\t\"p2.xlarge\":         {Region: \"eu-west-1\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.972},\n\t\t\"p2.8xlarge\":        {Region: \"eu-west-1\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 7.776},\n\t\t\"p2.16xlarge\":       {Region: \"eu-west-1\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 15.552},\n\t\t\"p3.2xlarge\":        {Region: \"eu-west-1\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 3.305},\n\t\t\"p3.8xlarge\":        {Region: \"eu-west-1\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 13.22},\n\t\t\"p3.16xlarge\":       {Region: \"eu-west-1\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 26.44},\n\t\t\"p3dn.24xlarge\":     {Region: \"eu-west-1\", Type: \"p3dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 33.711},\n\t\t\"p4d.24xlarge\":      {Region: \"eu-west-1\", Type: \"p4d.24xlarge\", Memory: kresource.MustParse(\"1179648Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 35.39655},\n\t\t\"r3.large\":          {Region: \"eu-west-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.185},\n\t\t\"r3.xlarge\":         {Region: \"eu-west-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.371},\n\t\t\"r3.2xlarge\":        {Region: \"eu-west-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.741},\n\t\t\"r3.4xlarge\":        {Region: \"eu-west-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.482},\n\t\t\"r3.8xlarge\":        {Region: \"eu-west-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.964},\n\t\t\"r4.large\":          {Region: \"eu-west-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1482},\n\t\t\"r4.xlarge\":         {Region: \"eu-west-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2964},\n\t\t\"r4.2xlarge\":        {Region: \"eu-west-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5928},\n\t\t\"r4.4xlarge\":        {Region: \"eu-west-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.1856},\n\t\t\"r4.8xlarge\":        {Region: \"eu-west-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.3712},\n\t\t\"r4.16xlarge\":       {Region: \"eu-west-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.7424},\n\t\t\"r5.large\":          {Region: \"eu-west-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.141},\n\t\t\"r5.xlarge\":         {Region: \"eu-west-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.282},\n\t\t\"r5.2xlarge\":        {Region: \"eu-west-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.564},\n\t\t\"r5.4xlarge\":        {Region: \"eu-west-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.128},\n\t\t\"r5.8xlarge\":        {Region: \"eu-west-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.256},\n\t\t\"r5.12xlarge\":       {Region: \"eu-west-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.384},\n\t\t\"r5.16xlarge\":       {Region: \"eu-west-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.512},\n\t\t\"r5.24xlarge\":       {Region: \"eu-west-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.768},\n\t\t\"r5.metal\":          {Region: \"eu-west-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.768},\n\t\t\"r5a.large\":         {Region: \"eu-west-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.127},\n\t\t\"r5a.xlarge\":        {Region: \"eu-west-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.254},\n\t\t\"r5a.2xlarge\":       {Region: \"eu-west-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.508},\n\t\t\"r5a.4xlarge\":       {Region: \"eu-west-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.016},\n\t\t\"r5a.8xlarge\":       {Region: \"eu-west-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.032},\n\t\t\"r5a.12xlarge\":      {Region: \"eu-west-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.048},\n\t\t\"r5a.16xlarge\":      {Region: \"eu-west-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.064},\n\t\t\"r5a.24xlarge\":      {Region: \"eu-west-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.096},\n\t\t\"r5ad.large\":        {Region: \"eu-west-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.146},\n\t\t\"r5ad.xlarge\":       {Region: \"eu-west-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.292},\n\t\t\"r5ad.2xlarge\":      {Region: \"eu-west-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.584},\n\t\t\"r5ad.4xlarge\":      {Region: \"eu-west-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.168},\n\t\t\"r5ad.8xlarge\":      {Region: \"eu-west-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.336},\n\t\t\"r5ad.12xlarge\":     {Region: \"eu-west-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.504},\n\t\t\"r5ad.16xlarge\":     {Region: \"eu-west-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.672},\n\t\t\"r5ad.24xlarge\":     {Region: \"eu-west-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.008},\n\t\t\"r5b.large\":         {Region: \"eu-west-1\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"r5b.xlarge\":        {Region: \"eu-west-1\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"r5b.2xlarge\":       {Region: \"eu-west-1\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"r5b.4xlarge\":       {Region: \"eu-west-1\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"r5b.8xlarge\":       {Region: \"eu-west-1\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"r5b.12xlarge\":      {Region: \"eu-west-1\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"r5b.16xlarge\":      {Region: \"eu-west-1\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"r5b.24xlarge\":      {Region: \"eu-west-1\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5b.metal\":         {Region: \"eu-west-1\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5d.large\":         {Region: \"eu-west-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.16},\n\t\t\"r5d.xlarge\":        {Region: \"eu-west-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.32},\n\t\t\"r5d.2xlarge\":       {Region: \"eu-west-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.64},\n\t\t\"r5d.4xlarge\":       {Region: \"eu-west-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.28},\n\t\t\"r5d.8xlarge\":       {Region: \"eu-west-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.56},\n\t\t\"r5d.12xlarge\":      {Region: \"eu-west-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.84},\n\t\t\"r5d.16xlarge\":      {Region: \"eu-west-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.12},\n\t\t\"r5d.24xlarge\":      {Region: \"eu-west-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.68},\n\t\t\"r5d.metal\":         {Region: \"eu-west-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.68},\n\t\t\"r5dn.large\":        {Region: \"eu-west-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"r5dn.xlarge\":       {Region: \"eu-west-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"r5dn.2xlarge\":      {Region: \"eu-west-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.744},\n\t\t\"r5dn.4xlarge\":      {Region: \"eu-west-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.488},\n\t\t\"r5dn.8xlarge\":      {Region: \"eu-west-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.976},\n\t\t\"r5dn.12xlarge\":     {Region: \"eu-west-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"r5dn.16xlarge\":     {Region: \"eu-west-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.952},\n\t\t\"r5dn.24xlarge\":     {Region: \"eu-west-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.928},\n\t\t\"r5dn.metal\":        {Region: \"eu-west-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.928},\n\t\t\"r5n.large\":         {Region: \"eu-west-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"r5n.xlarge\":        {Region: \"eu-west-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"r5n.2xlarge\":       {Region: \"eu-west-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"r5n.4xlarge\":       {Region: \"eu-west-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"r5n.8xlarge\":       {Region: \"eu-west-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"r5n.12xlarge\":      {Region: \"eu-west-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"r5n.16xlarge\":      {Region: \"eu-west-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"r5n.24xlarge\":      {Region: \"eu-west-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5n.metal\":         {Region: \"eu-west-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r6g.medium\":        {Region: \"eu-west-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0564},\n\t\t\"r6g.large\":         {Region: \"eu-west-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1128},\n\t\t\"r6g.xlarge\":        {Region: \"eu-west-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2256},\n\t\t\"r6g.2xlarge\":       {Region: \"eu-west-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4512},\n\t\t\"r6g.4xlarge\":       {Region: \"eu-west-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9024},\n\t\t\"r6g.8xlarge\":       {Region: \"eu-west-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.8048},\n\t\t\"r6g.12xlarge\":      {Region: \"eu-west-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.7072},\n\t\t\"r6g.16xlarge\":      {Region: \"eu-west-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.6096},\n\t\t\"r6g.metal\":         {Region: \"eu-west-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.6096},\n\t\t\"r6gd.medium\":       {Region: \"eu-west-1\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.064},\n\t\t\"r6gd.large\":        {Region: \"eu-west-1\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.128},\n\t\t\"r6gd.xlarge\":       {Region: \"eu-west-1\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.256},\n\t\t\"r6gd.2xlarge\":      {Region: \"eu-west-1\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.512},\n\t\t\"r6gd.4xlarge\":      {Region: \"eu-west-1\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.024},\n\t\t\"r6gd.8xlarge\":      {Region: \"eu-west-1\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.048},\n\t\t\"r6gd.12xlarge\":     {Region: \"eu-west-1\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"r6gd.16xlarge\":     {Region: \"eu-west-1\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.096},\n\t\t\"r6gd.metal\":        {Region: \"eu-west-1\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.096},\n\t\t\"r6i.large\":         {Region: \"eu-west-1\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.141},\n\t\t\"r6i.xlarge\":        {Region: \"eu-west-1\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.282},\n\t\t\"r6i.2xlarge\":       {Region: \"eu-west-1\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.564},\n\t\t\"r6i.4xlarge\":       {Region: \"eu-west-1\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.128},\n\t\t\"r6i.8xlarge\":       {Region: \"eu-west-1\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.256},\n\t\t\"r6i.12xlarge\":      {Region: \"eu-west-1\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.384},\n\t\t\"r6i.16xlarge\":      {Region: \"eu-west-1\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.512},\n\t\t\"r6i.24xlarge\":      {Region: \"eu-west-1\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.768},\n\t\t\"r6i.32xlarge\":      {Region: \"eu-west-1\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.024},\n\t\t\"r6i.metal\":         {Region: \"eu-west-1\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.024},\n\t\t\"t1.micro\":          {Region: \"eu-west-1\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.02},\n\t\t\"t2.nano\":           {Region: \"eu-west-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0063},\n\t\t\"t2.micro\":          {Region: \"eu-west-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0126},\n\t\t\"t2.small\":          {Region: \"eu-west-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.025},\n\t\t\"t2.medium\":         {Region: \"eu-west-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.05},\n\t\t\"t2.large\":          {Region: \"eu-west-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1008},\n\t\t\"t2.xlarge\":         {Region: \"eu-west-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2016},\n\t\t\"t2.2xlarge\":        {Region: \"eu-west-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4032},\n\t\t\"t3.nano\":           {Region: \"eu-west-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0057},\n\t\t\"t3.micro\":          {Region: \"eu-west-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0114},\n\t\t\"t3.small\":          {Region: \"eu-west-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0228},\n\t\t\"t3.medium\":         {Region: \"eu-west-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0456},\n\t\t\"t3.large\":          {Region: \"eu-west-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0912},\n\t\t\"t3.xlarge\":         {Region: \"eu-west-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1824},\n\t\t\"t3.2xlarge\":        {Region: \"eu-west-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3648},\n\t\t\"t3a.nano\":          {Region: \"eu-west-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0051},\n\t\t\"t3a.micro\":         {Region: \"eu-west-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0102},\n\t\t\"t3a.small\":         {Region: \"eu-west-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0204},\n\t\t\"t3a.medium\":        {Region: \"eu-west-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0408},\n\t\t\"t3a.large\":         {Region: \"eu-west-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0816},\n\t\t\"t3a.xlarge\":        {Region: \"eu-west-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1632},\n\t\t\"t3a.2xlarge\":       {Region: \"eu-west-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3264},\n\t\t\"t4g.nano\":          {Region: \"eu-west-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0046},\n\t\t\"t4g.micro\":         {Region: \"eu-west-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0092},\n\t\t\"t4g.small\":         {Region: \"eu-west-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0184},\n\t\t\"t4g.medium\":        {Region: \"eu-west-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0368},\n\t\t\"t4g.large\":         {Region: \"eu-west-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0736},\n\t\t\"t4g.xlarge\":        {Region: \"eu-west-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1472},\n\t\t\"t4g.2xlarge\":       {Region: \"eu-west-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2944},\n\t\t\"u-12tb1.112xlarge\": {Region: \"eu-west-1\", Type: \"u-12tb1.112xlarge\", Memory: kresource.MustParse(\"12582912Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 122.2},\n\t\t\"u-3tb1.56xlarge\":   {Region: \"eu-west-1\", Type: \"u-3tb1.56xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 30.55},\n\t\t\"u-6tb1.56xlarge\":   {Region: \"eu-west-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 51.92818},\n\t\t\"u-6tb1.112xlarge\":  {Region: \"eu-west-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 61.1},\n\t\t\"u-9tb1.112xlarge\":  {Region: \"eu-west-1\", Type: \"u-9tb1.112xlarge\", Memory: kresource.MustParse(\"9437184Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 91.65},\n\t\t\"vt1.3xlarge\":       {Region: \"eu-west-1\", Type: \"vt1.3xlarge\", Memory: kresource.MustParse(\"24576Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 0.73412},\n\t\t\"vt1.6xlarge\":       {Region: \"eu-west-1\", Type: \"vt1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 1.46824},\n\t\t\"vt1.24xlarge\":      {Region: \"eu-west-1\", Type: \"vt1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.87294},\n\t\t\"x1.16xlarge\":       {Region: \"eu-west-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 8.003},\n\t\t\"x1.32xlarge\":       {Region: \"eu-west-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 16.006},\n\t\t\"x1e.xlarge\":        {Region: \"eu-west-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"x1e.2xlarge\":       {Region: \"eu-west-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"x1e.4xlarge\":       {Region: \"eu-west-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.0},\n\t\t\"x1e.8xlarge\":       {Region: \"eu-west-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.0},\n\t\t\"x1e.16xlarge\":      {Region: \"eu-west-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 16.0},\n\t\t\"x1e.32xlarge\":      {Region: \"eu-west-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 32.0},\n\t\t\"x2gd.medium\":       {Region: \"eu-west-1\", Type: \"x2gd.medium\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"x2gd.large\":        {Region: \"eu-west-1\", Type: \"x2gd.large\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"x2gd.xlarge\":       {Region: \"eu-west-1\", Type: \"x2gd.xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"x2gd.2xlarge\":      {Region: \"eu-west-1\", Type: \"x2gd.2xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"x2gd.4xlarge\":      {Region: \"eu-west-1\", Type: \"x2gd.4xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.6},\n\t\t\"x2gd.8xlarge\":      {Region: \"eu-west-1\", Type: \"x2gd.8xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"x2gd.12xlarge\":     {Region: \"eu-west-1\", Type: \"x2gd.12xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.8},\n\t\t\"x2gd.16xlarge\":     {Region: \"eu-west-1\", Type: \"x2gd.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.4},\n\t\t\"x2gd.metal\":        {Region: \"eu-west-1\", Type: \"x2gd.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.4},\n\t\t\"x2idn.16xlarge\":    {Region: \"eu-west-1\", Type: \"x2idn.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 8.003},\n\t\t\"x2idn.24xlarge\":    {Region: \"eu-west-1\", Type: \"x2idn.24xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.0045},\n\t\t\"x2idn.32xlarge\":    {Region: \"eu-west-1\", Type: \"x2idn.32xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 16.006},\n\t\t\"x2iedn.xlarge\":     {Region: \"eu-west-1\", Type: \"x2iedn.xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.00038},\n\t\t\"x2iedn.2xlarge\":    {Region: \"eu-west-1\", Type: \"x2iedn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.00075},\n\t\t\"x2iedn.4xlarge\":    {Region: \"eu-west-1\", Type: \"x2iedn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.0015},\n\t\t\"x2iedn.8xlarge\":    {Region: \"eu-west-1\", Type: \"x2iedn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.003},\n\t\t\"x2iedn.16xlarge\":   {Region: \"eu-west-1\", Type: \"x2iedn.16xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 16.006},\n\t\t\"x2iedn.24xlarge\":   {Region: \"eu-west-1\", Type: \"x2iedn.24xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 24.009},\n\t\t\"x2iedn.32xlarge\":   {Region: \"eu-west-1\", Type: \"x2iedn.32xlarge\", Memory: kresource.MustParse(\"4194304Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 32.012},\n\t\t\"x2iezn.2xlarge\":    {Region: \"eu-west-1\", Type: \"x2iezn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"x2iezn.4xlarge\":    {Region: \"eu-west-1\", Type: \"x2iezn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.0},\n\t\t\"x2iezn.6xlarge\":    {Region: \"eu-west-1\", Type: \"x2iezn.6xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 6.0},\n\t\t\"x2iezn.8xlarge\":    {Region: \"eu-west-1\", Type: \"x2iezn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.0},\n\t\t\"x2iezn.12xlarge\":   {Region: \"eu-west-1\", Type: \"x2iezn.12xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 12.0},\n\t\t\"x2iezn.metal\":      {Region: \"eu-west-1\", Type: \"x2iezn.metal\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 12.0},\n\t\t\"z1d.large\":         {Region: \"eu-west-1\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.208},\n\t\t\"z1d.xlarge\":        {Region: \"eu-west-1\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.416},\n\t\t\"z1d.2xlarge\":       {Region: \"eu-west-1\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.832},\n\t\t\"z1d.3xlarge\":       {Region: \"eu-west-1\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.248},\n\t\t\"z1d.6xlarge\":       {Region: \"eu-west-1\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.496},\n\t\t\"z1d.12xlarge\":      {Region: \"eu-west-1\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"z1d.metal\":         {Region: \"eu-west-1\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.992},\n\t},\n\t\"eu-west-2\": {\n\t\t\"c4.large\":      {Region: \"eu-west-2\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.119},\n\t\t\"c4.xlarge\":     {Region: \"eu-west-2\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.237},\n\t\t\"c4.2xlarge\":    {Region: \"eu-west-2\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.476},\n\t\t\"c4.4xlarge\":    {Region: \"eu-west-2\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.95},\n\t\t\"c4.8xlarge\":    {Region: \"eu-west-2\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.902},\n\t\t\"c5.large\":      {Region: \"eu-west-2\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"c5.xlarge\":     {Region: \"eu-west-2\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"c5.2xlarge\":    {Region: \"eu-west-2\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"c5.4xlarge\":    {Region: \"eu-west-2\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"c5.9xlarge\":    {Region: \"eu-west-2\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.818},\n\t\t\"c5.12xlarge\":   {Region: \"eu-west-2\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"c5.18xlarge\":   {Region: \"eu-west-2\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.636},\n\t\t\"c5.24xlarge\":   {Region: \"eu-west-2\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c5.metal\":      {Region: \"eu-west-2\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c5a.large\":     {Region: \"eu-west-2\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.091},\n\t\t\"c5a.xlarge\":    {Region: \"eu-west-2\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.182},\n\t\t\"c5a.2xlarge\":   {Region: \"eu-west-2\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.364},\n\t\t\"c5a.4xlarge\":   {Region: \"eu-west-2\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.728},\n\t\t\"c5a.8xlarge\":   {Region: \"eu-west-2\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.456},\n\t\t\"c5a.12xlarge\":  {Region: \"eu-west-2\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.184},\n\t\t\"c5a.16xlarge\":  {Region: \"eu-west-2\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.912},\n\t\t\"c5a.24xlarge\":  {Region: \"eu-west-2\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.368},\n\t\t\"c5d.large\":     {Region: \"eu-west-2\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.115},\n\t\t\"c5d.xlarge\":    {Region: \"eu-west-2\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.23},\n\t\t\"c5d.2xlarge\":   {Region: \"eu-west-2\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.46},\n\t\t\"c5d.4xlarge\":   {Region: \"eu-west-2\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.92},\n\t\t\"c5d.9xlarge\":   {Region: \"eu-west-2\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.07},\n\t\t\"c5d.12xlarge\":  {Region: \"eu-west-2\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.76},\n\t\t\"c5d.18xlarge\":  {Region: \"eu-west-2\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.14},\n\t\t\"c5d.24xlarge\":  {Region: \"eu-west-2\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"c5d.metal\":     {Region: \"eu-west-2\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"c5n.large\":     {Region: \"eu-west-2\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.128},\n\t\t\"c5n.xlarge\":    {Region: \"eu-west-2\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.256},\n\t\t\"c5n.2xlarge\":   {Region: \"eu-west-2\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.512},\n\t\t\"c5n.4xlarge\":   {Region: \"eu-west-2\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.024},\n\t\t\"c5n.9xlarge\":   {Region: \"eu-west-2\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c5n.18xlarge\":  {Region: \"eu-west-2\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5n.metal\":     {Region: \"eu-west-2\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c6g.medium\":    {Region: \"eu-west-2\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0404},\n\t\t\"c6g.large\":     {Region: \"eu-west-2\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0808},\n\t\t\"c6g.xlarge\":    {Region: \"eu-west-2\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1616},\n\t\t\"c6g.2xlarge\":   {Region: \"eu-west-2\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3232},\n\t\t\"c6g.4xlarge\":   {Region: \"eu-west-2\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6464},\n\t\t\"c6g.8xlarge\":   {Region: \"eu-west-2\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.2928},\n\t\t\"c6g.12xlarge\":  {Region: \"eu-west-2\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.9392},\n\t\t\"c6g.16xlarge\":  {Region: \"eu-west-2\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.5856},\n\t\t\"c6g.metal\":     {Region: \"eu-west-2\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.5856},\n\t\t\"c6gd.medium\":   {Region: \"eu-west-2\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.046},\n\t\t\"c6gd.large\":    {Region: \"eu-west-2\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.092},\n\t\t\"c6gd.xlarge\":   {Region: \"eu-west-2\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.184},\n\t\t\"c6gd.2xlarge\":  {Region: \"eu-west-2\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.368},\n\t\t\"c6gd.4xlarge\":  {Region: \"eu-west-2\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.736},\n\t\t\"c6gd.8xlarge\":  {Region: \"eu-west-2\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.472},\n\t\t\"c6gd.12xlarge\": {Region: \"eu-west-2\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.208},\n\t\t\"c6gd.16xlarge\": {Region: \"eu-west-2\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.944},\n\t\t\"c6gd.metal\":    {Region: \"eu-west-2\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.944},\n\t\t\"c6gn.medium\":   {Region: \"eu-west-2\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.05125},\n\t\t\"c6gn.large\":    {Region: \"eu-west-2\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1025},\n\t\t\"c6gn.xlarge\":   {Region: \"eu-west-2\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.205},\n\t\t\"c6gn.2xlarge\":  {Region: \"eu-west-2\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.41},\n\t\t\"c6gn.4xlarge\":  {Region: \"eu-west-2\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.82},\n\t\t\"c6gn.8xlarge\":  {Region: \"eu-west-2\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.64},\n\t\t\"c6gn.12xlarge\": {Region: \"eu-west-2\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.46},\n\t\t\"c6gn.16xlarge\": {Region: \"eu-west-2\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.28},\n\t\t\"c6i.large\":     {Region: \"eu-west-2\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"c6i.xlarge\":    {Region: \"eu-west-2\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"c6i.2xlarge\":   {Region: \"eu-west-2\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"c6i.4xlarge\":   {Region: \"eu-west-2\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"c6i.8xlarge\":   {Region: \"eu-west-2\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"c6i.12xlarge\":  {Region: \"eu-west-2\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"c6i.16xlarge\":  {Region: \"eu-west-2\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"c6i.24xlarge\":  {Region: \"eu-west-2\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c6i.32xlarge\":  {Region: \"eu-west-2\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.464},\n\t\t\"c6i.metal\":     {Region: \"eu-west-2\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.464},\n\t\t\"d2.xlarge\":     {Region: \"eu-west-2\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.772},\n\t\t\"d2.2xlarge\":    {Region: \"eu-west-2\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.544},\n\t\t\"d2.4xlarge\":    {Region: \"eu-west-2\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.087},\n\t\t\"d2.8xlarge\":    {Region: \"eu-west-2\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.174},\n\t\t\"d3.xlarge\":     {Region: \"eu-west-2\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.64},\n\t\t\"d3.2xlarge\":    {Region: \"eu-west-2\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.28},\n\t\t\"d3.4xlarge\":    {Region: \"eu-west-2\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.559},\n\t\t\"d3.8xlarge\":    {Region: \"eu-west-2\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 5.11824},\n\t\t\"f1.2xlarge\":    {Region: \"eu-west-2\", Type: \"f1.2xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.906},\n\t\t\"f1.4xlarge\":    {Region: \"eu-west-2\", Type: \"f1.4xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.812},\n\t\t\"g3.4xlarge\":    {Region: \"eu-west-2\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.429},\n\t\t\"g3.8xlarge\":    {Region: \"eu-west-2\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.858},\n\t\t\"g3.16xlarge\":   {Region: \"eu-west-2\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 5.716},\n\t\t\"g3s.xlarge\":    {Region: \"eu-west-2\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.94},\n\t\t\"g4ad.xlarge\":   {Region: \"eu-west-2\", Type: \"g4ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.44271},\n\t\t\"g4ad.2xlarge\":  {Region: \"eu-west-2\", Type: \"g4ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.63292},\n\t\t\"g4ad.4xlarge\":  {Region: \"eu-west-2\", Type: \"g4ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.014},\n\t\t\"g4ad.8xlarge\":  {Region: \"eu-west-2\", Type: \"g4ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.028},\n\t\t\"g4ad.16xlarge\": {Region: \"eu-west-2\", Type: \"g4ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 4.056},\n\t\t\"g4dn.xlarge\":   {Region: \"eu-west-2\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.615},\n\t\t\"g4dn.2xlarge\":  {Region: \"eu-west-2\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.88},\n\t\t\"g4dn.4xlarge\":  {Region: \"eu-west-2\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.409},\n\t\t\"g4dn.8xlarge\":  {Region: \"eu-west-2\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.546},\n\t\t\"g4dn.12xlarge\": {Region: \"eu-west-2\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.577},\n\t\t\"g4dn.16xlarge\": {Region: \"eu-west-2\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.092},\n\t\t\"g4dn.metal\":    {Region: \"eu-west-2\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 9.154},\n\t\t\"i3.large\":      {Region: \"eu-west-2\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.181},\n\t\t\"i3.xlarge\":     {Region: \"eu-west-2\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.362},\n\t\t\"i3.2xlarge\":    {Region: \"eu-west-2\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.724},\n\t\t\"i3.4xlarge\":    {Region: \"eu-west-2\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.448},\n\t\t\"i3.8xlarge\":    {Region: \"eu-west-2\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.896},\n\t\t\"i3.16xlarge\":   {Region: \"eu-west-2\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"i3.metal\":      {Region: \"eu-west-2\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"i3en.large\":    {Region: \"eu-west-2\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.263},\n\t\t\"i3en.xlarge\":   {Region: \"eu-west-2\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.526},\n\t\t\"i3en.2xlarge\":  {Region: \"eu-west-2\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.052},\n\t\t\"i3en.3xlarge\":  {Region: \"eu-west-2\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.578},\n\t\t\"i3en.6xlarge\":  {Region: \"eu-west-2\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.156},\n\t\t\"i3en.12xlarge\": {Region: \"eu-west-2\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.312},\n\t\t\"i3en.24xlarge\": {Region: \"eu-west-2\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.624},\n\t\t\"i3en.metal\":    {Region: \"eu-west-2\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.624},\n\t\t\"inf1.xlarge\":   {Region: \"eu-west-2\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.267},\n\t\t\"inf1.2xlarge\":  {Region: \"eu-west-2\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.424},\n\t\t\"inf1.6xlarge\":  {Region: \"eu-west-2\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.382},\n\t\t\"inf1.24xlarge\": {Region: \"eu-west-2\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.53},\n\t\t\"m4.large\":      {Region: \"eu-west-2\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.116},\n\t\t\"m4.xlarge\":     {Region: \"eu-west-2\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.232},\n\t\t\"m4.2xlarge\":    {Region: \"eu-west-2\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.464},\n\t\t\"m4.4xlarge\":    {Region: \"eu-west-2\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.928},\n\t\t\"m4.10xlarge\":   {Region: \"eu-west-2\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.32},\n\t\t\"m4.16xlarge\":   {Region: \"eu-west-2\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.712},\n\t\t\"m5.large\":      {Region: \"eu-west-2\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.111},\n\t\t\"m5.xlarge\":     {Region: \"eu-west-2\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.222},\n\t\t\"m5.2xlarge\":    {Region: \"eu-west-2\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.444},\n\t\t\"m5.4xlarge\":    {Region: \"eu-west-2\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.888},\n\t\t\"m5.8xlarge\":    {Region: \"eu-west-2\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.776},\n\t\t\"m5.12xlarge\":   {Region: \"eu-west-2\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.664},\n\t\t\"m5.16xlarge\":   {Region: \"eu-west-2\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"m5.24xlarge\":   {Region: \"eu-west-2\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.328},\n\t\t\"m5.metal\":      {Region: \"eu-west-2\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.328},\n\t\t\"m5a.large\":     {Region: \"eu-west-2\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"m5a.xlarge\":    {Region: \"eu-west-2\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"m5a.2xlarge\":   {Region: \"eu-west-2\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"m5a.4xlarge\":   {Region: \"eu-west-2\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"m5a.8xlarge\":   {Region: \"eu-west-2\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6},\n\t\t\"m5a.12xlarge\":  {Region: \"eu-west-2\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.4},\n\t\t\"m5a.16xlarge\":  {Region: \"eu-west-2\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"m5a.24xlarge\":  {Region: \"eu-west-2\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.8},\n\t\t\"m5ad.large\":    {Region: \"eu-west-2\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"m5ad.xlarge\":   {Region: \"eu-west-2\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"m5ad.2xlarge\":  {Region: \"eu-west-2\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"m5ad.4xlarge\":  {Region: \"eu-west-2\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.96},\n\t\t\"m5ad.8xlarge\":  {Region: \"eu-west-2\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.92},\n\t\t\"m5ad.12xlarge\": {Region: \"eu-west-2\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m5ad.16xlarge\": {Region: \"eu-west-2\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.84},\n\t\t\"m5ad.24xlarge\": {Region: \"eu-west-2\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m5d.large\":     {Region: \"eu-west-2\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.131},\n\t\t\"m5d.xlarge\":    {Region: \"eu-west-2\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.262},\n\t\t\"m5d.2xlarge\":   {Region: \"eu-west-2\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"m5d.4xlarge\":   {Region: \"eu-west-2\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.048},\n\t\t\"m5d.8xlarge\":   {Region: \"eu-west-2\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.096},\n\t\t\"m5d.12xlarge\":  {Region: \"eu-west-2\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.144},\n\t\t\"m5d.16xlarge\":  {Region: \"eu-west-2\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.192},\n\t\t\"m5d.24xlarge\":  {Region: \"eu-west-2\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"m5d.metal\":     {Region: \"eu-west-2\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"m6g.medium\":    {Region: \"eu-west-2\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0444},\n\t\t\"m6g.large\":     {Region: \"eu-west-2\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0888},\n\t\t\"m6g.xlarge\":    {Region: \"eu-west-2\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1776},\n\t\t\"m6g.2xlarge\":   {Region: \"eu-west-2\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3552},\n\t\t\"m6g.4xlarge\":   {Region: \"eu-west-2\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7104},\n\t\t\"m6g.8xlarge\":   {Region: \"eu-west-2\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.4208},\n\t\t\"m6g.12xlarge\":  {Region: \"eu-west-2\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.1312},\n\t\t\"m6g.16xlarge\":  {Region: \"eu-west-2\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8416},\n\t\t\"m6g.metal\":     {Region: \"eu-west-2\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8416},\n\t\t\"m6gd.medium\":   {Region: \"eu-west-2\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0524},\n\t\t\"m6gd.large\":    {Region: \"eu-west-2\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1048},\n\t\t\"m6gd.xlarge\":   {Region: \"eu-west-2\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2096},\n\t\t\"m6gd.2xlarge\":  {Region: \"eu-west-2\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4192},\n\t\t\"m6gd.4xlarge\":  {Region: \"eu-west-2\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8384},\n\t\t\"m6gd.8xlarge\":  {Region: \"eu-west-2\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6768},\n\t\t\"m6gd.12xlarge\": {Region: \"eu-west-2\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.5152},\n\t\t\"m6gd.16xlarge\": {Region: \"eu-west-2\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.3536},\n\t\t\"m6gd.metal\":    {Region: \"eu-west-2\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.3536},\n\t\t\"m6i.large\":     {Region: \"eu-west-2\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.111},\n\t\t\"m6i.xlarge\":    {Region: \"eu-west-2\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.222},\n\t\t\"m6i.2xlarge\":   {Region: \"eu-west-2\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.444},\n\t\t\"m6i.4xlarge\":   {Region: \"eu-west-2\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.888},\n\t\t\"m6i.8xlarge\":   {Region: \"eu-west-2\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.776},\n\t\t\"m6i.12xlarge\":  {Region: \"eu-west-2\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.664},\n\t\t\"m6i.16xlarge\":  {Region: \"eu-west-2\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"m6i.24xlarge\":  {Region: \"eu-west-2\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.328},\n\t\t\"m6i.32xlarge\":  {Region: \"eu-west-2\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"m6i.metal\":     {Region: \"eu-west-2\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"p3.2xlarge\":    {Region: \"eu-west-2\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 3.589},\n\t\t\"p3.8xlarge\":    {Region: \"eu-west-2\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 14.356},\n\t\t\"p3.16xlarge\":   {Region: \"eu-west-2\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 28.712},\n\t\t\"r4.large\":      {Region: \"eu-west-2\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.156},\n\t\t\"r4.xlarge\":     {Region: \"eu-west-2\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.312},\n\t\t\"r4.2xlarge\":    {Region: \"eu-west-2\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.624},\n\t\t\"r4.4xlarge\":    {Region: \"eu-west-2\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.248},\n\t\t\"r4.8xlarge\":    {Region: \"eu-west-2\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.496},\n\t\t\"r4.16xlarge\":   {Region: \"eu-west-2\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"r5.large\":      {Region: \"eu-west-2\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.148},\n\t\t\"r5.xlarge\":     {Region: \"eu-west-2\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.296},\n\t\t\"r5.2xlarge\":    {Region: \"eu-west-2\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"r5.4xlarge\":    {Region: \"eu-west-2\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.184},\n\t\t\"r5.8xlarge\":    {Region: \"eu-west-2\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.368},\n\t\t\"r5.12xlarge\":   {Region: \"eu-west-2\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"r5.16xlarge\":   {Region: \"eu-west-2\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.736},\n\t\t\"r5.24xlarge\":   {Region: \"eu-west-2\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r5.metal\":      {Region: \"eu-west-2\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r5a.large\":     {Region: \"eu-west-2\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"r5a.xlarge\":    {Region: \"eu-west-2\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"r5a.2xlarge\":   {Region: \"eu-west-2\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"r5a.4xlarge\":   {Region: \"eu-west-2\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"r5a.8xlarge\":   {Region: \"eu-west-2\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"r5a.12xlarge\":  {Region: \"eu-west-2\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"r5a.16xlarge\":  {Region: \"eu-west-2\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"r5a.24xlarge\":  {Region: \"eu-west-2\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"r5ad.large\":    {Region: \"eu-west-2\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"r5ad.xlarge\":   {Region: \"eu-west-2\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"r5ad.2xlarge\":  {Region: \"eu-west-2\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"r5ad.4xlarge\":  {Region: \"eu-west-2\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.232},\n\t\t\"r5ad.8xlarge\":  {Region: \"eu-west-2\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"r5ad.12xlarge\": {Region: \"eu-west-2\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.696},\n\t\t\"r5ad.16xlarge\": {Region: \"eu-west-2\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.928},\n\t\t\"r5ad.24xlarge\": {Region: \"eu-west-2\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.392},\n\t\t\"r5b.large\":     {Region: \"eu-west-2\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.175},\n\t\t\"r5b.xlarge\":    {Region: \"eu-west-2\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.35},\n\t\t\"r5b.2xlarge\":   {Region: \"eu-west-2\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.7},\n\t\t\"r5b.4xlarge\":   {Region: \"eu-west-2\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.4},\n\t\t\"r5b.8xlarge\":   {Region: \"eu-west-2\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.8},\n\t\t\"r5b.12xlarge\":  {Region: \"eu-west-2\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.2},\n\t\t\"r5b.16xlarge\":  {Region: \"eu-west-2\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.6},\n\t\t\"r5b.24xlarge\":  {Region: \"eu-west-2\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"r5b.metal\":     {Region: \"eu-west-2\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"r5d.large\":     {Region: \"eu-west-2\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.169},\n\t\t\"r5d.xlarge\":    {Region: \"eu-west-2\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.338},\n\t\t\"r5d.2xlarge\":   {Region: \"eu-west-2\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.676},\n\t\t\"r5d.4xlarge\":   {Region: \"eu-west-2\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.352},\n\t\t\"r5d.8xlarge\":   {Region: \"eu-west-2\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.704},\n\t\t\"r5d.12xlarge\":  {Region: \"eu-west-2\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.056},\n\t\t\"r5d.16xlarge\":  {Region: \"eu-west-2\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.408},\n\t\t\"r5d.24xlarge\":  {Region: \"eu-west-2\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.112},\n\t\t\"r5d.metal\":     {Region: \"eu-west-2\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.112},\n\t\t\"r5n.large\":     {Region: \"eu-west-2\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.175},\n\t\t\"r5n.xlarge\":    {Region: \"eu-west-2\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.35},\n\t\t\"r5n.2xlarge\":   {Region: \"eu-west-2\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.7},\n\t\t\"r5n.4xlarge\":   {Region: \"eu-west-2\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.4},\n\t\t\"r5n.8xlarge\":   {Region: \"eu-west-2\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.8},\n\t\t\"r5n.12xlarge\":  {Region: \"eu-west-2\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.2},\n\t\t\"r5n.16xlarge\":  {Region: \"eu-west-2\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.6},\n\t\t\"r5n.24xlarge\":  {Region: \"eu-west-2\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"r5n.metal\":     {Region: \"eu-west-2\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"r6g.medium\":    {Region: \"eu-west-2\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0592},\n\t\t\"r6g.large\":     {Region: \"eu-west-2\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1184},\n\t\t\"r6g.xlarge\":    {Region: \"eu-west-2\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2368},\n\t\t\"r6g.2xlarge\":   {Region: \"eu-west-2\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4736},\n\t\t\"r6g.4xlarge\":   {Region: \"eu-west-2\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9472},\n\t\t\"r6g.8xlarge\":   {Region: \"eu-west-2\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.8944},\n\t\t\"r6g.12xlarge\":  {Region: \"eu-west-2\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.8416},\n\t\t\"r6g.16xlarge\":  {Region: \"eu-west-2\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.7888},\n\t\t\"r6g.metal\":     {Region: \"eu-west-2\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.7888},\n\t\t\"r6i.large\":     {Region: \"eu-west-2\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.148},\n\t\t\"r6i.xlarge\":    {Region: \"eu-west-2\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.296},\n\t\t\"r6i.2xlarge\":   {Region: \"eu-west-2\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"r6i.4xlarge\":   {Region: \"eu-west-2\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.184},\n\t\t\"r6i.8xlarge\":   {Region: \"eu-west-2\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.368},\n\t\t\"r6i.12xlarge\":  {Region: \"eu-west-2\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"r6i.16xlarge\":  {Region: \"eu-west-2\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.736},\n\t\t\"r6i.24xlarge\":  {Region: \"eu-west-2\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r6i.32xlarge\":  {Region: \"eu-west-2\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.472},\n\t\t\"r6i.metal\":     {Region: \"eu-west-2\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.472},\n\t\t\"t2.nano\":       {Region: \"eu-west-2\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0066},\n\t\t\"t2.micro\":      {Region: \"eu-west-2\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0132},\n\t\t\"t2.small\":      {Region: \"eu-west-2\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.026},\n\t\t\"t2.medium\":     {Region: \"eu-west-2\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.052},\n\t\t\"t2.large\":      {Region: \"eu-west-2\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1056},\n\t\t\"t2.xlarge\":     {Region: \"eu-west-2\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2112},\n\t\t\"t2.2xlarge\":    {Region: \"eu-west-2\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4224},\n\t\t\"t3.nano\":       {Region: \"eu-west-2\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0059},\n\t\t\"t3.micro\":      {Region: \"eu-west-2\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0118},\n\t\t\"t3.small\":      {Region: \"eu-west-2\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0236},\n\t\t\"t3.medium\":     {Region: \"eu-west-2\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0472},\n\t\t\"t3.large\":      {Region: \"eu-west-2\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0944},\n\t\t\"t3.xlarge\":     {Region: \"eu-west-2\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1888},\n\t\t\"t3.2xlarge\":    {Region: \"eu-west-2\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3776},\n\t\t\"t3a.nano\":      {Region: \"eu-west-2\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0053},\n\t\t\"t3a.micro\":     {Region: \"eu-west-2\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0106},\n\t\t\"t3a.small\":     {Region: \"eu-west-2\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0212},\n\t\t\"t3a.medium\":    {Region: \"eu-west-2\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0425},\n\t\t\"t3a.large\":     {Region: \"eu-west-2\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"t3a.xlarge\":    {Region: \"eu-west-2\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1699},\n\t\t\"t3a.2xlarge\":   {Region: \"eu-west-2\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3398},\n\t\t\"t4g.nano\":      {Region: \"eu-west-2\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0047},\n\t\t\"t4g.micro\":     {Region: \"eu-west-2\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0094},\n\t\t\"t4g.small\":     {Region: \"eu-west-2\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0188},\n\t\t\"t4g.medium\":    {Region: \"eu-west-2\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0376},\n\t\t\"t4g.large\":     {Region: \"eu-west-2\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0752},\n\t\t\"t4g.xlarge\":    {Region: \"eu-west-2\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1504},\n\t\t\"t4g.2xlarge\":   {Region: \"eu-west-2\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3008},\n\t\t\"x1.16xlarge\":   {Region: \"eu-west-2\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 8.403},\n\t\t\"x1.32xlarge\":   {Region: \"eu-west-2\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 16.806},\n\t\t\"z1d.large\":     {Region: \"eu-west-2\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.22},\n\t\t\"z1d.xlarge\":    {Region: \"eu-west-2\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.439},\n\t\t\"z1d.2xlarge\":   {Region: \"eu-west-2\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.879},\n\t\t\"z1d.3xlarge\":   {Region: \"eu-west-2\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.318},\n\t\t\"z1d.6xlarge\":   {Region: \"eu-west-2\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.636},\n\t\t\"z1d.12xlarge\":  {Region: \"eu-west-2\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.273},\n\t\t\"z1d.metal\":     {Region: \"eu-west-2\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.273},\n\t},\n\t\"eu-west-3\": {\n\t\t\"c5.large\":         {Region: \"eu-west-3\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"c5.xlarge\":        {Region: \"eu-west-3\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"c5.2xlarge\":       {Region: \"eu-west-3\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"c5.4xlarge\":       {Region: \"eu-west-3\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"c5.9xlarge\":       {Region: \"eu-west-3\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.818},\n\t\t\"c5.12xlarge\":      {Region: \"eu-west-3\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"c5.18xlarge\":      {Region: \"eu-west-3\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.636},\n\t\t\"c5.24xlarge\":      {Region: \"eu-west-3\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c5.metal\":         {Region: \"eu-west-3\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c5a.large\":        {Region: \"eu-west-3\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.091},\n\t\t\"c5a.xlarge\":       {Region: \"eu-west-3\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.182},\n\t\t\"c5a.2xlarge\":      {Region: \"eu-west-3\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.364},\n\t\t\"c5a.4xlarge\":      {Region: \"eu-west-3\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.728},\n\t\t\"c5a.8xlarge\":      {Region: \"eu-west-3\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.456},\n\t\t\"c5a.12xlarge\":     {Region: \"eu-west-3\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.184},\n\t\t\"c5a.16xlarge\":     {Region: \"eu-west-3\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.912},\n\t\t\"c5a.24xlarge\":     {Region: \"eu-west-3\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.368},\n\t\t\"c5d.large\":        {Region: \"eu-west-3\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.115},\n\t\t\"c5d.xlarge\":       {Region: \"eu-west-3\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.23},\n\t\t\"c5d.2xlarge\":      {Region: \"eu-west-3\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.46},\n\t\t\"c5d.4xlarge\":      {Region: \"eu-west-3\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.92},\n\t\t\"c5d.9xlarge\":      {Region: \"eu-west-3\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.07},\n\t\t\"c5d.18xlarge\":     {Region: \"eu-west-3\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.14},\n\t\t\"c5n.large\":        {Region: \"eu-west-3\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.128},\n\t\t\"c5n.xlarge\":       {Region: \"eu-west-3\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.256},\n\t\t\"c5n.2xlarge\":      {Region: \"eu-west-3\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.512},\n\t\t\"c5n.4xlarge\":      {Region: \"eu-west-3\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.024},\n\t\t\"c5n.9xlarge\":      {Region: \"eu-west-3\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c5n.18xlarge\":     {Region: \"eu-west-3\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5n.metal\":        {Region: \"eu-west-3\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c6g.medium\":       {Region: \"eu-west-3\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0405},\n\t\t\"c6g.large\":        {Region: \"eu-west-3\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.081},\n\t\t\"c6g.xlarge\":       {Region: \"eu-west-3\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.162},\n\t\t\"c6g.2xlarge\":      {Region: \"eu-west-3\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.324},\n\t\t\"c6g.4xlarge\":      {Region: \"eu-west-3\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.648},\n\t\t\"c6g.8xlarge\":      {Region: \"eu-west-3\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.296},\n\t\t\"c6g.12xlarge\":     {Region: \"eu-west-3\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.944},\n\t\t\"c6g.16xlarge\":     {Region: \"eu-west-3\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"c6g.metal\":        {Region: \"eu-west-3\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"c6i.large\":        {Region: \"eu-west-3\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"c6i.xlarge\":       {Region: \"eu-west-3\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"c6i.2xlarge\":      {Region: \"eu-west-3\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"c6i.4xlarge\":      {Region: \"eu-west-3\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"c6i.8xlarge\":      {Region: \"eu-west-3\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"c6i.12xlarge\":     {Region: \"eu-west-3\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"c6i.16xlarge\":     {Region: \"eu-west-3\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"c6i.24xlarge\":     {Region: \"eu-west-3\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"c6i.32xlarge\":     {Region: \"eu-west-3\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.464},\n\t\t\"c6i.metal\":        {Region: \"eu-west-3\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.464},\n\t\t\"d2.xlarge\":        {Region: \"eu-west-3\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.772},\n\t\t\"d2.2xlarge\":       {Region: \"eu-west-3\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.544},\n\t\t\"d2.4xlarge\":       {Region: \"eu-west-3\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.088},\n\t\t\"d2.8xlarge\":       {Region: \"eu-west-3\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.176},\n\t\t\"g4dn.xlarge\":      {Region: \"eu-west-3\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.615},\n\t\t\"g4dn.2xlarge\":     {Region: \"eu-west-3\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.879},\n\t\t\"g4dn.4xlarge\":     {Region: \"eu-west-3\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.408},\n\t\t\"g4dn.8xlarge\":     {Region: \"eu-west-3\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.544},\n\t\t\"g4dn.12xlarge\":    {Region: \"eu-west-3\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.574},\n\t\t\"g4dn.16xlarge\":    {Region: \"eu-west-3\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.088},\n\t\t\"g4dn.metal\":       {Region: \"eu-west-3\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 9.148},\n\t\t\"i3.large\":         {Region: \"eu-west-3\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.181},\n\t\t\"i3.xlarge\":        {Region: \"eu-west-3\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.362},\n\t\t\"i3.2xlarge\":       {Region: \"eu-west-3\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.724},\n\t\t\"i3.4xlarge\":       {Region: \"eu-west-3\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.448},\n\t\t\"i3.8xlarge\":       {Region: \"eu-west-3\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.896},\n\t\t\"i3.16xlarge\":      {Region: \"eu-west-3\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"i3.metal\":         {Region: \"eu-west-3\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"i3en.large\":       {Region: \"eu-west-3\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.263},\n\t\t\"i3en.xlarge\":      {Region: \"eu-west-3\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.526},\n\t\t\"i3en.2xlarge\":     {Region: \"eu-west-3\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.052},\n\t\t\"i3en.3xlarge\":     {Region: \"eu-west-3\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.578},\n\t\t\"i3en.6xlarge\":     {Region: \"eu-west-3\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.156},\n\t\t\"i3en.12xlarge\":    {Region: \"eu-west-3\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.312},\n\t\t\"i3en.24xlarge\":    {Region: \"eu-west-3\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.624},\n\t\t\"i3en.metal\":       {Region: \"eu-west-3\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.624},\n\t\t\"inf1.xlarge\":      {Region: \"eu-west-3\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.267},\n\t\t\"inf1.2xlarge\":     {Region: \"eu-west-3\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.423},\n\t\t\"inf1.6xlarge\":     {Region: \"eu-west-3\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.379},\n\t\t\"inf1.24xlarge\":    {Region: \"eu-west-3\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.517},\n\t\t\"m5.large\":         {Region: \"eu-west-3\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.112},\n\t\t\"m5.xlarge\":        {Region: \"eu-west-3\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.224},\n\t\t\"m5.2xlarge\":       {Region: \"eu-west-3\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.448},\n\t\t\"m5.4xlarge\":       {Region: \"eu-west-3\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.896},\n\t\t\"m5.8xlarge\":       {Region: \"eu-west-3\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.792},\n\t\t\"m5.12xlarge\":      {Region: \"eu-west-3\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"m5.16xlarge\":      {Region: \"eu-west-3\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.584},\n\t\t\"m5.24xlarge\":      {Region: \"eu-west-3\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m5.metal\":         {Region: \"eu-west-3\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m5a.large\":        {Region: \"eu-west-3\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"m5a.xlarge\":       {Region: \"eu-west-3\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"m5a.2xlarge\":      {Region: \"eu-west-3\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"m5a.4xlarge\":      {Region: \"eu-west-3\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"m5a.8xlarge\":      {Region: \"eu-west-3\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"m5a.12xlarge\":     {Region: \"eu-west-3\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"m5a.16xlarge\":     {Region: \"eu-west-3\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"m5a.24xlarge\":     {Region: \"eu-west-3\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"m5ad.large\":       {Region: \"eu-west-3\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.121},\n\t\t\"m5ad.xlarge\":      {Region: \"eu-west-3\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.242},\n\t\t\"m5ad.2xlarge\":     {Region: \"eu-west-3\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.484},\n\t\t\"m5ad.4xlarge\":     {Region: \"eu-west-3\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.968},\n\t\t\"m5ad.8xlarge\":     {Region: \"eu-west-3\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.936},\n\t\t\"m5ad.12xlarge\":    {Region: \"eu-west-3\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.904},\n\t\t\"m5ad.16xlarge\":    {Region: \"eu-west-3\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.872},\n\t\t\"m5ad.24xlarge\":    {Region: \"eu-west-3\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.808},\n\t\t\"m5d.large\":        {Region: \"eu-west-3\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.132},\n\t\t\"m5d.xlarge\":       {Region: \"eu-west-3\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.264},\n\t\t\"m5d.2xlarge\":      {Region: \"eu-west-3\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.528},\n\t\t\"m5d.4xlarge\":      {Region: \"eu-west-3\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.056},\n\t\t\"m5d.8xlarge\":      {Region: \"eu-west-3\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.112},\n\t\t\"m5d.12xlarge\":     {Region: \"eu-west-3\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.168},\n\t\t\"m5d.16xlarge\":     {Region: \"eu-west-3\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.224},\n\t\t\"m5d.24xlarge\":     {Region: \"eu-west-3\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.336},\n\t\t\"m5d.metal\":        {Region: \"eu-west-3\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.336},\n\t\t\"m6g.medium\":       {Region: \"eu-west-3\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.045},\n\t\t\"m6g.large\":        {Region: \"eu-west-3\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.09},\n\t\t\"m6g.xlarge\":       {Region: \"eu-west-3\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.18},\n\t\t\"m6g.2xlarge\":      {Region: \"eu-west-3\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.36},\n\t\t\"m6g.4xlarge\":      {Region: \"eu-west-3\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.72},\n\t\t\"m6g.8xlarge\":      {Region: \"eu-west-3\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.44},\n\t\t\"m6g.12xlarge\":     {Region: \"eu-west-3\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.16},\n\t\t\"m6g.16xlarge\":     {Region: \"eu-west-3\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m6g.metal\":        {Region: \"eu-west-3\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m6i.large\":        {Region: \"eu-west-3\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.112},\n\t\t\"m6i.xlarge\":       {Region: \"eu-west-3\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.224},\n\t\t\"m6i.2xlarge\":      {Region: \"eu-west-3\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.448},\n\t\t\"m6i.4xlarge\":      {Region: \"eu-west-3\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.896},\n\t\t\"m6i.8xlarge\":      {Region: \"eu-west-3\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.792},\n\t\t\"m6i.12xlarge\":     {Region: \"eu-west-3\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"m6i.16xlarge\":     {Region: \"eu-west-3\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.584},\n\t\t\"m6i.24xlarge\":     {Region: \"eu-west-3\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m6i.32xlarge\":     {Region: \"eu-west-3\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.168},\n\t\t\"m6i.metal\":        {Region: \"eu-west-3\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.168},\n\t\t\"r4.large\":         {Region: \"eu-west-3\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.156},\n\t\t\"r4.xlarge\":        {Region: \"eu-west-3\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.312},\n\t\t\"r4.2xlarge\":       {Region: \"eu-west-3\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.624},\n\t\t\"r4.4xlarge\":       {Region: \"eu-west-3\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.248},\n\t\t\"r4.8xlarge\":       {Region: \"eu-west-3\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.496},\n\t\t\"r4.16xlarge\":      {Region: \"eu-west-3\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"r5.large\":         {Region: \"eu-west-3\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.148},\n\t\t\"r5.xlarge\":        {Region: \"eu-west-3\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.296},\n\t\t\"r5.2xlarge\":       {Region: \"eu-west-3\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"r5.4xlarge\":       {Region: \"eu-west-3\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.184},\n\t\t\"r5.8xlarge\":       {Region: \"eu-west-3\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.368},\n\t\t\"r5.12xlarge\":      {Region: \"eu-west-3\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"r5.16xlarge\":      {Region: \"eu-west-3\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.736},\n\t\t\"r5.24xlarge\":      {Region: \"eu-west-3\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r5.metal\":         {Region: \"eu-west-3\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r5a.large\":        {Region: \"eu-west-3\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"r5a.xlarge\":       {Region: \"eu-west-3\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"r5a.2xlarge\":      {Region: \"eu-west-3\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"r5a.4xlarge\":      {Region: \"eu-west-3\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"r5a.8xlarge\":      {Region: \"eu-west-3\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"r5a.12xlarge\":     {Region: \"eu-west-3\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"r5a.16xlarge\":     {Region: \"eu-west-3\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"r5a.24xlarge\":     {Region: \"eu-west-3\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"r5ad.large\":       {Region: \"eu-west-3\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.153},\n\t\t\"r5ad.xlarge\":      {Region: \"eu-west-3\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.306},\n\t\t\"r5ad.2xlarge\":     {Region: \"eu-west-3\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.612},\n\t\t\"r5ad.4xlarge\":     {Region: \"eu-west-3\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.224},\n\t\t\"r5ad.8xlarge\":     {Region: \"eu-west-3\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"r5ad.12xlarge\":    {Region: \"eu-west-3\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"r5ad.16xlarge\":    {Region: \"eu-west-3\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"r5ad.24xlarge\":    {Region: \"eu-west-3\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"r5d.large\":        {Region: \"eu-west-3\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.169},\n\t\t\"r5d.xlarge\":       {Region: \"eu-west-3\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.338},\n\t\t\"r5d.2xlarge\":      {Region: \"eu-west-3\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.676},\n\t\t\"r5d.4xlarge\":      {Region: \"eu-west-3\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.352},\n\t\t\"r5d.8xlarge\":      {Region: \"eu-west-3\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.704},\n\t\t\"r5d.12xlarge\":     {Region: \"eu-west-3\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.056},\n\t\t\"r5d.16xlarge\":     {Region: \"eu-west-3\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.408},\n\t\t\"r5d.24xlarge\":     {Region: \"eu-west-3\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.112},\n\t\t\"r5d.metal\":        {Region: \"eu-west-3\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.112},\n\t\t\"r5dn.large\":       {Region: \"eu-west-3\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.196},\n\t\t\"r5dn.xlarge\":      {Region: \"eu-west-3\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.392},\n\t\t\"r5dn.2xlarge\":     {Region: \"eu-west-3\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.784},\n\t\t\"r5dn.4xlarge\":     {Region: \"eu-west-3\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.568},\n\t\t\"r5dn.8xlarge\":     {Region: \"eu-west-3\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.136},\n\t\t\"r5dn.12xlarge\":    {Region: \"eu-west-3\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.704},\n\t\t\"r5dn.16xlarge\":    {Region: \"eu-west-3\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.272},\n\t\t\"r5dn.24xlarge\":    {Region: \"eu-west-3\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.408},\n\t\t\"r5dn.metal\":       {Region: \"eu-west-3\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.408},\n\t\t\"r5n.large\":        {Region: \"eu-west-3\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.175},\n\t\t\"r5n.xlarge\":       {Region: \"eu-west-3\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.35},\n\t\t\"r5n.2xlarge\":      {Region: \"eu-west-3\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.7},\n\t\t\"r5n.4xlarge\":      {Region: \"eu-west-3\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.4},\n\t\t\"r5n.8xlarge\":      {Region: \"eu-west-3\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.8},\n\t\t\"r5n.12xlarge\":     {Region: \"eu-west-3\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.2},\n\t\t\"r5n.16xlarge\":     {Region: \"eu-west-3\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.6},\n\t\t\"r5n.24xlarge\":     {Region: \"eu-west-3\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"r5n.metal\":        {Region: \"eu-west-3\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.4},\n\t\t\"r6g.medium\":       {Region: \"eu-west-3\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.059},\n\t\t\"r6g.large\":        {Region: \"eu-west-3\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.118},\n\t\t\"r6g.xlarge\":       {Region: \"eu-west-3\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.236},\n\t\t\"r6g.2xlarge\":      {Region: \"eu-west-3\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.472},\n\t\t\"r6g.4xlarge\":      {Region: \"eu-west-3\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.944},\n\t\t\"r6g.8xlarge\":      {Region: \"eu-west-3\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.888},\n\t\t\"r6g.12xlarge\":     {Region: \"eu-west-3\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.832},\n\t\t\"r6g.16xlarge\":     {Region: \"eu-west-3\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.776},\n\t\t\"r6g.metal\":        {Region: \"eu-west-3\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.776},\n\t\t\"r6gd.medium\":      {Region: \"eu-west-3\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0676},\n\t\t\"r6gd.large\":       {Region: \"eu-west-3\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1352},\n\t\t\"r6gd.xlarge\":      {Region: \"eu-west-3\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2704},\n\t\t\"r6gd.2xlarge\":     {Region: \"eu-west-3\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5408},\n\t\t\"r6gd.4xlarge\":     {Region: \"eu-west-3\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.0816},\n\t\t\"r6gd.8xlarge\":     {Region: \"eu-west-3\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.1632},\n\t\t\"r6gd.12xlarge\":    {Region: \"eu-west-3\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.2448},\n\t\t\"r6gd.16xlarge\":    {Region: \"eu-west-3\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.3264},\n\t\t\"r6gd.metal\":       {Region: \"eu-west-3\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.3264},\n\t\t\"r6i.large\":        {Region: \"eu-west-3\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.148},\n\t\t\"r6i.xlarge\":       {Region: \"eu-west-3\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.296},\n\t\t\"r6i.2xlarge\":      {Region: \"eu-west-3\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"r6i.4xlarge\":      {Region: \"eu-west-3\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.184},\n\t\t\"r6i.8xlarge\":      {Region: \"eu-west-3\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.368},\n\t\t\"r6i.12xlarge\":     {Region: \"eu-west-3\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"r6i.16xlarge\":     {Region: \"eu-west-3\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.736},\n\t\t\"r6i.24xlarge\":     {Region: \"eu-west-3\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r6i.32xlarge\":     {Region: \"eu-west-3\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.472},\n\t\t\"r6i.metal\":        {Region: \"eu-west-3\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.472},\n\t\t\"t2.nano\":          {Region: \"eu-west-3\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0066},\n\t\t\"t2.micro\":         {Region: \"eu-west-3\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0132},\n\t\t\"t2.small\":         {Region: \"eu-west-3\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0264},\n\t\t\"t2.medium\":        {Region: \"eu-west-3\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0528},\n\t\t\"t2.large\":         {Region: \"eu-west-3\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1056},\n\t\t\"t2.xlarge\":        {Region: \"eu-west-3\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2112},\n\t\t\"t2.2xlarge\":       {Region: \"eu-west-3\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4224},\n\t\t\"t3.nano\":          {Region: \"eu-west-3\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0059},\n\t\t\"t3.micro\":         {Region: \"eu-west-3\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0118},\n\t\t\"t3.small\":         {Region: \"eu-west-3\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0236},\n\t\t\"t3.medium\":        {Region: \"eu-west-3\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0472},\n\t\t\"t3.large\":         {Region: \"eu-west-3\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0944},\n\t\t\"t3.xlarge\":        {Region: \"eu-west-3\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1888},\n\t\t\"t3.2xlarge\":       {Region: \"eu-west-3\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3776},\n\t\t\"t3a.nano\":         {Region: \"eu-west-3\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0053},\n\t\t\"t3a.micro\":        {Region: \"eu-west-3\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0106},\n\t\t\"t3a.small\":        {Region: \"eu-west-3\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0212},\n\t\t\"t3a.medium\":       {Region: \"eu-west-3\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0425},\n\t\t\"t3a.large\":        {Region: \"eu-west-3\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"t3a.xlarge\":       {Region: \"eu-west-3\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1699},\n\t\t\"t3a.2xlarge\":      {Region: \"eu-west-3\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3398},\n\t\t\"t4g.nano\":         {Region: \"eu-west-3\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0047},\n\t\t\"t4g.micro\":        {Region: \"eu-west-3\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0094},\n\t\t\"t4g.small\":        {Region: \"eu-west-3\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0188},\n\t\t\"t4g.medium\":       {Region: \"eu-west-3\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0376},\n\t\t\"t4g.large\":        {Region: \"eu-west-3\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0752},\n\t\t\"t4g.xlarge\":       {Region: \"eu-west-3\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1504},\n\t\t\"t4g.2xlarge\":      {Region: \"eu-west-3\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3008},\n\t\t\"u-6tb1.56xlarge\":  {Region: \"eu-west-3\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 54.50589},\n\t\t\"u-6tb1.112xlarge\": {Region: \"eu-west-3\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 64.133},\n\t\t\"x1.16xlarge\":      {Region: \"eu-west-3\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 8.403},\n\t\t\"x1.32xlarge\":      {Region: \"eu-west-3\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 16.806},\n\t},\n\t\"me-south-1\": {\n\t\t\"c5.large\":      {Region: \"me-south-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.106},\n\t\t\"c5.xlarge\":     {Region: \"me-south-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.211},\n\t\t\"c5.2xlarge\":    {Region: \"me-south-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.422},\n\t\t\"c5.4xlarge\":    {Region: \"me-south-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.845},\n\t\t\"c5.9xlarge\":    {Region: \"me-south-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.901},\n\t\t\"c5.12xlarge\":   {Region: \"me-south-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.534},\n\t\t\"c5.18xlarge\":   {Region: \"me-south-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.802},\n\t\t\"c5.24xlarge\":   {Region: \"me-south-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.069},\n\t\t\"c5.metal\":      {Region: \"me-south-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.069},\n\t\t\"c5a.large\":     {Region: \"me-south-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.095},\n\t\t\"c5a.xlarge\":    {Region: \"me-south-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.19},\n\t\t\"c5a.2xlarge\":   {Region: \"me-south-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.38},\n\t\t\"c5a.4xlarge\":   {Region: \"me-south-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.76},\n\t\t\"c5a.8xlarge\":   {Region: \"me-south-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.52},\n\t\t\"c5a.12xlarge\":  {Region: \"me-south-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.28},\n\t\t\"c5a.16xlarge\":  {Region: \"me-south-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.04},\n\t\t\"c5a.24xlarge\":  {Region: \"me-south-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.56},\n\t\t\"c5ad.large\":    {Region: \"me-south-1\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"c5ad.xlarge\":   {Region: \"me-south-1\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"c5ad.2xlarge\":  {Region: \"me-south-1\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"c5ad.4xlarge\":  {Region: \"me-south-1\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"c5ad.8xlarge\":  {Region: \"me-south-1\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"c5ad.12xlarge\": {Region: \"me-south-1\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"c5ad.16xlarge\": {Region: \"me-south-1\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"c5ad.24xlarge\": {Region: \"me-south-1\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.184},\n\t\t\"c5d.large\":     {Region: \"me-south-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"c5d.xlarge\":    {Region: \"me-south-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"c5d.2xlarge\":   {Region: \"me-south-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"c5d.4xlarge\":   {Region: \"me-south-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.959},\n\t\t\"c5d.9xlarge\":   {Region: \"me-south-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.158},\n\t\t\"c5d.12xlarge\":  {Region: \"me-south-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.878},\n\t\t\"c5d.18xlarge\":  {Region: \"me-south-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.316},\n\t\t\"c5d.24xlarge\":  {Region: \"me-south-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.755},\n\t\t\"c5d.metal\":     {Region: \"me-south-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.755},\n\t\t\"c5n.large\":     {Region: \"me-south-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.134},\n\t\t\"c5n.xlarge\":    {Region: \"me-south-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.268},\n\t\t\"c5n.2xlarge\":   {Region: \"me-south-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.536},\n\t\t\"c5n.4xlarge\":   {Region: \"me-south-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.072},\n\t\t\"c5n.9xlarge\":   {Region: \"me-south-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.412},\n\t\t\"c5n.18xlarge\":  {Region: \"me-south-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.824},\n\t\t\"c5n.metal\":     {Region: \"me-south-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.824},\n\t\t\"c6g.medium\":    {Region: \"me-south-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0425},\n\t\t\"c6g.large\":     {Region: \"me-south-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c6g.xlarge\":    {Region: \"me-south-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c6g.2xlarge\":   {Region: \"me-south-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c6g.4xlarge\":   {Region: \"me-south-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c6g.8xlarge\":   {Region: \"me-south-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.36},\n\t\t\"c6g.12xlarge\":  {Region: \"me-south-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c6g.16xlarge\":  {Region: \"me-south-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.72},\n\t\t\"c6g.metal\":     {Region: \"me-south-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.72},\n\t\t\"d2.xlarge\":     {Region: \"me-south-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.809},\n\t\t\"d2.2xlarge\":    {Region: \"me-south-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.617},\n\t\t\"d2.4xlarge\":    {Region: \"me-south-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.234},\n\t\t\"d2.8xlarge\":    {Region: \"me-south-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.468},\n\t\t\"g4dn.xlarge\":   {Region: \"me-south-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.645},\n\t\t\"g4dn.2xlarge\":  {Region: \"me-south-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.922},\n\t\t\"g4dn.4xlarge\":  {Region: \"me-south-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.477},\n\t\t\"g4dn.8xlarge\":  {Region: \"me-south-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.669},\n\t\t\"g4dn.12xlarge\": {Region: \"me-south-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.798},\n\t\t\"g4dn.16xlarge\": {Region: \"me-south-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.338},\n\t\t\"g4dn.metal\":    {Region: \"me-south-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 9.596},\n\t\t\"i3.large\":      {Region: \"me-south-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.189},\n\t\t\"i3.xlarge\":     {Region: \"me-south-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.378},\n\t\t\"i3.2xlarge\":    {Region: \"me-south-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.757},\n\t\t\"i3.4xlarge\":    {Region: \"me-south-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.514},\n\t\t\"i3.8xlarge\":    {Region: \"me-south-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.027},\n\t\t\"i3.16xlarge\":   {Region: \"me-south-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.054},\n\t\t\"i3en.large\":    {Region: \"me-south-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.274},\n\t\t\"i3en.xlarge\":   {Region: \"me-south-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.548},\n\t\t\"i3en.2xlarge\":  {Region: \"me-south-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.095},\n\t\t\"i3en.3xlarge\":  {Region: \"me-south-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.643},\n\t\t\"i3en.6xlarge\":  {Region: \"me-south-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.286},\n\t\t\"i3en.12xlarge\": {Region: \"me-south-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.571},\n\t\t\"i3en.24xlarge\": {Region: \"me-south-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.142},\n\t\t\"i3en.metal\":    {Region: \"me-south-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.142},\n\t\t\"inf1.xlarge\":   {Region: \"me-south-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.28},\n\t\t\"inf1.2xlarge\":  {Region: \"me-south-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.444},\n\t\t\"inf1.6xlarge\":  {Region: \"me-south-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.447},\n\t\t\"inf1.24xlarge\": {Region: \"me-south-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.786},\n\t\t\"m5.large\":      {Region: \"me-south-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.118},\n\t\t\"m5.xlarge\":     {Region: \"me-south-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.235},\n\t\t\"m5.2xlarge\":    {Region: \"me-south-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.471},\n\t\t\"m5.4xlarge\":    {Region: \"me-south-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.942},\n\t\t\"m5.8xlarge\":    {Region: \"me-south-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.883},\n\t\t\"m5.12xlarge\":   {Region: \"me-south-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.825},\n\t\t\"m5.16xlarge\":   {Region: \"me-south-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.766},\n\t\t\"m5.24xlarge\":   {Region: \"me-south-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.65},\n\t\t\"m5.metal\":      {Region: \"me-south-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.65},\n\t\t\"m5d.large\":     {Region: \"me-south-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.139},\n\t\t\"m5d.xlarge\":    {Region: \"me-south-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.277},\n\t\t\"m5d.2xlarge\":   {Region: \"me-south-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.554},\n\t\t\"m5d.4xlarge\":   {Region: \"me-south-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.109},\n\t\t\"m5d.8xlarge\":   {Region: \"me-south-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.218},\n\t\t\"m5d.12xlarge\":  {Region: \"me-south-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.326},\n\t\t\"m5d.16xlarge\":  {Region: \"me-south-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.435},\n\t\t\"m5d.24xlarge\":  {Region: \"me-south-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.653},\n\t\t\"m5d.metal\":     {Region: \"me-south-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.653},\n\t\t\"m6g.medium\":    {Region: \"me-south-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.047},\n\t\t\"m6g.large\":     {Region: \"me-south-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.094},\n\t\t\"m6g.xlarge\":    {Region: \"me-south-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.188},\n\t\t\"m6g.2xlarge\":   {Region: \"me-south-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.376},\n\t\t\"m6g.4xlarge\":   {Region: \"me-south-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.752},\n\t\t\"m6g.8xlarge\":   {Region: \"me-south-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.504},\n\t\t\"m6g.12xlarge\":  {Region: \"me-south-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.256},\n\t\t\"m6g.16xlarge\":  {Region: \"me-south-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.008},\n\t\t\"m6g.metal\":     {Region: \"me-south-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.008},\n\t\t\"r5.large\":      {Region: \"me-south-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.155},\n\t\t\"r5.xlarge\":     {Region: \"me-south-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.31},\n\t\t\"r5.2xlarge\":    {Region: \"me-south-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.62},\n\t\t\"r5.4xlarge\":    {Region: \"me-south-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.241},\n\t\t\"r5.8xlarge\":    {Region: \"me-south-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.482},\n\t\t\"r5.12xlarge\":   {Region: \"me-south-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.722},\n\t\t\"r5.16xlarge\":   {Region: \"me-south-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.963},\n\t\t\"r5.24xlarge\":   {Region: \"me-south-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.445},\n\t\t\"r5.metal\":      {Region: \"me-south-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.445},\n\t\t\"r5d.large\":     {Region: \"me-south-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.176},\n\t\t\"r5d.xlarge\":    {Region: \"me-south-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.352},\n\t\t\"r5d.2xlarge\":   {Region: \"me-south-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.704},\n\t\t\"r5d.4xlarge\":   {Region: \"me-south-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.408},\n\t\t\"r5d.8xlarge\":   {Region: \"me-south-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.816},\n\t\t\"r5d.12xlarge\":  {Region: \"me-south-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.224},\n\t\t\"r5d.16xlarge\":  {Region: \"me-south-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.632},\n\t\t\"r5d.24xlarge\":  {Region: \"me-south-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.448},\n\t\t\"r5d.metal\":     {Region: \"me-south-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.448},\n\t\t\"t3.nano\":       {Region: \"me-south-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0063},\n\t\t\"t3.micro\":      {Region: \"me-south-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0125},\n\t\t\"t3.small\":      {Region: \"me-south-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0251},\n\t\t\"t3.medium\":     {Region: \"me-south-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0502},\n\t\t\"t3.large\":      {Region: \"me-south-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1003},\n\t\t\"t3.xlarge\":     {Region: \"me-south-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2006},\n\t\t\"t3.2xlarge\":    {Region: \"me-south-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4013},\n\t},\n\t\"sa-east-1\": {\n\t\t\"c1.medium\":     {Region: \"sa-east-1\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.179},\n\t\t\"c1.xlarge\":     {Region: \"sa-east-1\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.718},\n\t\t\"c3.large\":      {Region: \"sa-east-1\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.163},\n\t\t\"c3.xlarge\":     {Region: \"sa-east-1\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.325},\n\t\t\"c3.2xlarge\":    {Region: \"sa-east-1\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.65},\n\t\t\"c3.4xlarge\":    {Region: \"sa-east-1\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.3},\n\t\t\"c3.8xlarge\":    {Region: \"sa-east-1\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.6},\n\t\t\"c4.large\":      {Region: \"sa-east-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.155},\n\t\t\"c4.xlarge\":     {Region: \"sa-east-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.309},\n\t\t\"c4.2xlarge\":    {Region: \"sa-east-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.618},\n\t\t\"c4.4xlarge\":    {Region: \"sa-east-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.235},\n\t\t\"c4.8xlarge\":    {Region: \"sa-east-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.47},\n\t\t\"c5.large\":      {Region: \"sa-east-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.131},\n\t\t\"c5.xlarge\":     {Region: \"sa-east-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.262},\n\t\t\"c5.2xlarge\":    {Region: \"sa-east-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"c5.4xlarge\":    {Region: \"sa-east-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.048},\n\t\t\"c5.9xlarge\":    {Region: \"sa-east-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.358},\n\t\t\"c5.12xlarge\":   {Region: \"sa-east-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.144},\n\t\t\"c5.18xlarge\":   {Region: \"sa-east-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.716},\n\t\t\"c5.24xlarge\":   {Region: \"sa-east-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"c5.metal\":      {Region: \"sa-east-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"c5a.large\":     {Region: \"sa-east-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.118},\n\t\t\"c5a.xlarge\":    {Region: \"sa-east-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.236},\n\t\t\"c5a.2xlarge\":   {Region: \"sa-east-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.472},\n\t\t\"c5a.4xlarge\":   {Region: \"sa-east-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.944},\n\t\t\"c5a.8xlarge\":   {Region: \"sa-east-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.888},\n\t\t\"c5a.12xlarge\":  {Region: \"sa-east-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.832},\n\t\t\"c5a.16xlarge\":  {Region: \"sa-east-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.776},\n\t\t\"c5a.24xlarge\":  {Region: \"sa-east-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.664},\n\t\t\"c5ad.large\":    {Region: \"sa-east-1\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.134},\n\t\t\"c5ad.xlarge\":   {Region: \"sa-east-1\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.268},\n\t\t\"c5ad.2xlarge\":  {Region: \"sa-east-1\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.536},\n\t\t\"c5ad.4xlarge\":  {Region: \"sa-east-1\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.072},\n\t\t\"c5ad.8xlarge\":  {Region: \"sa-east-1\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.144},\n\t\t\"c5ad.12xlarge\": {Region: \"sa-east-1\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.216},\n\t\t\"c5ad.16xlarge\": {Region: \"sa-east-1\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.288},\n\t\t\"c5ad.24xlarge\": {Region: \"sa-east-1\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.432},\n\t\t\"c5d.large\":     {Region: \"sa-east-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"c5d.xlarge\":    {Region: \"sa-east-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"c5d.2xlarge\":   {Region: \"sa-east-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"c5d.4xlarge\":   {Region: \"sa-east-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"c5d.9xlarge\":   {Region: \"sa-east-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.682},\n\t\t\"c5d.12xlarge\":  {Region: \"sa-east-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"c5d.18xlarge\":  {Region: \"sa-east-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.364},\n\t\t\"c5d.24xlarge\":  {Region: \"sa-east-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"c5d.metal\":     {Region: \"sa-east-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"c5n.large\":     {Region: \"sa-east-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.166},\n\t\t\"c5n.xlarge\":    {Region: \"sa-east-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.332},\n\t\t\"c5n.2xlarge\":   {Region: \"sa-east-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.664},\n\t\t\"c5n.4xlarge\":   {Region: \"sa-east-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.328},\n\t\t\"c5n.9xlarge\":   {Region: \"sa-east-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.988},\n\t\t\"c5n.18xlarge\":  {Region: \"sa-east-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.976},\n\t\t\"c5n.metal\":     {Region: \"sa-east-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 5.976},\n\t\t\"c6g.medium\":    {Region: \"sa-east-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0524},\n\t\t\"c6g.large\":     {Region: \"sa-east-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1048},\n\t\t\"c6g.xlarge\":    {Region: \"sa-east-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2096},\n\t\t\"c6g.2xlarge\":   {Region: \"sa-east-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4192},\n\t\t\"c6g.4xlarge\":   {Region: \"sa-east-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8384},\n\t\t\"c6g.8xlarge\":   {Region: \"sa-east-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6768},\n\t\t\"c6g.12xlarge\":  {Region: \"sa-east-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.5152},\n\t\t\"c6g.16xlarge\":  {Region: \"sa-east-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.3536},\n\t\t\"c6g.metal\":     {Region: \"sa-east-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.3536},\n\t\t\"c6gn.medium\":   {Region: \"sa-east-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0665},\n\t\t\"c6gn.large\":    {Region: \"sa-east-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"c6gn.xlarge\":   {Region: \"sa-east-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"c6gn.2xlarge\":  {Region: \"sa-east-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"c6gn.4xlarge\":  {Region: \"sa-east-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"c6gn.8xlarge\":  {Region: \"sa-east-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"c6gn.12xlarge\": {Region: \"sa-east-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"c6gn.16xlarge\": {Region: \"sa-east-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"c6i.large\":     {Region: \"sa-east-1\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.131},\n\t\t\"c6i.xlarge\":    {Region: \"sa-east-1\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.262},\n\t\t\"c6i.2xlarge\":   {Region: \"sa-east-1\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"c6i.4xlarge\":   {Region: \"sa-east-1\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.048},\n\t\t\"c6i.8xlarge\":   {Region: \"sa-east-1\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.096},\n\t\t\"c6i.12xlarge\":  {Region: \"sa-east-1\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.144},\n\t\t\"c6i.16xlarge\":  {Region: \"sa-east-1\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.192},\n\t\t\"c6i.24xlarge\":  {Region: \"sa-east-1\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"c6i.32xlarge\":  {Region: \"sa-east-1\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.384},\n\t\t\"c6i.metal\":     {Region: \"sa-east-1\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.384},\n\t\t\"g4dn.xlarge\":   {Region: \"sa-east-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.894},\n\t\t\"g4dn.2xlarge\":  {Region: \"sa-east-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.278},\n\t\t\"g4dn.4xlarge\":  {Region: \"sa-east-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 2.046},\n\t\t\"g4dn.8xlarge\":  {Region: \"sa-east-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 3.698},\n\t\t\"g4dn.12xlarge\": {Region: \"sa-east-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 6.649},\n\t\t\"g4dn.16xlarge\": {Region: \"sa-east-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 7.397},\n\t\t\"g4dn.metal\":    {Region: \"sa-east-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 13.298},\n\t\t\"i3.large\":      {Region: \"sa-east-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.249},\n\t\t\"i3.xlarge\":     {Region: \"sa-east-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.498},\n\t\t\"i3.2xlarge\":    {Region: \"sa-east-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.996},\n\t\t\"i3.4xlarge\":    {Region: \"sa-east-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.992},\n\t\t\"i3.8xlarge\":    {Region: \"sa-east-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.984},\n\t\t\"i3.16xlarge\":   {Region: \"sa-east-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 7.968},\n\t\t\"i3.metal\":      {Region: \"sa-east-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 7.968},\n\t\t\"i3en.large\":    {Region: \"sa-east-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.36},\n\t\t\"i3en.xlarge\":   {Region: \"sa-east-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.72},\n\t\t\"i3en.2xlarge\":  {Region: \"sa-east-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.44},\n\t\t\"i3en.3xlarge\":  {Region: \"sa-east-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 2.16},\n\t\t\"i3en.6xlarge\":  {Region: \"sa-east-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 4.32},\n\t\t\"i3en.12xlarge\": {Region: \"sa-east-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 8.64},\n\t\t\"i3en.24xlarge\": {Region: \"sa-east-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 17.28},\n\t\t\"i3en.metal\":    {Region: \"sa-east-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 17.28},\n\t\t\"inf1.xlarge\":   {Region: \"sa-east-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.377},\n\t\t\"inf1.2xlarge\":  {Region: \"sa-east-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.598},\n\t\t\"inf1.6xlarge\":  {Region: \"sa-east-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.95},\n\t\t\"inf1.24xlarge\": {Region: \"sa-east-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 7.8},\n\t\t\"m1.small\":      {Region: \"sa-east-1\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.058},\n\t\t\"m1.medium\":     {Region: \"sa-east-1\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.117},\n\t\t\"m1.large\":      {Region: \"sa-east-1\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.233},\n\t\t\"m1.xlarge\":     {Region: \"sa-east-1\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.467},\n\t\t\"m2.xlarge\":     {Region: \"sa-east-1\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.323},\n\t\t\"m2.2xlarge\":    {Region: \"sa-east-1\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.645},\n\t\t\"m2.4xlarge\":    {Region: \"sa-east-1\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.291},\n\t\t\"m3.medium\":     {Region: \"sa-east-1\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.095},\n\t\t\"m3.large\":      {Region: \"sa-east-1\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.19},\n\t\t\"m3.xlarge\":     {Region: \"sa-east-1\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.381},\n\t\t\"m3.2xlarge\":    {Region: \"sa-east-1\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.761},\n\t\t\"m4.large\":      {Region: \"sa-east-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.159},\n\t\t\"m4.xlarge\":     {Region: \"sa-east-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.318},\n\t\t\"m4.2xlarge\":    {Region: \"sa-east-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.636},\n\t\t\"m4.4xlarge\":    {Region: \"sa-east-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.272},\n\t\t\"m4.10xlarge\":   {Region: \"sa-east-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 3.18},\n\t\t\"m4.16xlarge\":   {Region: \"sa-east-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"m5.large\":      {Region: \"sa-east-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.153},\n\t\t\"m5.xlarge\":     {Region: \"sa-east-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.306},\n\t\t\"m5.2xlarge\":    {Region: \"sa-east-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.612},\n\t\t\"m5.4xlarge\":    {Region: \"sa-east-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.224},\n\t\t\"m5.8xlarge\":    {Region: \"sa-east-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"m5.12xlarge\":   {Region: \"sa-east-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"m5.16xlarge\":   {Region: \"sa-east-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"m5.24xlarge\":   {Region: \"sa-east-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"m5.metal\":      {Region: \"sa-east-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"m5a.large\":     {Region: \"sa-east-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.138},\n\t\t\"m5a.xlarge\":    {Region: \"sa-east-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.276},\n\t\t\"m5a.2xlarge\":   {Region: \"sa-east-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.552},\n\t\t\"m5a.4xlarge\":   {Region: \"sa-east-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.104},\n\t\t\"m5a.8xlarge\":   {Region: \"sa-east-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.208},\n\t\t\"m5a.12xlarge\":  {Region: \"sa-east-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.312},\n\t\t\"m5a.16xlarge\":  {Region: \"sa-east-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.416},\n\t\t\"m5a.24xlarge\":  {Region: \"sa-east-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.624},\n\t\t\"m5ad.large\":    {Region: \"sa-east-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.165},\n\t\t\"m5ad.xlarge\":   {Region: \"sa-east-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.33},\n\t\t\"m5ad.2xlarge\":  {Region: \"sa-east-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.66},\n\t\t\"m5ad.4xlarge\":  {Region: \"sa-east-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.32},\n\t\t\"m5ad.8xlarge\":  {Region: \"sa-east-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.64},\n\t\t\"m5ad.12xlarge\": {Region: \"sa-east-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.96},\n\t\t\"m5ad.16xlarge\": {Region: \"sa-east-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.28},\n\t\t\"m5ad.24xlarge\": {Region: \"sa-east-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.92},\n\t\t\"m5d.large\":     {Region: \"sa-east-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.18},\n\t\t\"m5d.xlarge\":    {Region: \"sa-east-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.36},\n\t\t\"m5d.2xlarge\":   {Region: \"sa-east-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.72},\n\t\t\"m5d.4xlarge\":   {Region: \"sa-east-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.44},\n\t\t\"m5d.8xlarge\":   {Region: \"sa-east-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"m5d.12xlarge\":  {Region: \"sa-east-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.32},\n\t\t\"m5d.16xlarge\":  {Region: \"sa-east-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"m5d.24xlarge\":  {Region: \"sa-east-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.64},\n\t\t\"m5d.metal\":     {Region: \"sa-east-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.64},\n\t\t\"m5zn.large\":    {Region: \"sa-east-1\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2632},\n\t\t\"m5zn.xlarge\":   {Region: \"sa-east-1\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.5265},\n\t\t\"m5zn.2xlarge\":  {Region: \"sa-east-1\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.053},\n\t\t\"m5zn.3xlarge\":  {Region: \"sa-east-1\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.5794},\n\t\t\"m5zn.6xlarge\":  {Region: \"sa-east-1\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.1589},\n\t\t\"m5zn.12xlarge\": {Region: \"sa-east-1\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.3178},\n\t\t\"m5zn.metal\":    {Region: \"sa-east-1\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.3178},\n\t\t\"m6g.medium\":    {Region: \"sa-east-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0612},\n\t\t\"m6g.large\":     {Region: \"sa-east-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1224},\n\t\t\"m6g.xlarge\":    {Region: \"sa-east-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2448},\n\t\t\"m6g.2xlarge\":   {Region: \"sa-east-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4896},\n\t\t\"m6g.4xlarge\":   {Region: \"sa-east-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9792},\n\t\t\"m6g.8xlarge\":   {Region: \"sa-east-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.9584},\n\t\t\"m6g.12xlarge\":  {Region: \"sa-east-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.9376},\n\t\t\"m6g.16xlarge\":  {Region: \"sa-east-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.9168},\n\t\t\"m6g.metal\":     {Region: \"sa-east-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.9168},\n\t\t\"m6i.large\":     {Region: \"sa-east-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.153},\n\t\t\"m6i.xlarge\":    {Region: \"sa-east-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.306},\n\t\t\"m6i.2xlarge\":   {Region: \"sa-east-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.612},\n\t\t\"m6i.4xlarge\":   {Region: \"sa-east-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.224},\n\t\t\"m6i.8xlarge\":   {Region: \"sa-east-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"m6i.12xlarge\":  {Region: \"sa-east-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"m6i.16xlarge\":  {Region: \"sa-east-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"m6i.24xlarge\":  {Region: \"sa-east-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"m6i.32xlarge\":  {Region: \"sa-east-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.792},\n\t\t\"m6i.metal\":     {Region: \"sa-east-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 9.792},\n\t\t\"r3.large\":      {Region: \"sa-east-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.35},\n\t\t\"r3.xlarge\":     {Region: \"sa-east-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.7},\n\t\t\"r3.2xlarge\":    {Region: \"sa-east-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.399},\n\t\t\"r3.4xlarge\":    {Region: \"sa-east-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.799},\n\t\t\"r3.8xlarge\":    {Region: \"sa-east-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 5.597},\n\t\t\"r4.large\":      {Region: \"sa-east-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.28},\n\t\t\"r4.xlarge\":     {Region: \"sa-east-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.56},\n\t\t\"r4.2xlarge\":    {Region: \"sa-east-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.12},\n\t\t\"r4.4xlarge\":    {Region: \"sa-east-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.24},\n\t\t\"r4.8xlarge\":    {Region: \"sa-east-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.48},\n\t\t\"r4.16xlarge\":   {Region: \"sa-east-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 8.96},\n\t\t\"r5.large\":      {Region: \"sa-east-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.201},\n\t\t\"r5.xlarge\":     {Region: \"sa-east-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.402},\n\t\t\"r5.2xlarge\":    {Region: \"sa-east-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.804},\n\t\t\"r5.4xlarge\":    {Region: \"sa-east-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.608},\n\t\t\"r5.8xlarge\":    {Region: \"sa-east-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.216},\n\t\t\"r5.12xlarge\":   {Region: \"sa-east-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.824},\n\t\t\"r5.16xlarge\":   {Region: \"sa-east-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.432},\n\t\t\"r5.24xlarge\":   {Region: \"sa-east-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.648},\n\t\t\"r5.metal\":      {Region: \"sa-east-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.648},\n\t\t\"r5a.large\":     {Region: \"sa-east-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.181},\n\t\t\"r5a.xlarge\":    {Region: \"sa-east-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.362},\n\t\t\"r5a.2xlarge\":   {Region: \"sa-east-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.724},\n\t\t\"r5a.4xlarge\":   {Region: \"sa-east-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.448},\n\t\t\"r5a.8xlarge\":   {Region: \"sa-east-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.896},\n\t\t\"r5a.12xlarge\":  {Region: \"sa-east-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.344},\n\t\t\"r5a.16xlarge\":  {Region: \"sa-east-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.792},\n\t\t\"r5a.24xlarge\":  {Region: \"sa-east-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.688},\n\t\t\"r5ad.large\":    {Region: \"sa-east-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.208},\n\t\t\"r5ad.xlarge\":   {Region: \"sa-east-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.416},\n\t\t\"r5ad.2xlarge\":  {Region: \"sa-east-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.832},\n\t\t\"r5ad.4xlarge\":  {Region: \"sa-east-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.664},\n\t\t\"r5ad.8xlarge\":  {Region: \"sa-east-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.328},\n\t\t\"r5ad.12xlarge\": {Region: \"sa-east-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"r5ad.16xlarge\": {Region: \"sa-east-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.656},\n\t\t\"r5ad.24xlarge\": {Region: \"sa-east-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.984},\n\t\t\"r5d.large\":     {Region: \"sa-east-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.228},\n\t\t\"r5d.xlarge\":    {Region: \"sa-east-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.456},\n\t\t\"r5d.2xlarge\":   {Region: \"sa-east-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.912},\n\t\t\"r5d.4xlarge\":   {Region: \"sa-east-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.824},\n\t\t\"r5d.8xlarge\":   {Region: \"sa-east-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.648},\n\t\t\"r5d.12xlarge\":  {Region: \"sa-east-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.472},\n\t\t\"r5d.16xlarge\":  {Region: \"sa-east-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 7.296},\n\t\t\"r5d.24xlarge\":  {Region: \"sa-east-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.944},\n\t\t\"r5d.metal\":     {Region: \"sa-east-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.944},\n\t\t\"r5n.large\":     {Region: \"sa-east-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.236},\n\t\t\"r5n.xlarge\":    {Region: \"sa-east-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.472},\n\t\t\"r5n.2xlarge\":   {Region: \"sa-east-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.944},\n\t\t\"r5n.4xlarge\":   {Region: \"sa-east-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.888},\n\t\t\"r5n.8xlarge\":   {Region: \"sa-east-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.776},\n\t\t\"r5n.12xlarge\":  {Region: \"sa-east-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.664},\n\t\t\"r5n.16xlarge\":  {Region: \"sa-east-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 7.552},\n\t\t\"r5n.24xlarge\":  {Region: \"sa-east-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 11.328},\n\t\t\"r5n.metal\":     {Region: \"sa-east-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 11.328},\n\t\t\"r6g.medium\":    {Region: \"sa-east-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0804},\n\t\t\"r6g.large\":     {Region: \"sa-east-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1608},\n\t\t\"r6g.xlarge\":    {Region: \"sa-east-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3216},\n\t\t\"r6g.2xlarge\":   {Region: \"sa-east-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.6432},\n\t\t\"r6g.4xlarge\":   {Region: \"sa-east-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.2864},\n\t\t\"r6g.8xlarge\":   {Region: \"sa-east-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.5728},\n\t\t\"r6g.12xlarge\":  {Region: \"sa-east-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.8592},\n\t\t\"r6g.16xlarge\":  {Region: \"sa-east-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.1456},\n\t\t\"r6g.metal\":     {Region: \"sa-east-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.1456},\n\t\t\"r6i.large\":     {Region: \"sa-east-1\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.201},\n\t\t\"r6i.xlarge\":    {Region: \"sa-east-1\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.402},\n\t\t\"r6i.2xlarge\":   {Region: \"sa-east-1\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.804},\n\t\t\"r6i.4xlarge\":   {Region: \"sa-east-1\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.608},\n\t\t\"r6i.8xlarge\":   {Region: \"sa-east-1\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.216},\n\t\t\"r6i.12xlarge\":  {Region: \"sa-east-1\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.824},\n\t\t\"r6i.16xlarge\":  {Region: \"sa-east-1\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.432},\n\t\t\"r6i.24xlarge\":  {Region: \"sa-east-1\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.648},\n\t\t\"r6i.32xlarge\":  {Region: \"sa-east-1\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 12.864},\n\t\t\"r6i.metal\":     {Region: \"sa-east-1\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 12.864},\n\t\t\"t1.micro\":      {Region: \"sa-east-1\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.027},\n\t\t\"t2.nano\":       {Region: \"sa-east-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0093},\n\t\t\"t2.micro\":      {Region: \"sa-east-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0186},\n\t\t\"t2.small\":      {Region: \"sa-east-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0372},\n\t\t\"t2.medium\":     {Region: \"sa-east-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0744},\n\t\t\"t2.large\":      {Region: \"sa-east-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1488},\n\t\t\"t2.xlarge\":     {Region: \"sa-east-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2976},\n\t\t\"t2.2xlarge\":    {Region: \"sa-east-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5952},\n\t\t\"t3.nano\":       {Region: \"sa-east-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0084},\n\t\t\"t3.micro\":      {Region: \"sa-east-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0168},\n\t\t\"t3.small\":      {Region: \"sa-east-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0336},\n\t\t\"t3.medium\":     {Region: \"sa-east-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0672},\n\t\t\"t3.large\":      {Region: \"sa-east-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1344},\n\t\t\"t3.xlarge\":     {Region: \"sa-east-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2688},\n\t\t\"t3.2xlarge\":    {Region: \"sa-east-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5376},\n\t\t\"t3a.nano\":      {Region: \"sa-east-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0076},\n\t\t\"t3a.micro\":     {Region: \"sa-east-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0151},\n\t\t\"t3a.small\":     {Region: \"sa-east-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0302},\n\t\t\"t3a.medium\":    {Region: \"sa-east-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0605},\n\t\t\"t3a.large\":     {Region: \"sa-east-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.121},\n\t\t\"t3a.xlarge\":    {Region: \"sa-east-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2419},\n\t\t\"t3a.2xlarge\":   {Region: \"sa-east-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4838},\n\t\t\"t4g.nano\":      {Region: \"sa-east-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0067},\n\t\t\"t4g.micro\":     {Region: \"sa-east-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0134},\n\t\t\"t4g.small\":     {Region: \"sa-east-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0268},\n\t\t\"t4g.medium\":    {Region: \"sa-east-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0536},\n\t\t\"t4g.large\":     {Region: \"sa-east-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1072},\n\t\t\"t4g.xlarge\":    {Region: \"sa-east-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2144},\n\t\t\"t4g.2xlarge\":   {Region: \"sa-east-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4288},\n\t\t\"x1.16xlarge\":   {Region: \"sa-east-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.005},\n\t\t\"x1.32xlarge\":   {Region: \"sa-east-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 26.01},\n\t\t\"x1e.xlarge\":    {Region: \"sa-east-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.626},\n\t\t\"x1e.2xlarge\":   {Region: \"sa-east-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 3.251},\n\t\t\"x1e.4xlarge\":   {Region: \"sa-east-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 6.502},\n\t\t\"x1e.8xlarge\":   {Region: \"sa-east-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 13.005},\n\t\t\"x1e.16xlarge\":  {Region: \"sa-east-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 26.01},\n\t\t\"x1e.32xlarge\":  {Region: \"sa-east-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 52.019},\n\t},\n\t\"us-east-1\": {\n\t\t\"a1.medium\":         {Region: \"us-east-1\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0255},\n\t\t\"a1.large\":          {Region: \"us-east-1\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.051},\n\t\t\"a1.xlarge\":         {Region: \"us-east-1\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.102},\n\t\t\"a1.2xlarge\":        {Region: \"us-east-1\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.204},\n\t\t\"a1.4xlarge\":        {Region: \"us-east-1\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"a1.metal\":          {Region: \"us-east-1\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"c1.medium\":         {Region: \"us-east-1\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"c1.xlarge\":         {Region: \"us-east-1\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"c3.large\":          {Region: \"us-east-1\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.105},\n\t\t\"c3.xlarge\":         {Region: \"us-east-1\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.21},\n\t\t\"c3.2xlarge\":        {Region: \"us-east-1\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.42},\n\t\t\"c3.4xlarge\":        {Region: \"us-east-1\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.84},\n\t\t\"c3.8xlarge\":        {Region: \"us-east-1\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.68},\n\t\t\"c4.large\":          {Region: \"us-east-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"c4.xlarge\":         {Region: \"us-east-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.199},\n\t\t\"c4.2xlarge\":        {Region: \"us-east-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.398},\n\t\t\"c4.4xlarge\":        {Region: \"us-east-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.796},\n\t\t\"c4.8xlarge\":        {Region: \"us-east-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.591},\n\t\t\"c5.large\":          {Region: \"us-east-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c5.xlarge\":         {Region: \"us-east-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c5.2xlarge\":        {Region: \"us-east-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c5.4xlarge\":        {Region: \"us-east-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c5.9xlarge\":        {Region: \"us-east-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.53},\n\t\t\"c5.12xlarge\":       {Region: \"us-east-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c5.18xlarge\":       {Region: \"us-east-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.06},\n\t\t\"c5.24xlarge\":       {Region: \"us-east-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c5.metal\":          {Region: \"us-east-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c5a.large\":         {Region: \"us-east-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.077},\n\t\t\"c5a.xlarge\":        {Region: \"us-east-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"c5a.2xlarge\":       {Region: \"us-east-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"c5a.4xlarge\":       {Region: \"us-east-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"c5a.8xlarge\":       {Region: \"us-east-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.232},\n\t\t\"c5a.12xlarge\":      {Region: \"us-east-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.848},\n\t\t\"c5a.16xlarge\":      {Region: \"us-east-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"c5a.24xlarge\":      {Region: \"us-east-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.696},\n\t\t\"c5ad.large\":        {Region: \"us-east-1\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"c5ad.xlarge\":       {Region: \"us-east-1\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"c5ad.2xlarge\":      {Region: \"us-east-1\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"c5ad.4xlarge\":      {Region: \"us-east-1\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"c5ad.8xlarge\":      {Region: \"us-east-1\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"c5ad.12xlarge\":     {Region: \"us-east-1\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"c5ad.16xlarge\":     {Region: \"us-east-1\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"c5ad.24xlarge\":     {Region: \"us-east-1\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"c5d.large\":         {Region: \"us-east-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"c5d.xlarge\":        {Region: \"us-east-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"c5d.2xlarge\":       {Region: \"us-east-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"c5d.4xlarge\":       {Region: \"us-east-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"c5d.9xlarge\":       {Region: \"us-east-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"c5d.12xlarge\":      {Region: \"us-east-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c5d.18xlarge\":      {Region: \"us-east-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"c5d.24xlarge\":      {Region: \"us-east-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5d.metal\":         {Region: \"us-east-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5n.large\":         {Region: \"us-east-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"c5n.xlarge\":        {Region: \"us-east-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"c5n.2xlarge\":       {Region: \"us-east-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"c5n.4xlarge\":       {Region: \"us-east-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"c5n.9xlarge\":       {Region: \"us-east-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.944},\n\t\t\"c5n.18xlarge\":      {Region: \"us-east-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c5n.metal\":         {Region: \"us-east-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c6a.large\":         {Region: \"us-east-1\", Type: \"c6a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0765},\n\t\t\"c6a.xlarge\":        {Region: \"us-east-1\", Type: \"c6a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.153},\n\t\t\"c6a.2xlarge\":       {Region: \"us-east-1\", Type: \"c6a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.306},\n\t\t\"c6a.4xlarge\":       {Region: \"us-east-1\", Type: \"c6a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.612},\n\t\t\"c6a.8xlarge\":       {Region: \"us-east-1\", Type: \"c6a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.224},\n\t\t\"c6a.12xlarge\":      {Region: \"us-east-1\", Type: \"c6a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.836},\n\t\t\"c6a.16xlarge\":      {Region: \"us-east-1\", Type: \"c6a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"c6a.24xlarge\":      {Region: \"us-east-1\", Type: \"c6a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"c6a.32xlarge\":      {Region: \"us-east-1\", Type: \"c6a.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"c6a.48xlarge\":      {Region: \"us-east-1\", Type: \"c6a.48xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"c6a.metal\":         {Region: \"us-east-1\", Type: \"c6a.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"c6g.medium\":        {Region: \"us-east-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.034},\n\t\t\"c6g.large\":         {Region: \"us-east-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.068},\n\t\t\"c6g.xlarge\":        {Region: \"us-east-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"c6g.2xlarge\":       {Region: \"us-east-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"c6g.4xlarge\":       {Region: \"us-east-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"c6g.8xlarge\":       {Region: \"us-east-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"c6g.12xlarge\":      {Region: \"us-east-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.632},\n\t\t\"c6g.16xlarge\":      {Region: \"us-east-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"c6g.metal\":         {Region: \"us-east-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"c6gd.medium\":       {Region: \"us-east-1\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0384},\n\t\t\"c6gd.large\":        {Region: \"us-east-1\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0768},\n\t\t\"c6gd.xlarge\":       {Region: \"us-east-1\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1536},\n\t\t\"c6gd.2xlarge\":      {Region: \"us-east-1\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3072},\n\t\t\"c6gd.4xlarge\":      {Region: \"us-east-1\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6144},\n\t\t\"c6gd.8xlarge\":      {Region: \"us-east-1\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.2288},\n\t\t\"c6gd.12xlarge\":     {Region: \"us-east-1\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.8432},\n\t\t\"c6gd.16xlarge\":     {Region: \"us-east-1\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.4576},\n\t\t\"c6gd.metal\":        {Region: \"us-east-1\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.4576},\n\t\t\"c6gn.medium\":       {Region: \"us-east-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0432},\n\t\t\"c6gn.large\":        {Region: \"us-east-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0864},\n\t\t\"c6gn.xlarge\":       {Region: \"us-east-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1728},\n\t\t\"c6gn.2xlarge\":      {Region: \"us-east-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3456},\n\t\t\"c6gn.4xlarge\":      {Region: \"us-east-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6912},\n\t\t\"c6gn.8xlarge\":      {Region: \"us-east-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3824},\n\t\t\"c6gn.12xlarge\":     {Region: \"us-east-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0736},\n\t\t\"c6gn.16xlarge\":     {Region: \"us-east-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7648},\n\t\t\"c6i.large\":         {Region: \"us-east-1\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c6i.xlarge\":        {Region: \"us-east-1\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c6i.2xlarge\":       {Region: \"us-east-1\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c6i.4xlarge\":       {Region: \"us-east-1\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c6i.8xlarge\":       {Region: \"us-east-1\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.36},\n\t\t\"c6i.12xlarge\":      {Region: \"us-east-1\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c6i.16xlarge\":      {Region: \"us-east-1\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.72},\n\t\t\"c6i.24xlarge\":      {Region: \"us-east-1\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c6i.32xlarge\":      {Region: \"us-east-1\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.44},\n\t\t\"c6i.metal\":         {Region: \"us-east-1\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.44},\n\t\t\"cc2.8xlarge\":       {Region: \"us-east-1\", Type: \"cc2.8xlarge\", Memory: kresource.MustParse(\"61952Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"cr1.8xlarge\":       {Region: \"us-east-1\", Type: \"cr1.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.5},\n\t\t\"d2.xlarge\":         {Region: \"us-east-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.69},\n\t\t\"d2.2xlarge\":        {Region: \"us-east-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.38},\n\t\t\"d2.4xlarge\":        {Region: \"us-east-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.76},\n\t\t\"d2.8xlarge\":        {Region: \"us-east-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"d3.xlarge\":         {Region: \"us-east-1\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.499},\n\t\t\"d3.2xlarge\":        {Region: \"us-east-1\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.999},\n\t\t\"d3.4xlarge\":        {Region: \"us-east-1\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.998},\n\t\t\"d3.8xlarge\":        {Region: \"us-east-1\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.99552},\n\t\t\"d3en.xlarge\":       {Region: \"us-east-1\", Type: \"d3en.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.526},\n\t\t\"d3en.2xlarge\":      {Region: \"us-east-1\", Type: \"d3en.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.051},\n\t\t\"d3en.4xlarge\":      {Region: \"us-east-1\", Type: \"d3en.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.103},\n\t\t\"d3en.6xlarge\":      {Region: \"us-east-1\", Type: \"d3en.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.154},\n\t\t\"d3en.8xlarge\":      {Region: \"us-east-1\", Type: \"d3en.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.20576},\n\t\t\"d3en.12xlarge\":     {Region: \"us-east-1\", Type: \"d3en.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.30864},\n\t\t\"dl1.24xlarge\":      {Region: \"us-east-1\", Type: \"dl1.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.10904},\n\t\t\"f1.2xlarge\":        {Region: \"us-east-1\", Type: \"f1.2xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.65},\n\t\t\"f1.4xlarge\":        {Region: \"us-east-1\", Type: \"f1.4xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.3},\n\t\t\"f1.16xlarge\":       {Region: \"us-east-1\", Type: \"f1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.2},\n\t\t\"g2.2xlarge\":        {Region: \"us-east-1\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.65},\n\t\t\"g2.8xlarge\":        {Region: \"us-east-1\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 2.6},\n\t\t\"g3.4xlarge\":        {Region: \"us-east-1\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.14},\n\t\t\"g3.8xlarge\":        {Region: \"us-east-1\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.28},\n\t\t\"g3.16xlarge\":       {Region: \"us-east-1\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 4.56},\n\t\t\"g3s.xlarge\":        {Region: \"us-east-1\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.75},\n\t\t\"g4ad.xlarge\":       {Region: \"us-east-1\", Type: \"g4ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.37853},\n\t\t\"g4ad.2xlarge\":      {Region: \"us-east-1\", Type: \"g4ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.54117},\n\t\t\"g4ad.4xlarge\":      {Region: \"us-east-1\", Type: \"g4ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 0.867},\n\t\t\"g4ad.8xlarge\":      {Region: \"us-east-1\", Type: \"g4ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 1.734},\n\t\t\"g4ad.16xlarge\":     {Region: \"us-east-1\", Type: \"g4ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 3.468},\n\t\t\"g4dn.xlarge\":       {Region: \"us-east-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.526},\n\t\t\"g4dn.2xlarge\":      {Region: \"us-east-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.752},\n\t\t\"g4dn.4xlarge\":      {Region: \"us-east-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.204},\n\t\t\"g4dn.8xlarge\":      {Region: \"us-east-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.176},\n\t\t\"g4dn.12xlarge\":     {Region: \"us-east-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 3.912},\n\t\t\"g4dn.16xlarge\":     {Region: \"us-east-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.352},\n\t\t\"g4dn.metal\":        {Region: \"us-east-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 7.824},\n\t\t\"g5.xlarge\":         {Region: \"us-east-1\", Type: \"g5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.006},\n\t\t\"g5.2xlarge\":        {Region: \"us-east-1\", Type: \"g5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.212},\n\t\t\"g5.4xlarge\":        {Region: \"us-east-1\", Type: \"g5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.624},\n\t\t\"g5.8xlarge\":        {Region: \"us-east-1\", Type: \"g5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.448},\n\t\t\"g5.12xlarge\":       {Region: \"us-east-1\", Type: \"g5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 5.672},\n\t\t\"g5.16xlarge\":       {Region: \"us-east-1\", Type: \"g5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.096},\n\t\t\"g5.24xlarge\":       {Region: \"us-east-1\", Type: \"g5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 4, Inf: 0, Price: 8.144},\n\t\t\"g5.48xlarge\":       {Region: \"us-east-1\", Type: \"g5.48xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 8, Inf: 0, Price: 16.288},\n\t\t\"g5g.xlarge\":        {Region: \"us-east-1\", Type: \"g5g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.42},\n\t\t\"g5g.2xlarge\":       {Region: \"us-east-1\", Type: \"g5g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.556},\n\t\t\"g5g.4xlarge\":       {Region: \"us-east-1\", Type: \"g5g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 0.828},\n\t\t\"g5g.8xlarge\":       {Region: \"us-east-1\", Type: \"g5g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 1.372},\n\t\t\"g5g.16xlarge\":      {Region: \"us-east-1\", Type: \"g5g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 2.744},\n\t\t\"g5g.metal\":         {Region: \"us-east-1\", Type: \"g5g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 2.744},\n\t\t\"h1.2xlarge\":        {Region: \"us-east-1\", Type: \"h1.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.468},\n\t\t\"h1.4xlarge\":        {Region: \"us-east-1\", Type: \"h1.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.936},\n\t\t\"h1.8xlarge\":        {Region: \"us-east-1\", Type: \"h1.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.872},\n\t\t\"h1.16xlarge\":       {Region: \"us-east-1\", Type: \"h1.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.744},\n\t\t\"hs1.8xlarge\":       {Region: \"us-east-1\", Type: \"hs1.8xlarge\", Memory: kresource.MustParse(\"119808Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.6},\n\t\t\"i2.xlarge\":         {Region: \"us-east-1\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.853},\n\t\t\"i2.2xlarge\":        {Region: \"us-east-1\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.705},\n\t\t\"i2.4xlarge\":        {Region: \"us-east-1\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.41},\n\t\t\"i2.8xlarge\":        {Region: \"us-east-1\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.82},\n\t\t\"i3.large\":          {Region: \"us-east-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.156},\n\t\t\"i3.xlarge\":         {Region: \"us-east-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.312},\n\t\t\"i3.2xlarge\":        {Region: \"us-east-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.624},\n\t\t\"i3.4xlarge\":        {Region: \"us-east-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.248},\n\t\t\"i3.8xlarge\":        {Region: \"us-east-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.496},\n\t\t\"i3.16xlarge\":       {Region: \"us-east-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"i3.metal\":          {Region: \"us-east-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"i3en.large\":        {Region: \"us-east-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"i3en.xlarge\":       {Region: \"us-east-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"i3en.2xlarge\":      {Region: \"us-east-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"i3en.3xlarge\":      {Region: \"us-east-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.356},\n\t\t\"i3en.6xlarge\":      {Region: \"us-east-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"i3en.12xlarge\":     {Region: \"us-east-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"i3en.24xlarge\":     {Region: \"us-east-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.848},\n\t\t\"i3en.metal\":        {Region: \"us-east-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.848},\n\t\t\"im4gn.large\":       {Region: \"us-east-1\", Type: \"im4gn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1819},\n\t\t\"im4gn.xlarge\":      {Region: \"us-east-1\", Type: \"im4gn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.36379},\n\t\t\"im4gn.2xlarge\":     {Region: \"us-east-1\", Type: \"im4gn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.72758},\n\t\t\"im4gn.4xlarge\":     {Region: \"us-east-1\", Type: \"im4gn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.45517},\n\t\t\"im4gn.8xlarge\":     {Region: \"us-east-1\", Type: \"im4gn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.91034},\n\t\t\"im4gn.16xlarge\":    {Region: \"us-east-1\", Type: \"im4gn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.82067},\n\t\t\"inf1.xlarge\":       {Region: \"us-east-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.228},\n\t\t\"inf1.2xlarge\":      {Region: \"us-east-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.362},\n\t\t\"inf1.6xlarge\":      {Region: \"us-east-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.18},\n\t\t\"inf1.24xlarge\":     {Region: \"us-east-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 4.721},\n\t\t\"is4gen.medium\":     {Region: \"us-east-1\", Type: \"is4gen.medium\", Memory: kresource.MustParse(\"6144Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.14408},\n\t\t\"is4gen.large\":      {Region: \"us-east-1\", Type: \"is4gen.large\", Memory: kresource.MustParse(\"12288Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.28815},\n\t\t\"is4gen.xlarge\":     {Region: \"us-east-1\", Type: \"is4gen.xlarge\", Memory: kresource.MustParse(\"24576Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.5763},\n\t\t\"is4gen.2xlarge\":    {Region: \"us-east-1\", Type: \"is4gen.2xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.1526},\n\t\t\"is4gen.4xlarge\":    {Region: \"us-east-1\", Type: \"is4gen.4xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.3052},\n\t\t\"is4gen.8xlarge\":    {Region: \"us-east-1\", Type: \"is4gen.8xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.6104},\n\t\t\"m1.small\":          {Region: \"us-east-1\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.044},\n\t\t\"m1.medium\":         {Region: \"us-east-1\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.087},\n\t\t\"m1.large\":          {Region: \"us-east-1\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.175},\n\t\t\"m1.xlarge\":         {Region: \"us-east-1\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.35},\n\t\t\"m2.xlarge\":         {Region: \"us-east-1\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.245},\n\t\t\"m2.2xlarge\":        {Region: \"us-east-1\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.49},\n\t\t\"m2.4xlarge\":        {Region: \"us-east-1\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.98},\n\t\t\"m3.medium\":         {Region: \"us-east-1\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.067},\n\t\t\"m3.large\":          {Region: \"us-east-1\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"m3.xlarge\":         {Region: \"us-east-1\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"m3.2xlarge\":        {Region: \"us-east-1\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"m4.large\":          {Region: \"us-east-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"m4.xlarge\":         {Region: \"us-east-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"m4.2xlarge\":        {Region: \"us-east-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"m4.4xlarge\":        {Region: \"us-east-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"m4.10xlarge\":       {Region: \"us-east-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"m4.16xlarge\":       {Region: \"us-east-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"m5.large\":          {Region: \"us-east-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m5.xlarge\":         {Region: \"us-east-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m5.2xlarge\":        {Region: \"us-east-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m5.4xlarge\":        {Region: \"us-east-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m5.8xlarge\":        {Region: \"us-east-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m5.12xlarge\":       {Region: \"us-east-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m5.16xlarge\":       {Region: \"us-east-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m5.24xlarge\":       {Region: \"us-east-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m5.metal\":          {Region: \"us-east-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m5a.large\":         {Region: \"us-east-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"m5a.xlarge\":        {Region: \"us-east-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"m5a.2xlarge\":       {Region: \"us-east-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"m5a.4xlarge\":       {Region: \"us-east-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"m5a.8xlarge\":       {Region: \"us-east-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"m5a.12xlarge\":      {Region: \"us-east-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"m5a.16xlarge\":      {Region: \"us-east-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"m5a.24xlarge\":      {Region: \"us-east-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"m5ad.large\":        {Region: \"us-east-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.103},\n\t\t\"m5ad.xlarge\":       {Region: \"us-east-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.206},\n\t\t\"m5ad.2xlarge\":      {Region: \"us-east-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.412},\n\t\t\"m5ad.4xlarge\":      {Region: \"us-east-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.824},\n\t\t\"m5ad.8xlarge\":      {Region: \"us-east-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.648},\n\t\t\"m5ad.12xlarge\":     {Region: \"us-east-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.472},\n\t\t\"m5ad.16xlarge\":     {Region: \"us-east-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.296},\n\t\t\"m5ad.24xlarge\":     {Region: \"us-east-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.944},\n\t\t\"m5d.large\":         {Region: \"us-east-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"m5d.xlarge\":        {Region: \"us-east-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"m5d.2xlarge\":       {Region: \"us-east-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"m5d.4xlarge\":       {Region: \"us-east-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"m5d.8xlarge\":       {Region: \"us-east-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"m5d.12xlarge\":      {Region: \"us-east-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"m5d.16xlarge\":      {Region: \"us-east-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"m5d.24xlarge\":      {Region: \"us-east-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"m5d.metal\":         {Region: \"us-east-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"m5dn.large\":        {Region: \"us-east-1\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"m5dn.xlarge\":       {Region: \"us-east-1\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"m5dn.2xlarge\":      {Region: \"us-east-1\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"m5dn.4xlarge\":      {Region: \"us-east-1\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"m5dn.8xlarge\":      {Region: \"us-east-1\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"m5dn.12xlarge\":     {Region: \"us-east-1\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"m5dn.16xlarge\":     {Region: \"us-east-1\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"m5dn.24xlarge\":     {Region: \"us-east-1\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"m5dn.metal\":        {Region: \"us-east-1\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"m5n.large\":         {Region: \"us-east-1\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.119},\n\t\t\"m5n.xlarge\":        {Region: \"us-east-1\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.238},\n\t\t\"m5n.2xlarge\":       {Region: \"us-east-1\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.476},\n\t\t\"m5n.4xlarge\":       {Region: \"us-east-1\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.952},\n\t\t\"m5n.8xlarge\":       {Region: \"us-east-1\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.904},\n\t\t\"m5n.12xlarge\":      {Region: \"us-east-1\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.856},\n\t\t\"m5n.16xlarge\":      {Region: \"us-east-1\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.808},\n\t\t\"m5n.24xlarge\":      {Region: \"us-east-1\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.712},\n\t\t\"m5n.metal\":         {Region: \"us-east-1\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.712},\n\t\t\"m5zn.large\":        {Region: \"us-east-1\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1652},\n\t\t\"m5zn.xlarge\":       {Region: \"us-east-1\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3303},\n\t\t\"m5zn.2xlarge\":      {Region: \"us-east-1\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.6607},\n\t\t\"m5zn.3xlarge\":      {Region: \"us-east-1\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 0.991},\n\t\t\"m5zn.6xlarge\":      {Region: \"us-east-1\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 1.982},\n\t\t\"m5zn.12xlarge\":     {Region: \"us-east-1\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.9641},\n\t\t\"m5zn.metal\":        {Region: \"us-east-1\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.9641},\n\t\t\"m6a.large\":         {Region: \"us-east-1\", Type: \"m6a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0864},\n\t\t\"m6a.xlarge\":        {Region: \"us-east-1\", Type: \"m6a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1728},\n\t\t\"m6a.2xlarge\":       {Region: \"us-east-1\", Type: \"m6a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3456},\n\t\t\"m6a.4xlarge\":       {Region: \"us-east-1\", Type: \"m6a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6912},\n\t\t\"m6a.8xlarge\":       {Region: \"us-east-1\", Type: \"m6a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3824},\n\t\t\"m6a.12xlarge\":      {Region: \"us-east-1\", Type: \"m6a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0736},\n\t\t\"m6a.16xlarge\":      {Region: \"us-east-1\", Type: \"m6a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7648},\n\t\t\"m6a.24xlarge\":      {Region: \"us-east-1\", Type: \"m6a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.1472},\n\t\t\"m6a.32xlarge\":      {Region: \"us-east-1\", Type: \"m6a.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.5296},\n\t\t\"m6a.48xlarge\":      {Region: \"us-east-1\", Type: \"m6a.48xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 8.2944},\n\t\t\"m6a.metal\":         {Region: \"us-east-1\", Type: \"m6a.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 8.2944},\n\t\t\"m6g.medium\":        {Region: \"us-east-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0385},\n\t\t\"m6g.large\":         {Region: \"us-east-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.077},\n\t\t\"m6g.xlarge\":        {Region: \"us-east-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"m6g.2xlarge\":       {Region: \"us-east-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"m6g.4xlarge\":       {Region: \"us-east-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"m6g.8xlarge\":       {Region: \"us-east-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.232},\n\t\t\"m6g.12xlarge\":      {Region: \"us-east-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.848},\n\t\t\"m6g.16xlarge\":      {Region: \"us-east-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"m6g.metal\":         {Region: \"us-east-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"m6gd.medium\":       {Region: \"us-east-1\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0452},\n\t\t\"m6gd.large\":        {Region: \"us-east-1\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0904},\n\t\t\"m6gd.xlarge\":       {Region: \"us-east-1\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1808},\n\t\t\"m6gd.2xlarge\":      {Region: \"us-east-1\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3616},\n\t\t\"m6gd.4xlarge\":      {Region: \"us-east-1\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7232},\n\t\t\"m6gd.8xlarge\":      {Region: \"us-east-1\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.4464},\n\t\t\"m6gd.12xlarge\":     {Region: \"us-east-1\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.1696},\n\t\t\"m6gd.16xlarge\":     {Region: \"us-east-1\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8928},\n\t\t\"m6gd.metal\":        {Region: \"us-east-1\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8928},\n\t\t\"m6i.large\":         {Region: \"us-east-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m6i.xlarge\":        {Region: \"us-east-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m6i.2xlarge\":       {Region: \"us-east-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m6i.4xlarge\":       {Region: \"us-east-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m6i.8xlarge\":       {Region: \"us-east-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m6i.12xlarge\":      {Region: \"us-east-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m6i.16xlarge\":      {Region: \"us-east-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m6i.24xlarge\":      {Region: \"us-east-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m6i.32xlarge\":      {Region: \"us-east-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.144},\n\t\t\"m6i.metal\":         {Region: \"us-east-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.144},\n\t\t\"p2.xlarge\":         {Region: \"us-east-1\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.9},\n\t\t\"p2.8xlarge\":        {Region: \"us-east-1\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 7.2},\n\t\t\"p2.16xlarge\":       {Region: \"us-east-1\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 14.4},\n\t\t\"p3.2xlarge\":        {Region: \"us-east-1\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 3.06},\n\t\t\"p3.8xlarge\":        {Region: \"us-east-1\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 12.24},\n\t\t\"p3.16xlarge\":       {Region: \"us-east-1\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 24.48},\n\t\t\"p3dn.24xlarge\":     {Region: \"us-east-1\", Type: \"p3dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 31.212},\n\t\t\"p4d.24xlarge\":      {Region: \"us-east-1\", Type: \"p4d.24xlarge\", Memory: kresource.MustParse(\"1179648Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 32.7726},\n\t\t\"r3.large\":          {Region: \"us-east-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.166},\n\t\t\"r3.xlarge\":         {Region: \"us-east-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.333},\n\t\t\"r3.2xlarge\":        {Region: \"us-east-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.665},\n\t\t\"r3.4xlarge\":        {Region: \"us-east-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.33},\n\t\t\"r3.8xlarge\":        {Region: \"us-east-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.66},\n\t\t\"r4.large\":          {Region: \"us-east-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"r4.xlarge\":         {Region: \"us-east-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"r4.2xlarge\":        {Region: \"us-east-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"r4.4xlarge\":        {Region: \"us-east-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"r4.8xlarge\":        {Region: \"us-east-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"r4.16xlarge\":       {Region: \"us-east-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"r5.large\":          {Region: \"us-east-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"r5.xlarge\":         {Region: \"us-east-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"r5.2xlarge\":        {Region: \"us-east-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"r5.4xlarge\":        {Region: \"us-east-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"r5.8xlarge\":        {Region: \"us-east-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"r5.12xlarge\":       {Region: \"us-east-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"r5.16xlarge\":       {Region: \"us-east-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r5.24xlarge\":       {Region: \"us-east-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r5.metal\":          {Region: \"us-east-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r5a.large\":         {Region: \"us-east-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"r5a.xlarge\":        {Region: \"us-east-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"r5a.2xlarge\":       {Region: \"us-east-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"r5a.4xlarge\":       {Region: \"us-east-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"r5a.8xlarge\":       {Region: \"us-east-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"r5a.12xlarge\":      {Region: \"us-east-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"r5a.16xlarge\":      {Region: \"us-east-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"r5a.24xlarge\":      {Region: \"us-east-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"r5ad.large\":        {Region: \"us-east-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.131},\n\t\t\"r5ad.xlarge\":       {Region: \"us-east-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.262},\n\t\t\"r5ad.2xlarge\":      {Region: \"us-east-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"r5ad.4xlarge\":      {Region: \"us-east-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.048},\n\t\t\"r5ad.8xlarge\":      {Region: \"us-east-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.096},\n\t\t\"r5ad.12xlarge\":     {Region: \"us-east-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.144},\n\t\t\"r5ad.16xlarge\":     {Region: \"us-east-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.192},\n\t\t\"r5ad.24xlarge\":     {Region: \"us-east-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"r5b.large\":         {Region: \"us-east-1\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"r5b.xlarge\":        {Region: \"us-east-1\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"r5b.2xlarge\":       {Region: \"us-east-1\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"r5b.4xlarge\":       {Region: \"us-east-1\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"r5b.8xlarge\":       {Region: \"us-east-1\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.384},\n\t\t\"r5b.12xlarge\":      {Region: \"us-east-1\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"r5b.16xlarge\":      {Region: \"us-east-1\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.768},\n\t\t\"r5b.24xlarge\":      {Region: \"us-east-1\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5b.metal\":         {Region: \"us-east-1\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5d.large\":         {Region: \"us-east-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.144},\n\t\t\"r5d.xlarge\":        {Region: \"us-east-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.288},\n\t\t\"r5d.2xlarge\":       {Region: \"us-east-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.576},\n\t\t\"r5d.4xlarge\":       {Region: \"us-east-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.152},\n\t\t\"r5d.8xlarge\":       {Region: \"us-east-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"r5d.12xlarge\":      {Region: \"us-east-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"r5d.16xlarge\":      {Region: \"us-east-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"r5d.24xlarge\":      {Region: \"us-east-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.912},\n\t\t\"r5d.metal\":         {Region: \"us-east-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.912},\n\t\t\"r5dn.large\":        {Region: \"us-east-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"r5dn.xlarge\":       {Region: \"us-east-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"r5dn.2xlarge\":      {Region: \"us-east-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"r5dn.4xlarge\":      {Region: \"us-east-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"r5dn.8xlarge\":      {Region: \"us-east-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"r5dn.12xlarge\":     {Region: \"us-east-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"r5dn.16xlarge\":     {Region: \"us-east-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"r5dn.24xlarge\":     {Region: \"us-east-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5dn.metal\":        {Region: \"us-east-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5n.large\":         {Region: \"us-east-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"r5n.xlarge\":        {Region: \"us-east-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"r5n.2xlarge\":       {Region: \"us-east-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"r5n.4xlarge\":       {Region: \"us-east-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"r5n.8xlarge\":       {Region: \"us-east-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.384},\n\t\t\"r5n.12xlarge\":      {Region: \"us-east-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"r5n.16xlarge\":      {Region: \"us-east-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.768},\n\t\t\"r5n.24xlarge\":      {Region: \"us-east-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5n.metal\":         {Region: \"us-east-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r6g.medium\":        {Region: \"us-east-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0504},\n\t\t\"r6g.large\":         {Region: \"us-east-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1008},\n\t\t\"r6g.xlarge\":        {Region: \"us-east-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2016},\n\t\t\"r6g.2xlarge\":       {Region: \"us-east-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4032},\n\t\t\"r6g.4xlarge\":       {Region: \"us-east-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8064},\n\t\t\"r6g.8xlarge\":       {Region: \"us-east-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6128},\n\t\t\"r6g.12xlarge\":      {Region: \"us-east-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.4192},\n\t\t\"r6g.16xlarge\":      {Region: \"us-east-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2256},\n\t\t\"r6g.metal\":         {Region: \"us-east-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2256},\n\t\t\"r6gd.medium\":       {Region: \"us-east-1\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0576},\n\t\t\"r6gd.large\":        {Region: \"us-east-1\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1152},\n\t\t\"r6gd.xlarge\":       {Region: \"us-east-1\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2304},\n\t\t\"r6gd.2xlarge\":      {Region: \"us-east-1\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4608},\n\t\t\"r6gd.4xlarge\":      {Region: \"us-east-1\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9216},\n\t\t\"r6gd.8xlarge\":      {Region: \"us-east-1\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.8432},\n\t\t\"r6gd.12xlarge\":     {Region: \"us-east-1\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.7648},\n\t\t\"r6gd.16xlarge\":     {Region: \"us-east-1\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.6864},\n\t\t\"r6gd.metal\":        {Region: \"us-east-1\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.6864},\n\t\t\"r6i.large\":         {Region: \"us-east-1\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"r6i.xlarge\":        {Region: \"us-east-1\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"r6i.2xlarge\":       {Region: \"us-east-1\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"r6i.4xlarge\":       {Region: \"us-east-1\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"r6i.8xlarge\":       {Region: \"us-east-1\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"r6i.12xlarge\":      {Region: \"us-east-1\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"r6i.16xlarge\":      {Region: \"us-east-1\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r6i.24xlarge\":      {Region: \"us-east-1\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r6i.32xlarge\":      {Region: \"us-east-1\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"r6i.metal\":         {Region: \"us-east-1\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"t1.micro\":          {Region: \"us-east-1\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.02},\n\t\t\"t2.nano\":           {Region: \"us-east-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0058},\n\t\t\"t2.micro\":          {Region: \"us-east-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0116},\n\t\t\"t2.small\":          {Region: \"us-east-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.023},\n\t\t\"t2.medium\":         {Region: \"us-east-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0464},\n\t\t\"t2.large\":          {Region: \"us-east-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0928},\n\t\t\"t2.xlarge\":         {Region: \"us-east-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1856},\n\t\t\"t2.2xlarge\":        {Region: \"us-east-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3712},\n\t\t\"t3.nano\":           {Region: \"us-east-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0052},\n\t\t\"t3.micro\":          {Region: \"us-east-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0104},\n\t\t\"t3.small\":          {Region: \"us-east-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0208},\n\t\t\"t3.medium\":         {Region: \"us-east-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0416},\n\t\t\"t3.large\":          {Region: \"us-east-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0832},\n\t\t\"t3.xlarge\":         {Region: \"us-east-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1664},\n\t\t\"t3.2xlarge\":        {Region: \"us-east-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3328},\n\t\t\"t3a.nano\":          {Region: \"us-east-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0047},\n\t\t\"t3a.micro\":         {Region: \"us-east-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0094},\n\t\t\"t3a.small\":         {Region: \"us-east-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0188},\n\t\t\"t3a.medium\":        {Region: \"us-east-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0376},\n\t\t\"t3a.large\":         {Region: \"us-east-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0752},\n\t\t\"t3a.xlarge\":        {Region: \"us-east-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1504},\n\t\t\"t3a.2xlarge\":       {Region: \"us-east-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3008},\n\t\t\"t4g.nano\":          {Region: \"us-east-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0042},\n\t\t\"t4g.micro\":         {Region: \"us-east-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0084},\n\t\t\"t4g.small\":         {Region: \"us-east-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0168},\n\t\t\"t4g.medium\":        {Region: \"us-east-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0336},\n\t\t\"t4g.large\":         {Region: \"us-east-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0672},\n\t\t\"t4g.xlarge\":        {Region: \"us-east-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1344},\n\t\t\"t4g.2xlarge\":       {Region: \"us-east-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2688},\n\t\t\"u-12tb1.112xlarge\": {Region: \"us-east-1\", Type: \"u-12tb1.112xlarge\", Memory: kresource.MustParse(\"12582912Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 109.2},\n\t\t\"u-3tb1.56xlarge\":   {Region: \"us-east-1\", Type: \"u-3tb1.56xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 27.3},\n\t\t\"u-6tb1.56xlarge\":   {Region: \"us-east-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 46.40391},\n\t\t\"u-6tb1.112xlarge\":  {Region: \"us-east-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 54.6},\n\t\t\"u-9tb1.112xlarge\":  {Region: \"us-east-1\", Type: \"u-9tb1.112xlarge\", Memory: kresource.MustParse(\"9437184Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 81.9},\n\t\t\"vt1.3xlarge\":       {Region: \"us-east-1\", Type: \"vt1.3xlarge\", Memory: kresource.MustParse(\"24576Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 0.65},\n\t\t\"vt1.6xlarge\":       {Region: \"us-east-1\", Type: \"vt1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 1.3},\n\t\t\"vt1.24xlarge\":      {Region: \"us-east-1\", Type: \"vt1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.2},\n\t\t\"x1.16xlarge\":       {Region: \"us-east-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.669},\n\t\t\"x1.32xlarge\":       {Region: \"us-east-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 13.338},\n\t\t\"x1e.xlarge\":        {Region: \"us-east-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.834},\n\t\t\"x1e.2xlarge\":       {Region: \"us-east-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.668},\n\t\t\"x1e.4xlarge\":       {Region: \"us-east-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"x1e.8xlarge\":       {Region: \"us-east-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.672},\n\t\t\"x1e.16xlarge\":      {Region: \"us-east-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.344},\n\t\t\"x1e.32xlarge\":      {Region: \"us-east-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 26.688},\n\t\t\"x2gd.medium\":       {Region: \"us-east-1\", Type: \"x2gd.medium\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0835},\n\t\t\"x2gd.large\":        {Region: \"us-east-1\", Type: \"x2gd.large\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"x2gd.xlarge\":       {Region: \"us-east-1\", Type: \"x2gd.xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"x2gd.2xlarge\":      {Region: \"us-east-1\", Type: \"x2gd.2xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"x2gd.4xlarge\":      {Region: \"us-east-1\", Type: \"x2gd.4xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"x2gd.8xlarge\":      {Region: \"us-east-1\", Type: \"x2gd.8xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"x2gd.12xlarge\":     {Region: \"us-east-1\", Type: \"x2gd.12xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"x2gd.16xlarge\":     {Region: \"us-east-1\", Type: \"x2gd.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"x2gd.metal\":        {Region: \"us-east-1\", Type: \"x2gd.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"x2idn.16xlarge\":    {Region: \"us-east-1\", Type: \"x2idn.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.669},\n\t\t\"x2idn.24xlarge\":    {Region: \"us-east-1\", Type: \"x2idn.24xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.0035},\n\t\t\"x2idn.32xlarge\":    {Region: \"us-east-1\", Type: \"x2idn.32xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 13.338},\n\t\t\"x2iedn.xlarge\":     {Region: \"us-east-1\", Type: \"x2iedn.xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.83363},\n\t\t\"x2iedn.2xlarge\":    {Region: \"us-east-1\", Type: \"x2iedn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.66725},\n\t\t\"x2iedn.4xlarge\":    {Region: \"us-east-1\", Type: \"x2iedn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.3345},\n\t\t\"x2iedn.8xlarge\":    {Region: \"us-east-1\", Type: \"x2iedn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.669},\n\t\t\"x2iedn.16xlarge\":   {Region: \"us-east-1\", Type: \"x2iedn.16xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.338},\n\t\t\"x2iedn.24xlarge\":   {Region: \"us-east-1\", Type: \"x2iedn.24xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 20.007},\n\t\t\"x2iedn.32xlarge\":   {Region: \"us-east-1\", Type: \"x2iedn.32xlarge\", Memory: kresource.MustParse(\"4194304Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 26.676},\n\t\t\"x2iezn.2xlarge\":    {Region: \"us-east-1\", Type: \"x2iezn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.668},\n\t\t\"x2iezn.4xlarge\":    {Region: \"us-east-1\", Type: \"x2iezn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"x2iezn.6xlarge\":    {Region: \"us-east-1\", Type: \"x2iezn.6xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 5.004},\n\t\t\"x2iezn.8xlarge\":    {Region: \"us-east-1\", Type: \"x2iezn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.672},\n\t\t\"x2iezn.12xlarge\":   {Region: \"us-east-1\", Type: \"x2iezn.12xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 10.008},\n\t\t\"x2iezn.metal\":      {Region: \"us-east-1\", Type: \"x2iezn.metal\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 10.008},\n\t\t\"z1d.large\":         {Region: \"us-east-1\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"z1d.xlarge\":        {Region: \"us-east-1\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"z1d.2xlarge\":       {Region: \"us-east-1\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.744},\n\t\t\"z1d.3xlarge\":       {Region: \"us-east-1\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.116},\n\t\t\"z1d.6xlarge\":       {Region: \"us-east-1\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.232},\n\t\t\"z1d.12xlarge\":      {Region: \"us-east-1\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"z1d.metal\":         {Region: \"us-east-1\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.464},\n\t},\n\t\"us-east-2\": {\n\t\t\"a1.medium\":        {Region: \"us-east-2\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0255},\n\t\t\"a1.large\":         {Region: \"us-east-2\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.051},\n\t\t\"a1.xlarge\":        {Region: \"us-east-2\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.102},\n\t\t\"a1.2xlarge\":       {Region: \"us-east-2\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.204},\n\t\t\"a1.4xlarge\":       {Region: \"us-east-2\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"a1.metal\":         {Region: \"us-east-2\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"c4.large\":         {Region: \"us-east-2\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"c4.xlarge\":        {Region: \"us-east-2\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.199},\n\t\t\"c4.2xlarge\":       {Region: \"us-east-2\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.398},\n\t\t\"c4.4xlarge\":       {Region: \"us-east-2\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.796},\n\t\t\"c4.8xlarge\":       {Region: \"us-east-2\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.591},\n\t\t\"c5.large\":         {Region: \"us-east-2\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c5.xlarge\":        {Region: \"us-east-2\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c5.2xlarge\":       {Region: \"us-east-2\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c5.4xlarge\":       {Region: \"us-east-2\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c5.9xlarge\":       {Region: \"us-east-2\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.53},\n\t\t\"c5.12xlarge\":      {Region: \"us-east-2\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c5.18xlarge\":      {Region: \"us-east-2\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.06},\n\t\t\"c5.24xlarge\":      {Region: \"us-east-2\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c5.metal\":         {Region: \"us-east-2\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c5a.large\":        {Region: \"us-east-2\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.077},\n\t\t\"c5a.xlarge\":       {Region: \"us-east-2\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"c5a.2xlarge\":      {Region: \"us-east-2\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"c5a.4xlarge\":      {Region: \"us-east-2\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"c5a.8xlarge\":      {Region: \"us-east-2\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.232},\n\t\t\"c5a.12xlarge\":     {Region: \"us-east-2\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.848},\n\t\t\"c5a.16xlarge\":     {Region: \"us-east-2\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"c5a.24xlarge\":     {Region: \"us-east-2\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.696},\n\t\t\"c5ad.large\":       {Region: \"us-east-2\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"c5ad.xlarge\":      {Region: \"us-east-2\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"c5ad.2xlarge\":     {Region: \"us-east-2\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"c5ad.4xlarge\":     {Region: \"us-east-2\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"c5ad.8xlarge\":     {Region: \"us-east-2\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"c5ad.12xlarge\":    {Region: \"us-east-2\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"c5ad.16xlarge\":    {Region: \"us-east-2\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"c5ad.24xlarge\":    {Region: \"us-east-2\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"c5d.large\":        {Region: \"us-east-2\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"c5d.xlarge\":       {Region: \"us-east-2\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"c5d.2xlarge\":      {Region: \"us-east-2\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"c5d.4xlarge\":      {Region: \"us-east-2\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"c5d.9xlarge\":      {Region: \"us-east-2\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"c5d.12xlarge\":     {Region: \"us-east-2\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c5d.18xlarge\":     {Region: \"us-east-2\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"c5d.24xlarge\":     {Region: \"us-east-2\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5d.metal\":        {Region: \"us-east-2\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5n.large\":        {Region: \"us-east-2\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"c5n.xlarge\":       {Region: \"us-east-2\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"c5n.2xlarge\":      {Region: \"us-east-2\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"c5n.4xlarge\":      {Region: \"us-east-2\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"c5n.9xlarge\":      {Region: \"us-east-2\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.944},\n\t\t\"c5n.18xlarge\":     {Region: \"us-east-2\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c5n.metal\":        {Region: \"us-east-2\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c6g.medium\":       {Region: \"us-east-2\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.034},\n\t\t\"c6g.large\":        {Region: \"us-east-2\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.068},\n\t\t\"c6g.xlarge\":       {Region: \"us-east-2\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"c6g.2xlarge\":      {Region: \"us-east-2\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"c6g.4xlarge\":      {Region: \"us-east-2\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"c6g.8xlarge\":      {Region: \"us-east-2\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"c6g.12xlarge\":     {Region: \"us-east-2\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.632},\n\t\t\"c6g.16xlarge\":     {Region: \"us-east-2\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"c6g.metal\":        {Region: \"us-east-2\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"c6gd.medium\":      {Region: \"us-east-2\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0384},\n\t\t\"c6gd.large\":       {Region: \"us-east-2\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0768},\n\t\t\"c6gd.xlarge\":      {Region: \"us-east-2\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1536},\n\t\t\"c6gd.2xlarge\":     {Region: \"us-east-2\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3072},\n\t\t\"c6gd.4xlarge\":     {Region: \"us-east-2\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6144},\n\t\t\"c6gd.8xlarge\":     {Region: \"us-east-2\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.2288},\n\t\t\"c6gd.12xlarge\":    {Region: \"us-east-2\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.8432},\n\t\t\"c6gd.16xlarge\":    {Region: \"us-east-2\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.4576},\n\t\t\"c6gd.metal\":       {Region: \"us-east-2\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.4576},\n\t\t\"c6gn.medium\":      {Region: \"us-east-2\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0432},\n\t\t\"c6gn.large\":       {Region: \"us-east-2\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0864},\n\t\t\"c6gn.xlarge\":      {Region: \"us-east-2\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1728},\n\t\t\"c6gn.2xlarge\":     {Region: \"us-east-2\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3456},\n\t\t\"c6gn.4xlarge\":     {Region: \"us-east-2\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6912},\n\t\t\"c6gn.8xlarge\":     {Region: \"us-east-2\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3824},\n\t\t\"c6gn.12xlarge\":    {Region: \"us-east-2\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0736},\n\t\t\"c6gn.16xlarge\":    {Region: \"us-east-2\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7648},\n\t\t\"c6i.large\":        {Region: \"us-east-2\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c6i.xlarge\":       {Region: \"us-east-2\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c6i.2xlarge\":      {Region: \"us-east-2\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c6i.4xlarge\":      {Region: \"us-east-2\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c6i.8xlarge\":      {Region: \"us-east-2\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.36},\n\t\t\"c6i.12xlarge\":     {Region: \"us-east-2\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c6i.16xlarge\":     {Region: \"us-east-2\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.72},\n\t\t\"c6i.24xlarge\":     {Region: \"us-east-2\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c6i.32xlarge\":     {Region: \"us-east-2\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.44},\n\t\t\"c6i.metal\":        {Region: \"us-east-2\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.44},\n\t\t\"d2.xlarge\":        {Region: \"us-east-2\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.69},\n\t\t\"d2.2xlarge\":       {Region: \"us-east-2\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.38},\n\t\t\"d2.4xlarge\":       {Region: \"us-east-2\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.76},\n\t\t\"d2.8xlarge\":       {Region: \"us-east-2\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"d3.xlarge\":        {Region: \"us-east-2\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.499},\n\t\t\"d3.2xlarge\":       {Region: \"us-east-2\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.999},\n\t\t\"d3.4xlarge\":       {Region: \"us-east-2\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.998},\n\t\t\"d3.8xlarge\":       {Region: \"us-east-2\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.99552},\n\t\t\"g3.4xlarge\":       {Region: \"us-east-2\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.14},\n\t\t\"g3.8xlarge\":       {Region: \"us-east-2\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.28},\n\t\t\"g3.16xlarge\":      {Region: \"us-east-2\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 4.56},\n\t\t\"g3s.xlarge\":       {Region: \"us-east-2\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.75},\n\t\t\"g4ad.xlarge\":      {Region: \"us-east-2\", Type: \"g4ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.37853},\n\t\t\"g4ad.2xlarge\":     {Region: \"us-east-2\", Type: \"g4ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.54117},\n\t\t\"g4ad.4xlarge\":     {Region: \"us-east-2\", Type: \"g4ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 0.867},\n\t\t\"g4ad.8xlarge\":     {Region: \"us-east-2\", Type: \"g4ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 1.734},\n\t\t\"g4ad.16xlarge\":    {Region: \"us-east-2\", Type: \"g4ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 3.468},\n\t\t\"g4dn.xlarge\":      {Region: \"us-east-2\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.526},\n\t\t\"g4dn.2xlarge\":     {Region: \"us-east-2\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.752},\n\t\t\"g4dn.4xlarge\":     {Region: \"us-east-2\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.204},\n\t\t\"g4dn.8xlarge\":     {Region: \"us-east-2\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.176},\n\t\t\"g4dn.12xlarge\":    {Region: \"us-east-2\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 3.912},\n\t\t\"g4dn.16xlarge\":    {Region: \"us-east-2\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.352},\n\t\t\"g4dn.metal\":       {Region: \"us-east-2\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 7.824},\n\t\t\"h1.2xlarge\":       {Region: \"us-east-2\", Type: \"h1.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.468},\n\t\t\"h1.4xlarge\":       {Region: \"us-east-2\", Type: \"h1.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.936},\n\t\t\"h1.8xlarge\":       {Region: \"us-east-2\", Type: \"h1.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.872},\n\t\t\"h1.16xlarge\":      {Region: \"us-east-2\", Type: \"h1.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.744},\n\t\t\"hpc6a.48xlarge\":   {Region: \"us-east-2\", Type: \"hpc6a.48xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"i2.xlarge\":        {Region: \"us-east-2\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.853},\n\t\t\"i2.2xlarge\":       {Region: \"us-east-2\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.705},\n\t\t\"i2.4xlarge\":       {Region: \"us-east-2\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.41},\n\t\t\"i2.8xlarge\":       {Region: \"us-east-2\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.82},\n\t\t\"i3.large\":         {Region: \"us-east-2\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.156},\n\t\t\"i3.xlarge\":        {Region: \"us-east-2\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.312},\n\t\t\"i3.2xlarge\":       {Region: \"us-east-2\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.624},\n\t\t\"i3.4xlarge\":       {Region: \"us-east-2\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.248},\n\t\t\"i3.8xlarge\":       {Region: \"us-east-2\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.496},\n\t\t\"i3.16xlarge\":      {Region: \"us-east-2\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"i3.metal\":         {Region: \"us-east-2\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"i3en.large\":       {Region: \"us-east-2\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"i3en.xlarge\":      {Region: \"us-east-2\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"i3en.2xlarge\":     {Region: \"us-east-2\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"i3en.3xlarge\":     {Region: \"us-east-2\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.356},\n\t\t\"i3en.6xlarge\":     {Region: \"us-east-2\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"i3en.12xlarge\":    {Region: \"us-east-2\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"i3en.24xlarge\":    {Region: \"us-east-2\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.848},\n\t\t\"i3en.metal\":       {Region: \"us-east-2\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.848},\n\t\t\"im4gn.large\":      {Region: \"us-east-2\", Type: \"im4gn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1819},\n\t\t\"im4gn.xlarge\":     {Region: \"us-east-2\", Type: \"im4gn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.36379},\n\t\t\"im4gn.2xlarge\":    {Region: \"us-east-2\", Type: \"im4gn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.72758},\n\t\t\"im4gn.4xlarge\":    {Region: \"us-east-2\", Type: \"im4gn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.45517},\n\t\t\"im4gn.8xlarge\":    {Region: \"us-east-2\", Type: \"im4gn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.91034},\n\t\t\"im4gn.16xlarge\":   {Region: \"us-east-2\", Type: \"im4gn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.82067},\n\t\t\"inf1.xlarge\":      {Region: \"us-east-2\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.228},\n\t\t\"inf1.2xlarge\":     {Region: \"us-east-2\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.362},\n\t\t\"inf1.6xlarge\":     {Region: \"us-east-2\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.18},\n\t\t\"inf1.24xlarge\":    {Region: \"us-east-2\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 4.721},\n\t\t\"is4gen.medium\":    {Region: \"us-east-2\", Type: \"is4gen.medium\", Memory: kresource.MustParse(\"6144Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.14408},\n\t\t\"is4gen.large\":     {Region: \"us-east-2\", Type: \"is4gen.large\", Memory: kresource.MustParse(\"12288Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.28815},\n\t\t\"is4gen.xlarge\":    {Region: \"us-east-2\", Type: \"is4gen.xlarge\", Memory: kresource.MustParse(\"24576Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.5763},\n\t\t\"is4gen.2xlarge\":   {Region: \"us-east-2\", Type: \"is4gen.2xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.1526},\n\t\t\"is4gen.4xlarge\":   {Region: \"us-east-2\", Type: \"is4gen.4xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.3052},\n\t\t\"is4gen.8xlarge\":   {Region: \"us-east-2\", Type: \"is4gen.8xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.6104},\n\t\t\"m4.large\":         {Region: \"us-east-2\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"m4.xlarge\":        {Region: \"us-east-2\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"m4.2xlarge\":       {Region: \"us-east-2\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"m4.4xlarge\":       {Region: \"us-east-2\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"m4.10xlarge\":      {Region: \"us-east-2\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"m4.16xlarge\":      {Region: \"us-east-2\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"m5.large\":         {Region: \"us-east-2\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m5.xlarge\":        {Region: \"us-east-2\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m5.2xlarge\":       {Region: \"us-east-2\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m5.4xlarge\":       {Region: \"us-east-2\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m5.8xlarge\":       {Region: \"us-east-2\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m5.12xlarge\":      {Region: \"us-east-2\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m5.16xlarge\":      {Region: \"us-east-2\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m5.24xlarge\":      {Region: \"us-east-2\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m5.metal\":         {Region: \"us-east-2\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m5a.large\":        {Region: \"us-east-2\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"m5a.xlarge\":       {Region: \"us-east-2\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"m5a.2xlarge\":      {Region: \"us-east-2\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"m5a.4xlarge\":      {Region: \"us-east-2\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"m5a.8xlarge\":      {Region: \"us-east-2\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"m5a.12xlarge\":     {Region: \"us-east-2\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"m5a.16xlarge\":     {Region: \"us-east-2\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"m5a.24xlarge\":     {Region: \"us-east-2\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"m5ad.large\":       {Region: \"us-east-2\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.103},\n\t\t\"m5ad.xlarge\":      {Region: \"us-east-2\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.206},\n\t\t\"m5ad.2xlarge\":     {Region: \"us-east-2\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.412},\n\t\t\"m5ad.4xlarge\":     {Region: \"us-east-2\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.824},\n\t\t\"m5ad.8xlarge\":     {Region: \"us-east-2\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.648},\n\t\t\"m5ad.12xlarge\":    {Region: \"us-east-2\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.472},\n\t\t\"m5ad.16xlarge\":    {Region: \"us-east-2\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.296},\n\t\t\"m5ad.24xlarge\":    {Region: \"us-east-2\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.944},\n\t\t\"m5d.large\":        {Region: \"us-east-2\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"m5d.xlarge\":       {Region: \"us-east-2\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"m5d.2xlarge\":      {Region: \"us-east-2\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"m5d.4xlarge\":      {Region: \"us-east-2\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"m5d.8xlarge\":      {Region: \"us-east-2\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"m5d.12xlarge\":     {Region: \"us-east-2\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"m5d.16xlarge\":     {Region: \"us-east-2\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"m5d.24xlarge\":     {Region: \"us-east-2\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"m5d.metal\":        {Region: \"us-east-2\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"m5dn.large\":       {Region: \"us-east-2\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"m5dn.xlarge\":      {Region: \"us-east-2\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"m5dn.2xlarge\":     {Region: \"us-east-2\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"m5dn.4xlarge\":     {Region: \"us-east-2\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"m5dn.8xlarge\":     {Region: \"us-east-2\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"m5dn.12xlarge\":    {Region: \"us-east-2\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"m5dn.16xlarge\":    {Region: \"us-east-2\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"m5dn.24xlarge\":    {Region: \"us-east-2\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"m5dn.metal\":       {Region: \"us-east-2\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"m5n.large\":        {Region: \"us-east-2\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.119},\n\t\t\"m5n.xlarge\":       {Region: \"us-east-2\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.238},\n\t\t\"m5n.2xlarge\":      {Region: \"us-east-2\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.476},\n\t\t\"m5n.4xlarge\":      {Region: \"us-east-2\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.952},\n\t\t\"m5n.8xlarge\":      {Region: \"us-east-2\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.904},\n\t\t\"m5n.12xlarge\":     {Region: \"us-east-2\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.856},\n\t\t\"m5n.16xlarge\":     {Region: \"us-east-2\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.808},\n\t\t\"m5n.24xlarge\":     {Region: \"us-east-2\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.712},\n\t\t\"m5n.metal\":        {Region: \"us-east-2\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.712},\n\t\t\"m5zn.large\":       {Region: \"us-east-2\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1652},\n\t\t\"m5zn.xlarge\":      {Region: \"us-east-2\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3303},\n\t\t\"m5zn.2xlarge\":     {Region: \"us-east-2\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.6607},\n\t\t\"m5zn.3xlarge\":     {Region: \"us-east-2\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 0.991},\n\t\t\"m5zn.6xlarge\":     {Region: \"us-east-2\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 1.982},\n\t\t\"m5zn.12xlarge\":    {Region: \"us-east-2\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.9641},\n\t\t\"m5zn.metal\":       {Region: \"us-east-2\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.9641},\n\t\t\"m6g.medium\":       {Region: \"us-east-2\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0385},\n\t\t\"m6g.large\":        {Region: \"us-east-2\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.077},\n\t\t\"m6g.xlarge\":       {Region: \"us-east-2\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"m6g.2xlarge\":      {Region: \"us-east-2\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"m6g.4xlarge\":      {Region: \"us-east-2\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"m6g.8xlarge\":      {Region: \"us-east-2\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.232},\n\t\t\"m6g.12xlarge\":     {Region: \"us-east-2\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.848},\n\t\t\"m6g.16xlarge\":     {Region: \"us-east-2\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"m6g.metal\":        {Region: \"us-east-2\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"m6gd.medium\":      {Region: \"us-east-2\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0452},\n\t\t\"m6gd.large\":       {Region: \"us-east-2\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0904},\n\t\t\"m6gd.xlarge\":      {Region: \"us-east-2\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1808},\n\t\t\"m6gd.2xlarge\":     {Region: \"us-east-2\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3616},\n\t\t\"m6gd.4xlarge\":     {Region: \"us-east-2\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7232},\n\t\t\"m6gd.8xlarge\":     {Region: \"us-east-2\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.4464},\n\t\t\"m6gd.12xlarge\":    {Region: \"us-east-2\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.1696},\n\t\t\"m6gd.16xlarge\":    {Region: \"us-east-2\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8928},\n\t\t\"m6gd.metal\":       {Region: \"us-east-2\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8928},\n\t\t\"m6i.large\":        {Region: \"us-east-2\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m6i.xlarge\":       {Region: \"us-east-2\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m6i.2xlarge\":      {Region: \"us-east-2\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m6i.4xlarge\":      {Region: \"us-east-2\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m6i.8xlarge\":      {Region: \"us-east-2\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m6i.12xlarge\":     {Region: \"us-east-2\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m6i.16xlarge\":     {Region: \"us-east-2\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m6i.24xlarge\":     {Region: \"us-east-2\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m6i.32xlarge\":     {Region: \"us-east-2\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.144},\n\t\t\"m6i.metal\":        {Region: \"us-east-2\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.144},\n\t\t\"p2.xlarge\":        {Region: \"us-east-2\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.9},\n\t\t\"p2.8xlarge\":       {Region: \"us-east-2\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 7.2},\n\t\t\"p2.16xlarge\":      {Region: \"us-east-2\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 14.4},\n\t\t\"p3.2xlarge\":       {Region: \"us-east-2\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 3.06},\n\t\t\"p3.8xlarge\":       {Region: \"us-east-2\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 12.24},\n\t\t\"p3.16xlarge\":      {Region: \"us-east-2\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 24.48},\n\t\t\"p4d.24xlarge\":     {Region: \"us-east-2\", Type: \"p4d.24xlarge\", Memory: kresource.MustParse(\"1179648Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 32.7726},\n\t\t\"r3.large\":         {Region: \"us-east-2\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.166},\n\t\t\"r3.xlarge\":        {Region: \"us-east-2\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.332},\n\t\t\"r3.2xlarge\":       {Region: \"us-east-2\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.664},\n\t\t\"r3.4xlarge\":       {Region: \"us-east-2\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.328},\n\t\t\"r3.8xlarge\":       {Region: \"us-east-2\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.656},\n\t\t\"r4.large\":         {Region: \"us-east-2\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"r4.xlarge\":        {Region: \"us-east-2\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"r4.2xlarge\":       {Region: \"us-east-2\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"r4.4xlarge\":       {Region: \"us-east-2\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"r4.8xlarge\":       {Region: \"us-east-2\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"r4.16xlarge\":      {Region: \"us-east-2\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"r5.large\":         {Region: \"us-east-2\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"r5.xlarge\":        {Region: \"us-east-2\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"r5.2xlarge\":       {Region: \"us-east-2\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"r5.4xlarge\":       {Region: \"us-east-2\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"r5.8xlarge\":       {Region: \"us-east-2\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"r5.12xlarge\":      {Region: \"us-east-2\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"r5.16xlarge\":      {Region: \"us-east-2\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r5.24xlarge\":      {Region: \"us-east-2\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r5.metal\":         {Region: \"us-east-2\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r5a.large\":        {Region: \"us-east-2\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"r5a.xlarge\":       {Region: \"us-east-2\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"r5a.2xlarge\":      {Region: \"us-east-2\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"r5a.4xlarge\":      {Region: \"us-east-2\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"r5a.8xlarge\":      {Region: \"us-east-2\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"r5a.12xlarge\":     {Region: \"us-east-2\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"r5a.16xlarge\":     {Region: \"us-east-2\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"r5a.24xlarge\":     {Region: \"us-east-2\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"r5ad.large\":       {Region: \"us-east-2\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.131},\n\t\t\"r5ad.xlarge\":      {Region: \"us-east-2\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.262},\n\t\t\"r5ad.2xlarge\":     {Region: \"us-east-2\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"r5ad.4xlarge\":     {Region: \"us-east-2\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.048},\n\t\t\"r5ad.8xlarge\":     {Region: \"us-east-2\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.096},\n\t\t\"r5ad.12xlarge\":    {Region: \"us-east-2\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.144},\n\t\t\"r5ad.16xlarge\":    {Region: \"us-east-2\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.192},\n\t\t\"r5ad.24xlarge\":    {Region: \"us-east-2\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"r5b.large\":        {Region: \"us-east-2\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"r5b.xlarge\":       {Region: \"us-east-2\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"r5b.2xlarge\":      {Region: \"us-east-2\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"r5b.4xlarge\":      {Region: \"us-east-2\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"r5b.8xlarge\":      {Region: \"us-east-2\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.384},\n\t\t\"r5b.12xlarge\":     {Region: \"us-east-2\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"r5b.16xlarge\":     {Region: \"us-east-2\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.768},\n\t\t\"r5b.24xlarge\":     {Region: \"us-east-2\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5b.metal\":        {Region: \"us-east-2\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5d.large\":        {Region: \"us-east-2\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.144},\n\t\t\"r5d.xlarge\":       {Region: \"us-east-2\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.288},\n\t\t\"r5d.2xlarge\":      {Region: \"us-east-2\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.576},\n\t\t\"r5d.4xlarge\":      {Region: \"us-east-2\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.152},\n\t\t\"r5d.8xlarge\":      {Region: \"us-east-2\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"r5d.12xlarge\":     {Region: \"us-east-2\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"r5d.16xlarge\":     {Region: \"us-east-2\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"r5d.24xlarge\":     {Region: \"us-east-2\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.912},\n\t\t\"r5d.metal\":        {Region: \"us-east-2\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.912},\n\t\t\"r5dn.large\":       {Region: \"us-east-2\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"r5dn.xlarge\":      {Region: \"us-east-2\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"r5dn.2xlarge\":     {Region: \"us-east-2\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"r5dn.4xlarge\":     {Region: \"us-east-2\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"r5dn.8xlarge\":     {Region: \"us-east-2\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"r5dn.12xlarge\":    {Region: \"us-east-2\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"r5dn.16xlarge\":    {Region: \"us-east-2\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"r5dn.24xlarge\":    {Region: \"us-east-2\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5dn.metal\":       {Region: \"us-east-2\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5n.large\":        {Region: \"us-east-2\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"r5n.xlarge\":       {Region: \"us-east-2\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"r5n.2xlarge\":      {Region: \"us-east-2\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"r5n.4xlarge\":      {Region: \"us-east-2\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"r5n.8xlarge\":      {Region: \"us-east-2\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.384},\n\t\t\"r5n.12xlarge\":     {Region: \"us-east-2\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"r5n.16xlarge\":     {Region: \"us-east-2\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.768},\n\t\t\"r5n.24xlarge\":     {Region: \"us-east-2\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5n.metal\":        {Region: \"us-east-2\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r6g.medium\":       {Region: \"us-east-2\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0504},\n\t\t\"r6g.large\":        {Region: \"us-east-2\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1008},\n\t\t\"r6g.xlarge\":       {Region: \"us-east-2\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2016},\n\t\t\"r6g.2xlarge\":      {Region: \"us-east-2\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4032},\n\t\t\"r6g.4xlarge\":      {Region: \"us-east-2\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8064},\n\t\t\"r6g.8xlarge\":      {Region: \"us-east-2\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6128},\n\t\t\"r6g.12xlarge\":     {Region: \"us-east-2\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.4192},\n\t\t\"r6g.16xlarge\":     {Region: \"us-east-2\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2256},\n\t\t\"r6g.metal\":        {Region: \"us-east-2\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2256},\n\t\t\"r6gd.medium\":      {Region: \"us-east-2\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0576},\n\t\t\"r6gd.large\":       {Region: \"us-east-2\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1152},\n\t\t\"r6gd.xlarge\":      {Region: \"us-east-2\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2304},\n\t\t\"r6gd.2xlarge\":     {Region: \"us-east-2\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4608},\n\t\t\"r6gd.4xlarge\":     {Region: \"us-east-2\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9216},\n\t\t\"r6gd.8xlarge\":     {Region: \"us-east-2\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.8432},\n\t\t\"r6gd.12xlarge\":    {Region: \"us-east-2\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.7648},\n\t\t\"r6gd.16xlarge\":    {Region: \"us-east-2\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.6864},\n\t\t\"r6gd.metal\":       {Region: \"us-east-2\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.6864},\n\t\t\"r6i.large\":        {Region: \"us-east-2\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"r6i.xlarge\":       {Region: \"us-east-2\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"r6i.2xlarge\":      {Region: \"us-east-2\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"r6i.4xlarge\":      {Region: \"us-east-2\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"r6i.8xlarge\":      {Region: \"us-east-2\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"r6i.12xlarge\":     {Region: \"us-east-2\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"r6i.16xlarge\":     {Region: \"us-east-2\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r6i.24xlarge\":     {Region: \"us-east-2\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r6i.32xlarge\":     {Region: \"us-east-2\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"r6i.metal\":        {Region: \"us-east-2\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"t2.nano\":          {Region: \"us-east-2\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0058},\n\t\t\"t2.micro\":         {Region: \"us-east-2\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0116},\n\t\t\"t2.small\":         {Region: \"us-east-2\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.023},\n\t\t\"t2.medium\":        {Region: \"us-east-2\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0464},\n\t\t\"t2.large\":         {Region: \"us-east-2\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0928},\n\t\t\"t2.xlarge\":        {Region: \"us-east-2\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1856},\n\t\t\"t2.2xlarge\":       {Region: \"us-east-2\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3712},\n\t\t\"t3.nano\":          {Region: \"us-east-2\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0052},\n\t\t\"t3.micro\":         {Region: \"us-east-2\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0104},\n\t\t\"t3.small\":         {Region: \"us-east-2\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0208},\n\t\t\"t3.medium\":        {Region: \"us-east-2\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0416},\n\t\t\"t3.large\":         {Region: \"us-east-2\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0832},\n\t\t\"t3.xlarge\":        {Region: \"us-east-2\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1664},\n\t\t\"t3.2xlarge\":       {Region: \"us-east-2\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3328},\n\t\t\"t3a.nano\":         {Region: \"us-east-2\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0047},\n\t\t\"t3a.micro\":        {Region: \"us-east-2\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0094},\n\t\t\"t3a.small\":        {Region: \"us-east-2\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0188},\n\t\t\"t3a.medium\":       {Region: \"us-east-2\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0376},\n\t\t\"t3a.large\":        {Region: \"us-east-2\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0752},\n\t\t\"t3a.xlarge\":       {Region: \"us-east-2\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1504},\n\t\t\"t3a.2xlarge\":      {Region: \"us-east-2\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3008},\n\t\t\"t4g.nano\":         {Region: \"us-east-2\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0042},\n\t\t\"t4g.micro\":        {Region: \"us-east-2\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0084},\n\t\t\"t4g.small\":        {Region: \"us-east-2\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0168},\n\t\t\"t4g.medium\":       {Region: \"us-east-2\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0336},\n\t\t\"t4g.large\":        {Region: \"us-east-2\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0672},\n\t\t\"t4g.xlarge\":       {Region: \"us-east-2\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1344},\n\t\t\"t4g.2xlarge\":      {Region: \"us-east-2\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2688},\n\t\t\"u-6tb1.56xlarge\":  {Region: \"us-east-2\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 46.40391},\n\t\t\"u-6tb1.112xlarge\": {Region: \"us-east-2\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 54.6},\n\t\t\"x1.16xlarge\":      {Region: \"us-east-2\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.669},\n\t\t\"x1.32xlarge\":      {Region: \"us-east-2\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 13.338},\n\t\t\"x1e.xlarge\":       {Region: \"us-east-2\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.834},\n\t\t\"x1e.2xlarge\":      {Region: \"us-east-2\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.668},\n\t\t\"x1e.4xlarge\":      {Region: \"us-east-2\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"x1e.8xlarge\":      {Region: \"us-east-2\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.672},\n\t\t\"x1e.16xlarge\":     {Region: \"us-east-2\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.344},\n\t\t\"x1e.32xlarge\":     {Region: \"us-east-2\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 26.688},\n\t\t\"x2gd.medium\":      {Region: \"us-east-2\", Type: \"x2gd.medium\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0835},\n\t\t\"x2gd.large\":       {Region: \"us-east-2\", Type: \"x2gd.large\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"x2gd.xlarge\":      {Region: \"us-east-2\", Type: \"x2gd.xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"x2gd.2xlarge\":     {Region: \"us-east-2\", Type: \"x2gd.2xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"x2gd.4xlarge\":     {Region: \"us-east-2\", Type: \"x2gd.4xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"x2gd.8xlarge\":     {Region: \"us-east-2\", Type: \"x2gd.8xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"x2gd.12xlarge\":    {Region: \"us-east-2\", Type: \"x2gd.12xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"x2gd.16xlarge\":    {Region: \"us-east-2\", Type: \"x2gd.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"x2gd.metal\":       {Region: \"us-east-2\", Type: \"x2gd.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"x2iedn.xlarge\":    {Region: \"us-east-2\", Type: \"x2iedn.xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.83363},\n\t\t\"x2iedn.2xlarge\":   {Region: \"us-east-2\", Type: \"x2iedn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.66725},\n\t\t\"x2iedn.4xlarge\":   {Region: \"us-east-2\", Type: \"x2iedn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.3345},\n\t\t\"x2iedn.8xlarge\":   {Region: \"us-east-2\", Type: \"x2iedn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.669},\n\t\t\"x2iedn.16xlarge\":  {Region: \"us-east-2\", Type: \"x2iedn.16xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.338},\n\t\t\"x2iedn.24xlarge\":  {Region: \"us-east-2\", Type: \"x2iedn.24xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 20.007},\n\t\t\"x2iedn.32xlarge\":  {Region: \"us-east-2\", Type: \"x2iedn.32xlarge\", Memory: kresource.MustParse(\"4194304Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 26.676},\n\t\t\"z1d.large\":        {Region: \"us-east-2\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"z1d.xlarge\":       {Region: \"us-east-2\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"z1d.2xlarge\":      {Region: \"us-east-2\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.744},\n\t\t\"z1d.3xlarge\":      {Region: \"us-east-2\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.116},\n\t\t\"z1d.6xlarge\":      {Region: \"us-east-2\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.232},\n\t\t\"z1d.12xlarge\":     {Region: \"us-east-2\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"z1d.metal\":        {Region: \"us-east-2\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.464},\n\t},\n\t\"us-gov-east-1\": {\n\t\t\"c5.large\":         {Region: \"us-gov-east-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.102},\n\t\t\"c5.xlarge\":        {Region: \"us-gov-east-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.204},\n\t\t\"c5.2xlarge\":       {Region: \"us-gov-east-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"c5.4xlarge\":       {Region: \"us-gov-east-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.816},\n\t\t\"c5.9xlarge\":       {Region: \"us-gov-east-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.836},\n\t\t\"c5.12xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"c5.18xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"c5.24xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"c5.metal\":         {Region: \"us-gov-east-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"c5a.large\":        {Region: \"us-gov-east-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.092},\n\t\t\"c5a.xlarge\":       {Region: \"us-gov-east-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.184},\n\t\t\"c5a.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.368},\n\t\t\"c5a.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.736},\n\t\t\"c5a.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.472},\n\t\t\"c5a.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.208},\n\t\t\"c5a.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.944},\n\t\t\"c5a.24xlarge\":     {Region: \"us-gov-east-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.416},\n\t\t\"c5d.large\":        {Region: \"us-gov-east-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.116},\n\t\t\"c5d.xlarge\":       {Region: \"us-gov-east-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.232},\n\t\t\"c5d.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.464},\n\t\t\"c5d.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.928},\n\t\t\"c5d.9xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.088},\n\t\t\"c5d.18xlarge\":     {Region: \"us-gov-east-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"c5n.large\":        {Region: \"us-gov-east-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"c5n.xlarge\":       {Region: \"us-gov-east-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.26},\n\t\t\"c5n.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"c5n.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.04},\n\t\t\"c5n.9xlarge\":      {Region: \"us-gov-east-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.34},\n\t\t\"c5n.18xlarge\":     {Region: \"us-gov-east-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.68},\n\t\t\"c5n.metal\":        {Region: \"us-gov-east-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.68},\n\t\t\"c6g.medium\":       {Region: \"us-gov-east-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0408},\n\t\t\"c6g.large\":        {Region: \"us-gov-east-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0816},\n\t\t\"c6g.xlarge\":       {Region: \"us-gov-east-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1632},\n\t\t\"c6g.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3264},\n\t\t\"c6g.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6528},\n\t\t\"c6g.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3056},\n\t\t\"c6g.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.9584},\n\t\t\"c6g.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.6112},\n\t\t\"c6g.metal\":        {Region: \"us-gov-east-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.6112},\n\t\t\"d2.xlarge\":        {Region: \"us-gov-east-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.828},\n\t\t\"d2.2xlarge\":       {Region: \"us-gov-east-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.656},\n\t\t\"d2.4xlarge\":       {Region: \"us-gov-east-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.312},\n\t\t\"d2.8xlarge\":       {Region: \"us-gov-east-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.624},\n\t\t\"g4dn.xlarge\":      {Region: \"us-gov-east-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.663},\n\t\t\"g4dn.2xlarge\":     {Region: \"us-gov-east-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.948},\n\t\t\"g4dn.4xlarge\":     {Region: \"us-gov-east-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.518},\n\t\t\"g4dn.8xlarge\":     {Region: \"us-gov-east-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.743},\n\t\t\"g4dn.12xlarge\":    {Region: \"us-gov-east-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.931},\n\t\t\"g4dn.16xlarge\":    {Region: \"us-gov-east-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.486},\n\t\t\"i3.large\":         {Region: \"us-gov-east-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.188},\n\t\t\"i3.xlarge\":        {Region: \"us-gov-east-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.376},\n\t\t\"i3.2xlarge\":       {Region: \"us-gov-east-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.752},\n\t\t\"i3.4xlarge\":       {Region: \"us-gov-east-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.504},\n\t\t\"i3.8xlarge\":       {Region: \"us-gov-east-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.008},\n\t\t\"i3.16xlarge\":      {Region: \"us-gov-east-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.016},\n\t\t\"i3.metal\":         {Region: \"us-gov-east-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.016},\n\t\t\"i3en.large\":       {Region: \"us-gov-east-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.273},\n\t\t\"i3en.xlarge\":      {Region: \"us-gov-east-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.546},\n\t\t\"i3en.2xlarge\":     {Region: \"us-gov-east-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.092},\n\t\t\"i3en.3xlarge\":     {Region: \"us-gov-east-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.638},\n\t\t\"i3en.6xlarge\":     {Region: \"us-gov-east-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.276},\n\t\t\"i3en.12xlarge\":    {Region: \"us-gov-east-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.552},\n\t\t\"i3en.24xlarge\":    {Region: \"us-gov-east-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.104},\n\t\t\"i3en.metal\":       {Region: \"us-gov-east-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.104},\n\t\t\"inf1.xlarge\":      {Region: \"us-gov-east-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.288},\n\t\t\"inf1.2xlarge\":     {Region: \"us-gov-east-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.456},\n\t\t\"inf1.6xlarge\":     {Region: \"us-gov-east-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.488},\n\t\t\"inf1.24xlarge\":    {Region: \"us-gov-east-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.953},\n\t\t\"m5.large\":         {Region: \"us-gov-east-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.121},\n\t\t\"m5.xlarge\":        {Region: \"us-gov-east-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.242},\n\t\t\"m5.2xlarge\":       {Region: \"us-gov-east-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.484},\n\t\t\"m5.4xlarge\":       {Region: \"us-gov-east-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.968},\n\t\t\"m5.8xlarge\":       {Region: \"us-gov-east-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.936},\n\t\t\"m5.12xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.904},\n\t\t\"m5.16xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.872},\n\t\t\"m5.24xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.808},\n\t\t\"m5.metal\":         {Region: \"us-gov-east-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.808},\n\t\t\"m5a.large\":        {Region: \"us-gov-east-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.109},\n\t\t\"m5a.xlarge\":       {Region: \"us-gov-east-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.218},\n\t\t\"m5a.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.436},\n\t\t\"m5a.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.872},\n\t\t\"m5a.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.744},\n\t\t\"m5a.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.616},\n\t\t\"m5a.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.488},\n\t\t\"m5a.24xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.232},\n\t\t\"m5d.large\":        {Region: \"us-gov-east-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.143},\n\t\t\"m5d.xlarge\":       {Region: \"us-gov-east-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.286},\n\t\t\"m5d.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.572},\n\t\t\"m5d.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.144},\n\t\t\"m5d.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.288},\n\t\t\"m5d.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.432},\n\t\t\"m5d.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.576},\n\t\t\"m5d.24xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.864},\n\t\t\"m5d.metal\":        {Region: \"us-gov-east-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.864},\n\t\t\"m5dn.large\":       {Region: \"us-gov-east-1\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.171},\n\t\t\"m5dn.xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.342},\n\t\t\"m5dn.2xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.684},\n\t\t\"m5dn.4xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.368},\n\t\t\"m5dn.8xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.736},\n\t\t\"m5dn.12xlarge\":    {Region: \"us-gov-east-1\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.104},\n\t\t\"m5dn.16xlarge\":    {Region: \"us-gov-east-1\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.472},\n\t\t\"m5dn.24xlarge\":    {Region: \"us-gov-east-1\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.208},\n\t\t\"m5dn.metal\":       {Region: \"us-gov-east-1\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.208},\n\t\t\"m5n.large\":        {Region: \"us-gov-east-1\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"m5n.xlarge\":       {Region: \"us-gov-east-1\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"m5n.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"m5n.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"m5n.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.384},\n\t\t\"m5n.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"m5n.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.768},\n\t\t\"m5n.24xlarge\":     {Region: \"us-gov-east-1\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"m5n.metal\":        {Region: \"us-gov-east-1\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"m6g.medium\":       {Region: \"us-gov-east-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0484},\n\t\t\"m6g.large\":        {Region: \"us-gov-east-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0968},\n\t\t\"m6g.xlarge\":       {Region: \"us-gov-east-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1936},\n\t\t\"m6g.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3872},\n\t\t\"m6g.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7744},\n\t\t\"m6g.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.5488},\n\t\t\"m6g.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.3232},\n\t\t\"m6g.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.0976},\n\t\t\"m6g.metal\":        {Region: \"us-gov-east-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.0976},\n\t\t\"p3dn.24xlarge\":    {Region: \"us-gov-east-1\", Type: \"p3dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 37.454},\n\t\t\"r5.large\":         {Region: \"us-gov-east-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.151},\n\t\t\"r5.xlarge\":        {Region: \"us-gov-east-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.302},\n\t\t\"r5.2xlarge\":       {Region: \"us-gov-east-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.604},\n\t\t\"r5.4xlarge\":       {Region: \"us-gov-east-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.208},\n\t\t\"r5.8xlarge\":       {Region: \"us-gov-east-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.416},\n\t\t\"r5.12xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.624},\n\t\t\"r5.16xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.832},\n\t\t\"r5.24xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r5.metal\":         {Region: \"us-gov-east-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r5a.large\":        {Region: \"us-gov-east-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"r5a.xlarge\":       {Region: \"us-gov-east-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"r5a.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"r5a.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"r5a.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"r5a.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"r5a.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"r5a.24xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"r5d.large\":        {Region: \"us-gov-east-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.173},\n\t\t\"r5d.xlarge\":       {Region: \"us-gov-east-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.346},\n\t\t\"r5d.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.692},\n\t\t\"r5d.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.384},\n\t\t\"r5d.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.768},\n\t\t\"r5d.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.152},\n\t\t\"r5d.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.536},\n\t\t\"r5d.24xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.304},\n\t\t\"r5d.metal\":        {Region: \"us-gov-east-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.304},\n\t\t\"r5dn.large\":       {Region: \"us-gov-east-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.201},\n\t\t\"r5dn.xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.402},\n\t\t\"r5dn.2xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.804},\n\t\t\"r5dn.4xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.608},\n\t\t\"r5dn.8xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.216},\n\t\t\"r5dn.12xlarge\":    {Region: \"us-gov-east-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.824},\n\t\t\"r5dn.16xlarge\":    {Region: \"us-gov-east-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.432},\n\t\t\"r5dn.24xlarge\":    {Region: \"us-gov-east-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.648},\n\t\t\"r5dn.metal\":       {Region: \"us-gov-east-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.648},\n\t\t\"r5n.large\":        {Region: \"us-gov-east-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.179},\n\t\t\"r5n.xlarge\":       {Region: \"us-gov-east-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.358},\n\t\t\"r5n.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.716},\n\t\t\"r5n.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.432},\n\t\t\"r5n.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.864},\n\t\t\"r5n.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.296},\n\t\t\"r5n.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.728},\n\t\t\"r5n.24xlarge\":     {Region: \"us-gov-east-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.592},\n\t\t\"r5n.metal\":        {Region: \"us-gov-east-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.592},\n\t\t\"r6g.medium\":       {Region: \"us-gov-east-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0604},\n\t\t\"r6g.large\":        {Region: \"us-gov-east-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1208},\n\t\t\"r6g.xlarge\":       {Region: \"us-gov-east-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2416},\n\t\t\"r6g.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4832},\n\t\t\"r6g.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9664},\n\t\t\"r6g.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.9328},\n\t\t\"r6g.12xlarge\":     {Region: \"us-gov-east-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.8992},\n\t\t\"r6g.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8656},\n\t\t\"r6g.metal\":        {Region: \"us-gov-east-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8656},\n\t\t\"t3.nano\":          {Region: \"us-gov-east-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0061},\n\t\t\"t3.micro\":         {Region: \"us-gov-east-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0122},\n\t\t\"t3.small\":         {Region: \"us-gov-east-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0244},\n\t\t\"t3.medium\":        {Region: \"us-gov-east-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0488},\n\t\t\"t3.large\":         {Region: \"us-gov-east-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0976},\n\t\t\"t3.xlarge\":        {Region: \"us-gov-east-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1952},\n\t\t\"t3.2xlarge\":       {Region: \"us-gov-east-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3904},\n\t\t\"t3a.nano\":         {Region: \"us-gov-east-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0055},\n\t\t\"t3a.micro\":        {Region: \"us-gov-east-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.011},\n\t\t\"t3a.small\":        {Region: \"us-gov-east-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.022},\n\t\t\"t3a.medium\":       {Region: \"us-gov-east-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0439},\n\t\t\"t3a.large\":        {Region: \"us-gov-east-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0878},\n\t\t\"t3a.xlarge\":       {Region: \"us-gov-east-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1757},\n\t\t\"t3a.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3514},\n\t\t\"t4g.nano\":         {Region: \"us-gov-east-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0049},\n\t\t\"t4g.micro\":        {Region: \"us-gov-east-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0098},\n\t\t\"t4g.small\":        {Region: \"us-gov-east-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0196},\n\t\t\"t4g.medium\":       {Region: \"us-gov-east-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0392},\n\t\t\"t4g.large\":        {Region: \"us-gov-east-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0784},\n\t\t\"t4g.xlarge\":       {Region: \"us-gov-east-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1568},\n\t\t\"t4g.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3136},\n\t\t\"u-6tb1.56xlarge\":  {Region: \"us-gov-east-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 55.61075},\n\t\t\"u-6tb1.112xlarge\": {Region: \"us-gov-east-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 65.433},\n\t\t\"x1.16xlarge\":      {Region: \"us-gov-east-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 8.003},\n\t\t\"x1.32xlarge\":      {Region: \"us-gov-east-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 16.006},\n\t\t\"x1e.xlarge\":       {Region: \"us-gov-east-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"x1e.2xlarge\":      {Region: \"us-gov-east-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"x1e.4xlarge\":      {Region: \"us-gov-east-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.0},\n\t\t\"x1e.8xlarge\":      {Region: \"us-gov-east-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.0},\n\t\t\"x1e.16xlarge\":     {Region: \"us-gov-east-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 16.0},\n\t\t\"x1e.32xlarge\":     {Region: \"us-gov-east-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 32.0},\n\t},\n\t\"us-gov-west-1\": {\n\t\t\"c1.medium\":         {Region: \"us-gov-west-1\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.157},\n\t\t\"c1.xlarge\":         {Region: \"us-gov-west-1\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.628},\n\t\t\"c3.large\":          {Region: \"us-gov-west-1\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"c3.xlarge\":         {Region: \"us-gov-west-1\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"c3.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"c3.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"c3.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"c4.large\":          {Region: \"us-gov-west-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"c4.xlarge\":         {Region: \"us-gov-west-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.239},\n\t\t\"c4.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.479},\n\t\t\"c4.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.958},\n\t\t\"c4.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.915},\n\t\t\"c5.large\":          {Region: \"us-gov-west-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.102},\n\t\t\"c5.xlarge\":         {Region: \"us-gov-west-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.204},\n\t\t\"c5.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"c5.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.816},\n\t\t\"c5.9xlarge\":        {Region: \"us-gov-west-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.836},\n\t\t\"c5.12xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"c5.18xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"c5.24xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"c5.metal\":          {Region: \"us-gov-west-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"c5a.large\":         {Region: \"us-gov-west-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.092},\n\t\t\"c5a.xlarge\":        {Region: \"us-gov-west-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.184},\n\t\t\"c5a.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.368},\n\t\t\"c5a.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.736},\n\t\t\"c5a.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.472},\n\t\t\"c5a.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.208},\n\t\t\"c5a.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.944},\n\t\t\"c5a.24xlarge\":      {Region: \"us-gov-west-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.416},\n\t\t\"c5d.large\":         {Region: \"us-gov-west-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.116},\n\t\t\"c5d.xlarge\":        {Region: \"us-gov-west-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.232},\n\t\t\"c5d.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.464},\n\t\t\"c5d.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.928},\n\t\t\"c5d.9xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.088},\n\t\t\"c5d.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.784},\n\t\t\"c5d.18xlarge\":      {Region: \"us-gov-west-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.176},\n\t\t\"c5d.24xlarge\":      {Region: \"us-gov-west-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.568},\n\t\t\"c5d.metal\":         {Region: \"us-gov-west-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.568},\n\t\t\"c5n.large\":         {Region: \"us-gov-west-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"c5n.xlarge\":        {Region: \"us-gov-west-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.26},\n\t\t\"c5n.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"c5n.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.04},\n\t\t\"c5n.9xlarge\":       {Region: \"us-gov-west-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.34},\n\t\t\"c5n.18xlarge\":      {Region: \"us-gov-west-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.68},\n\t\t\"c5n.metal\":         {Region: \"us-gov-west-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.68},\n\t\t\"c6g.medium\":        {Region: \"us-gov-west-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0408},\n\t\t\"c6g.large\":         {Region: \"us-gov-west-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0816},\n\t\t\"c6g.xlarge\":        {Region: \"us-gov-west-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1632},\n\t\t\"c6g.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3264},\n\t\t\"c6g.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6528},\n\t\t\"c6g.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3056},\n\t\t\"c6g.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.9584},\n\t\t\"c6g.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.6112},\n\t\t\"c6g.metal\":         {Region: \"us-gov-west-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.6112},\n\t\t\"cc2.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"cc2.8xlarge\", Memory: kresource.MustParse(\"61952Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.25},\n\t\t\"d2.xlarge\":         {Region: \"us-gov-west-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.828},\n\t\t\"d2.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.656},\n\t\t\"d2.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.312},\n\t\t\"d2.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.624},\n\t\t\"d3.xlarge\":         {Region: \"us-gov-west-1\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.598},\n\t\t\"d3.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.197},\n\t\t\"d3.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.394},\n\t\t\"d3.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.78776},\n\t\t\"f1.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"f1.2xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.98},\n\t\t\"f1.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"f1.4xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.96},\n\t\t\"f1.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"f1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 15.84},\n\t\t\"g3.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.32},\n\t\t\"g3.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.64},\n\t\t\"g3.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 5.28},\n\t\t\"g3s.xlarge\":        {Region: \"us-gov-west-1\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.868},\n\t\t\"g4dn.xlarge\":       {Region: \"us-gov-west-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.663},\n\t\t\"g4dn.2xlarge\":      {Region: \"us-gov-west-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.948},\n\t\t\"g4dn.4xlarge\":      {Region: \"us-gov-west-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.518},\n\t\t\"g4dn.8xlarge\":      {Region: \"us-gov-west-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.743},\n\t\t\"g4dn.12xlarge\":     {Region: \"us-gov-west-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.931},\n\t\t\"g4dn.16xlarge\":     {Region: \"us-gov-west-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.486},\n\t\t\"g4dn.metal\":        {Region: \"us-gov-west-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 9.862},\n\t\t\"hpc6a.48xlarge\":    {Region: \"us-gov-west-1\", Type: \"hpc6a.48xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.467},\n\t\t\"hs1.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"hs1.8xlarge\", Memory: kresource.MustParse(\"119808Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"i2.xlarge\":         {Region: \"us-gov-west-1\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.023},\n\t\t\"i2.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.046},\n\t\t\"i2.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.092},\n\t\t\"i2.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.184},\n\t\t\"i3.large\":          {Region: \"us-gov-west-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.188},\n\t\t\"i3.xlarge\":         {Region: \"us-gov-west-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.376},\n\t\t\"i3.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.752},\n\t\t\"i3.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.504},\n\t\t\"i3.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.008},\n\t\t\"i3.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.016},\n\t\t\"i3.metal\":          {Region: \"us-gov-west-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.016},\n\t\t\"i3en.large\":        {Region: \"us-gov-west-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.273},\n\t\t\"i3en.xlarge\":       {Region: \"us-gov-west-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.546},\n\t\t\"i3en.2xlarge\":      {Region: \"us-gov-west-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.092},\n\t\t\"i3en.3xlarge\":      {Region: \"us-gov-west-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.638},\n\t\t\"i3en.6xlarge\":      {Region: \"us-gov-west-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.276},\n\t\t\"i3en.12xlarge\":     {Region: \"us-gov-west-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.552},\n\t\t\"i3en.24xlarge\":     {Region: \"us-gov-west-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.104},\n\t\t\"i3en.metal\":        {Region: \"us-gov-west-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.104},\n\t\t\"i3p.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"i3p.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.016},\n\t\t\"inf1.xlarge\":       {Region: \"us-gov-west-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.288},\n\t\t\"inf1.2xlarge\":      {Region: \"us-gov-west-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.456},\n\t\t\"inf1.6xlarge\":      {Region: \"us-gov-west-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.488},\n\t\t\"inf1.24xlarge\":     {Region: \"us-gov-west-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.953},\n\t\t\"m1.small\":          {Region: \"us-gov-west-1\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.053},\n\t\t\"m1.medium\":         {Region: \"us-gov-west-1\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.106},\n\t\t\"m1.large\":          {Region: \"us-gov-west-1\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.211},\n\t\t\"m1.xlarge\":         {Region: \"us-gov-west-1\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.423},\n\t\t\"m2.xlarge\":         {Region: \"us-gov-west-1\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.293},\n\t\t\"m2.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.586},\n\t\t\"m2.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.171},\n\t\t\"m3.medium\":         {Region: \"us-gov-west-1\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.084},\n\t\t\"m3.large\":          {Region: \"us-gov-west-1\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.168},\n\t\t\"m3.xlarge\":         {Region: \"us-gov-west-1\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.336},\n\t\t\"m3.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.672},\n\t\t\"m4.large\":          {Region: \"us-gov-west-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"m4.xlarge\":         {Region: \"us-gov-west-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"m4.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"m4.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"m4.10xlarge\":       {Region: \"us-gov-west-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.52},\n\t\t\"m4.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"m5.large\":          {Region: \"us-gov-west-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.121},\n\t\t\"m5.xlarge\":         {Region: \"us-gov-west-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.242},\n\t\t\"m5.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.484},\n\t\t\"m5.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.968},\n\t\t\"m5.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.936},\n\t\t\"m5.12xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.904},\n\t\t\"m5.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.872},\n\t\t\"m5.24xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.808},\n\t\t\"m5.metal\":          {Region: \"us-gov-west-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.808},\n\t\t\"m5a.large\":         {Region: \"us-gov-west-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.109},\n\t\t\"m5a.xlarge\":        {Region: \"us-gov-west-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.218},\n\t\t\"m5a.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.436},\n\t\t\"m5a.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.872},\n\t\t\"m5a.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.744},\n\t\t\"m5a.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.616},\n\t\t\"m5a.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.488},\n\t\t\"m5a.24xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.232},\n\t\t\"m5ad.large\":        {Region: \"us-gov-west-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.131},\n\t\t\"m5ad.xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.262},\n\t\t\"m5ad.2xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"m5ad.4xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.048},\n\t\t\"m5ad.8xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.096},\n\t\t\"m5ad.12xlarge\":     {Region: \"us-gov-west-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.144},\n\t\t\"m5ad.16xlarge\":     {Region: \"us-gov-west-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.192},\n\t\t\"m5ad.24xlarge\":     {Region: \"us-gov-west-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"m5d.large\":         {Region: \"us-gov-west-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.143},\n\t\t\"m5d.xlarge\":        {Region: \"us-gov-west-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.286},\n\t\t\"m5d.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.572},\n\t\t\"m5d.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.144},\n\t\t\"m5d.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.288},\n\t\t\"m5d.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.432},\n\t\t\"m5d.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.576},\n\t\t\"m5d.24xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.864},\n\t\t\"m5d.metal\":         {Region: \"us-gov-west-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.864},\n\t\t\"m5dn.large\":        {Region: \"us-gov-west-1\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.171},\n\t\t\"m5dn.xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.342},\n\t\t\"m5dn.2xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.684},\n\t\t\"m5dn.4xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.368},\n\t\t\"m5dn.8xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.736},\n\t\t\"m5dn.12xlarge\":     {Region: \"us-gov-west-1\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.104},\n\t\t\"m5dn.16xlarge\":     {Region: \"us-gov-west-1\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.472},\n\t\t\"m5dn.24xlarge\":     {Region: \"us-gov-west-1\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.208},\n\t\t\"m5dn.metal\":        {Region: \"us-gov-west-1\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.208},\n\t\t\"m5n.large\":         {Region: \"us-gov-west-1\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"m5n.xlarge\":        {Region: \"us-gov-west-1\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"m5n.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"m5n.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"m5n.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.384},\n\t\t\"m5n.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"m5n.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.768},\n\t\t\"m5n.24xlarge\":      {Region: \"us-gov-west-1\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"m5n.metal\":         {Region: \"us-gov-west-1\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"m6g.medium\":        {Region: \"us-gov-west-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0484},\n\t\t\"m6g.large\":         {Region: \"us-gov-west-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0968},\n\t\t\"m6g.xlarge\":        {Region: \"us-gov-west-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1936},\n\t\t\"m6g.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3872},\n\t\t\"m6g.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7744},\n\t\t\"m6g.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.5488},\n\t\t\"m6g.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.3232},\n\t\t\"m6g.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.0976},\n\t\t\"m6g.metal\":         {Region: \"us-gov-west-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.0976},\n\t\t\"p2.xlarge\":         {Region: \"us-gov-west-1\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.08},\n\t\t\"p2.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 8.64},\n\t\t\"p2.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 17.28},\n\t\t\"p3.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 3.672},\n\t\t\"p3.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 14.688},\n\t\t\"p3.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 29.376},\n\t\t\"p3dn.24xlarge\":     {Region: \"us-gov-west-1\", Type: \"p3dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 37.454},\n\t\t\"r3.large\":          {Region: \"us-gov-west-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"r3.xlarge\":         {Region: \"us-gov-west-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.399},\n\t\t\"r3.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.798},\n\t\t\"r3.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.596},\n\t\t\"r3.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"r4.large\":          {Region: \"us-gov-west-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1596},\n\t\t\"r4.xlarge\":         {Region: \"us-gov-west-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3192},\n\t\t\"r4.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.6384},\n\t\t\"r4.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.2768},\n\t\t\"r4.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.5536},\n\t\t\"r4.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.1072},\n\t\t\"r5.large\":          {Region: \"us-gov-west-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.151},\n\t\t\"r5.xlarge\":         {Region: \"us-gov-west-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.302},\n\t\t\"r5.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.604},\n\t\t\"r5.4xlarge\":        {Region: \"us-gov-west-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.208},\n\t\t\"r5.8xlarge\":        {Region: \"us-gov-west-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.416},\n\t\t\"r5.12xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.624},\n\t\t\"r5.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.832},\n\t\t\"r5.24xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r5.metal\":          {Region: \"us-gov-west-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.248},\n\t\t\"r5a.large\":         {Region: \"us-gov-west-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"r5a.xlarge\":        {Region: \"us-gov-west-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"r5a.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"r5a.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"r5a.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"r5a.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"r5a.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"r5a.24xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"r5ad.large\":        {Region: \"us-gov-west-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.158},\n\t\t\"r5ad.xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.316},\n\t\t\"r5ad.2xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.632},\n\t\t\"r5ad.4xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.264},\n\t\t\"r5ad.8xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.528},\n\t\t\"r5ad.12xlarge\":     {Region: \"us-gov-west-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.792},\n\t\t\"r5ad.16xlarge\":     {Region: \"us-gov-west-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.056},\n\t\t\"r5ad.24xlarge\":     {Region: \"us-gov-west-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.584},\n\t\t\"r5d.large\":         {Region: \"us-gov-west-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.173},\n\t\t\"r5d.xlarge\":        {Region: \"us-gov-west-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.346},\n\t\t\"r5d.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.692},\n\t\t\"r5d.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.384},\n\t\t\"r5d.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.768},\n\t\t\"r5d.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.152},\n\t\t\"r5d.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.536},\n\t\t\"r5d.24xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.304},\n\t\t\"r5d.metal\":         {Region: \"us-gov-west-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.304},\n\t\t\"r5dn.large\":        {Region: \"us-gov-west-1\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.201},\n\t\t\"r5dn.xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.402},\n\t\t\"r5dn.2xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.804},\n\t\t\"r5dn.4xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.608},\n\t\t\"r5dn.8xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.216},\n\t\t\"r5dn.12xlarge\":     {Region: \"us-gov-west-1\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.824},\n\t\t\"r5dn.16xlarge\":     {Region: \"us-gov-west-1\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.432},\n\t\t\"r5dn.24xlarge\":     {Region: \"us-gov-west-1\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.648},\n\t\t\"r5dn.metal\":        {Region: \"us-gov-west-1\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 9.648},\n\t\t\"r5n.large\":         {Region: \"us-gov-west-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.179},\n\t\t\"r5n.xlarge\":        {Region: \"us-gov-west-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.358},\n\t\t\"r5n.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.716},\n\t\t\"r5n.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.432},\n\t\t\"r5n.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.864},\n\t\t\"r5n.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.296},\n\t\t\"r5n.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.728},\n\t\t\"r5n.24xlarge\":      {Region: \"us-gov-west-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.592},\n\t\t\"r5n.metal\":         {Region: \"us-gov-west-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.592},\n\t\t\"r6g.medium\":        {Region: \"us-gov-west-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0604},\n\t\t\"r6g.large\":         {Region: \"us-gov-west-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1208},\n\t\t\"r6g.xlarge\":        {Region: \"us-gov-west-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2416},\n\t\t\"r6g.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4832},\n\t\t\"r6g.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9664},\n\t\t\"r6g.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.9328},\n\t\t\"r6g.12xlarge\":      {Region: \"us-gov-west-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.8992},\n\t\t\"r6g.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8656},\n\t\t\"r6g.metal\":         {Region: \"us-gov-west-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.8656},\n\t\t\"t1.micro\":          {Region: \"us-gov-west-1\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.024},\n\t\t\"t2.nano\":           {Region: \"us-gov-west-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0068},\n\t\t\"t2.micro\":          {Region: \"us-gov-west-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0136},\n\t\t\"t2.small\":          {Region: \"us-gov-west-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0272},\n\t\t\"t2.medium\":         {Region: \"us-gov-west-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0544},\n\t\t\"t2.large\":          {Region: \"us-gov-west-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1088},\n\t\t\"t2.xlarge\":         {Region: \"us-gov-west-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2176},\n\t\t\"t2.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4352},\n\t\t\"t3.nano\":           {Region: \"us-gov-west-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0061},\n\t\t\"t3.micro\":          {Region: \"us-gov-west-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0122},\n\t\t\"t3.small\":          {Region: \"us-gov-west-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0244},\n\t\t\"t3.medium\":         {Region: \"us-gov-west-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0488},\n\t\t\"t3.large\":          {Region: \"us-gov-west-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0976},\n\t\t\"t3.xlarge\":         {Region: \"us-gov-west-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1952},\n\t\t\"t3.2xlarge\":        {Region: \"us-gov-west-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3904},\n\t\t\"t3a.nano\":          {Region: \"us-gov-west-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0055},\n\t\t\"t3a.micro\":         {Region: \"us-gov-west-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.011},\n\t\t\"t3a.small\":         {Region: \"us-gov-west-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.022},\n\t\t\"t3a.medium\":        {Region: \"us-gov-west-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0439},\n\t\t\"t3a.large\":         {Region: \"us-gov-west-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0878},\n\t\t\"t3a.xlarge\":        {Region: \"us-gov-west-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1757},\n\t\t\"t3a.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3514},\n\t\t\"t4g.nano\":          {Region: \"us-gov-west-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0049},\n\t\t\"t4g.micro\":         {Region: \"us-gov-west-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0098},\n\t\t\"t4g.small\":         {Region: \"us-gov-west-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0196},\n\t\t\"t4g.medium\":        {Region: \"us-gov-west-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0392},\n\t\t\"t4g.large\":         {Region: \"us-gov-west-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0784},\n\t\t\"t4g.xlarge\":        {Region: \"us-gov-west-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1568},\n\t\t\"t4g.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3136},\n\t\t\"u-12tb1.112xlarge\": {Region: \"us-gov-west-1\", Type: \"u-12tb1.112xlarge\", Memory: kresource.MustParse(\"12582912Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 130.867},\n\t\t\"u-6tb1.56xlarge\":   {Region: \"us-gov-west-1\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 55.61075},\n\t\t\"u-6tb1.112xlarge\":  {Region: \"us-gov-west-1\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 65.433},\n\t\t\"u-9tb1.112xlarge\":  {Region: \"us-gov-west-1\", Type: \"u-9tb1.112xlarge\", Memory: kresource.MustParse(\"9437184Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 98.15},\n\t\t\"x1.16xlarge\":       {Region: \"us-gov-west-1\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 8.003},\n\t\t\"x1.32xlarge\":       {Region: \"us-gov-west-1\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 16.006},\n\t\t\"x1e.xlarge\":        {Region: \"us-gov-west-1\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"x1e.2xlarge\":       {Region: \"us-gov-west-1\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"x1e.4xlarge\":       {Region: \"us-gov-west-1\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.0},\n\t\t\"x1e.8xlarge\":       {Region: \"us-gov-west-1\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 8.0},\n\t\t\"x1e.16xlarge\":      {Region: \"us-gov-west-1\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 16.0},\n\t\t\"x1e.32xlarge\":      {Region: \"us-gov-west-1\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 32.0},\n\t},\n\t\"us-west-1\": {\n\t\t\"c1.medium\":     {Region: \"us-west-1\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.148},\n\t\t\"c1.xlarge\":     {Region: \"us-west-1\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"c3.large\":      {Region: \"us-west-1\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"c3.xlarge\":     {Region: \"us-west-1\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.239},\n\t\t\"c3.2xlarge\":    {Region: \"us-west-1\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.478},\n\t\t\"c3.4xlarge\":    {Region: \"us-west-1\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.956},\n\t\t\"c3.8xlarge\":    {Region: \"us-west-1\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.912},\n\t\t\"c4.large\":      {Region: \"us-west-1\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.124},\n\t\t\"c4.xlarge\":     {Region: \"us-west-1\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.249},\n\t\t\"c4.2xlarge\":    {Region: \"us-west-1\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.498},\n\t\t\"c4.4xlarge\":    {Region: \"us-west-1\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.997},\n\t\t\"c4.8xlarge\":    {Region: \"us-west-1\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.993},\n\t\t\"c5.large\":      {Region: \"us-west-1\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.106},\n\t\t\"c5.xlarge\":     {Region: \"us-west-1\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.212},\n\t\t\"c5.2xlarge\":    {Region: \"us-west-1\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.424},\n\t\t\"c5.4xlarge\":    {Region: \"us-west-1\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.848},\n\t\t\"c5.9xlarge\":    {Region: \"us-west-1\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.908},\n\t\t\"c5.12xlarge\":   {Region: \"us-west-1\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"c5.18xlarge\":   {Region: \"us-west-1\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.816},\n\t\t\"c5.24xlarge\":   {Region: \"us-west-1\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"c5.metal\":      {Region: \"us-west-1\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"c5a.large\":     {Region: \"us-west-1\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.095},\n\t\t\"c5a.xlarge\":    {Region: \"us-west-1\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.19},\n\t\t\"c5a.2xlarge\":   {Region: \"us-west-1\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.38},\n\t\t\"c5a.4xlarge\":   {Region: \"us-west-1\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.76},\n\t\t\"c5a.8xlarge\":   {Region: \"us-west-1\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.52},\n\t\t\"c5a.12xlarge\":  {Region: \"us-west-1\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.28},\n\t\t\"c5a.16xlarge\":  {Region: \"us-west-1\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.04},\n\t\t\"c5a.24xlarge\":  {Region: \"us-west-1\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.56},\n\t\t\"c5d.large\":     {Region: \"us-west-1\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.12},\n\t\t\"c5d.xlarge\":    {Region: \"us-west-1\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.24},\n\t\t\"c5d.2xlarge\":   {Region: \"us-west-1\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.48},\n\t\t\"c5d.4xlarge\":   {Region: \"us-west-1\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.96},\n\t\t\"c5d.9xlarge\":   {Region: \"us-west-1\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.16},\n\t\t\"c5d.12xlarge\":  {Region: \"us-west-1\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.88},\n\t\t\"c5d.18xlarge\":  {Region: \"us-west-1\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.32},\n\t\t\"c5d.24xlarge\":  {Region: \"us-west-1\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"c5d.metal\":     {Region: \"us-west-1\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.76},\n\t\t\"c5n.large\":     {Region: \"us-west-1\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.135},\n\t\t\"c5n.xlarge\":    {Region: \"us-west-1\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.27},\n\t\t\"c5n.2xlarge\":   {Region: \"us-west-1\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.54},\n\t\t\"c5n.4xlarge\":   {Region: \"us-west-1\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.08},\n\t\t\"c5n.9xlarge\":   {Region: \"us-west-1\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 2.43},\n\t\t\"c5n.18xlarge\":  {Region: \"us-west-1\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.86},\n\t\t\"c5n.metal\":     {Region: \"us-west-1\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 4.86},\n\t\t\"c6g.medium\":    {Region: \"us-west-1\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0424},\n\t\t\"c6g.large\":     {Region: \"us-west-1\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0848},\n\t\t\"c6g.xlarge\":    {Region: \"us-west-1\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1696},\n\t\t\"c6g.2xlarge\":   {Region: \"us-west-1\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3392},\n\t\t\"c6g.4xlarge\":   {Region: \"us-west-1\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6784},\n\t\t\"c6g.8xlarge\":   {Region: \"us-west-1\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3568},\n\t\t\"c6g.12xlarge\":  {Region: \"us-west-1\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0352},\n\t\t\"c6g.16xlarge\":  {Region: \"us-west-1\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7136},\n\t\t\"c6g.metal\":     {Region: \"us-west-1\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7136},\n\t\t\"c6gd.medium\":   {Region: \"us-west-1\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.048},\n\t\t\"c6gd.large\":    {Region: \"us-west-1\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"c6gd.xlarge\":   {Region: \"us-west-1\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"c6gd.2xlarge\":  {Region: \"us-west-1\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"c6gd.4xlarge\":  {Region: \"us-west-1\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"c6gd.8xlarge\":  {Region: \"us-west-1\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"c6gd.12xlarge\": {Region: \"us-west-1\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c6gd.16xlarge\": {Region: \"us-west-1\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"c6gd.metal\":    {Region: \"us-west-1\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"c6gn.medium\":   {Region: \"us-west-1\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.054},\n\t\t\"c6gn.large\":    {Region: \"us-west-1\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"c6gn.xlarge\":   {Region: \"us-west-1\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"c6gn.2xlarge\":  {Region: \"us-west-1\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"c6gn.4xlarge\":  {Region: \"us-west-1\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"c6gn.8xlarge\":  {Region: \"us-west-1\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"c6gn.12xlarge\": {Region: \"us-west-1\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"c6gn.16xlarge\": {Region: \"us-west-1\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"c6i.large\":     {Region: \"us-west-1\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.106},\n\t\t\"c6i.xlarge\":    {Region: \"us-west-1\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.212},\n\t\t\"c6i.2xlarge\":   {Region: \"us-west-1\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.424},\n\t\t\"c6i.4xlarge\":   {Region: \"us-west-1\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.848},\n\t\t\"c6i.8xlarge\":   {Region: \"us-west-1\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.696},\n\t\t\"c6i.12xlarge\":  {Region: \"us-west-1\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"c6i.16xlarge\":  {Region: \"us-west-1\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.392},\n\t\t\"c6i.24xlarge\":  {Region: \"us-west-1\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.088},\n\t\t\"c6i.32xlarge\":  {Region: \"us-west-1\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.784},\n\t\t\"c6i.metal\":     {Region: \"us-west-1\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.784},\n\t\t\"d2.xlarge\":     {Region: \"us-west-1\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.781},\n\t\t\"d2.2xlarge\":    {Region: \"us-west-1\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.563},\n\t\t\"d2.4xlarge\":    {Region: \"us-west-1\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.125},\n\t\t\"d2.8xlarge\":    {Region: \"us-west-1\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 6.25},\n\t\t\"g2.2xlarge\":    {Region: \"us-west-1\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.702},\n\t\t\"g2.8xlarge\":    {Region: \"us-west-1\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 2.808},\n\t\t\"g3.4xlarge\":    {Region: \"us-west-1\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.534},\n\t\t\"g3.8xlarge\":    {Region: \"us-west-1\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 3.068},\n\t\t\"g3.16xlarge\":   {Region: \"us-west-1\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 6.136},\n\t\t\"g4dn.xlarge\":   {Region: \"us-west-1\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.631},\n\t\t\"g4dn.2xlarge\":  {Region: \"us-west-1\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.902},\n\t\t\"g4dn.4xlarge\":  {Region: \"us-west-1\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.445},\n\t\t\"g4dn.8xlarge\":  {Region: \"us-west-1\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.611},\n\t\t\"g4dn.12xlarge\": {Region: \"us-west-1\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 4.694},\n\t\t\"g4dn.16xlarge\": {Region: \"us-west-1\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 5.222},\n\t\t\"g4dn.metal\":    {Region: \"us-west-1\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 9.389},\n\t\t\"i2.xlarge\":     {Region: \"us-west-1\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.938},\n\t\t\"i2.2xlarge\":    {Region: \"us-west-1\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.876},\n\t\t\"i2.4xlarge\":    {Region: \"us-west-1\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.751},\n\t\t\"i2.8xlarge\":    {Region: \"us-west-1\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 7.502},\n\t\t\"i3.large\":      {Region: \"us-west-1\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"i3.xlarge\":     {Region: \"us-west-1\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"i3.2xlarge\":    {Region: \"us-west-1\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"i3.4xlarge\":    {Region: \"us-west-1\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"i3.8xlarge\":    {Region: \"us-west-1\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"i3.16xlarge\":   {Region: \"us-west-1\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.504},\n\t\t\"i3.metal\":      {Region: \"us-west-1\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.504},\n\t\t\"i3en.large\":    {Region: \"us-west-1\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.25},\n\t\t\"i3en.xlarge\":   {Region: \"us-west-1\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.5},\n\t\t\"i3en.2xlarge\":  {Region: \"us-west-1\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.0},\n\t\t\"i3en.3xlarge\":  {Region: \"us-west-1\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.5},\n\t\t\"i3en.6xlarge\":  {Region: \"us-west-1\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.0},\n\t\t\"i3en.12xlarge\": {Region: \"us-west-1\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.0},\n\t\t\"i3en.24xlarge\": {Region: \"us-west-1\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.0},\n\t\t\"i3en.metal\":    {Region: \"us-west-1\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 12.0},\n\t\t\"inf1.xlarge\":   {Region: \"us-west-1\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.274},\n\t\t\"inf1.2xlarge\":  {Region: \"us-west-1\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.435},\n\t\t\"inf1.6xlarge\":  {Region: \"us-west-1\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.418},\n\t\t\"inf1.24xlarge\": {Region: \"us-west-1\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 5.671},\n\t\t\"m1.small\":      {Region: \"us-west-1\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.047},\n\t\t\"m1.medium\":     {Region: \"us-west-1\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.095},\n\t\t\"m1.large\":      {Region: \"us-west-1\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.19},\n\t\t\"m1.xlarge\":     {Region: \"us-west-1\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.379},\n\t\t\"m2.xlarge\":     {Region: \"us-west-1\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.275},\n\t\t\"m2.2xlarge\":    {Region: \"us-west-1\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.55},\n\t\t\"m2.4xlarge\":    {Region: \"us-west-1\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.1},\n\t\t\"m3.medium\":     {Region: \"us-west-1\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.077},\n\t\t\"m3.large\":      {Region: \"us-west-1\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"m3.xlarge\":     {Region: \"us-west-1\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"m3.2xlarge\":    {Region: \"us-west-1\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"m4.large\":      {Region: \"us-west-1\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.117},\n\t\t\"m4.xlarge\":     {Region: \"us-west-1\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.234},\n\t\t\"m4.2xlarge\":    {Region: \"us-west-1\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.468},\n\t\t\"m4.4xlarge\":    {Region: \"us-west-1\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.936},\n\t\t\"m4.10xlarge\":   {Region: \"us-west-1\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.34},\n\t\t\"m4.16xlarge\":   {Region: \"us-west-1\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.744},\n\t\t\"m5.large\":      {Region: \"us-west-1\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.112},\n\t\t\"m5.xlarge\":     {Region: \"us-west-1\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.224},\n\t\t\"m5.2xlarge\":    {Region: \"us-west-1\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.448},\n\t\t\"m5.4xlarge\":    {Region: \"us-west-1\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.896},\n\t\t\"m5.8xlarge\":    {Region: \"us-west-1\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.792},\n\t\t\"m5.12xlarge\":   {Region: \"us-west-1\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"m5.16xlarge\":   {Region: \"us-west-1\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.584},\n\t\t\"m5.24xlarge\":   {Region: \"us-west-1\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m5.metal\":      {Region: \"us-west-1\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m5a.large\":     {Region: \"us-west-1\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.101},\n\t\t\"m5a.xlarge\":    {Region: \"us-west-1\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.202},\n\t\t\"m5a.2xlarge\":   {Region: \"us-west-1\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.404},\n\t\t\"m5a.4xlarge\":   {Region: \"us-west-1\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.808},\n\t\t\"m5a.8xlarge\":   {Region: \"us-west-1\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.616},\n\t\t\"m5a.12xlarge\":  {Region: \"us-west-1\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.424},\n\t\t\"m5a.16xlarge\":  {Region: \"us-west-1\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.232},\n\t\t\"m5a.24xlarge\":  {Region: \"us-west-1\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.848},\n\t\t\"m5ad.large\":    {Region: \"us-west-1\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.122},\n\t\t\"m5ad.xlarge\":   {Region: \"us-west-1\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.244},\n\t\t\"m5ad.2xlarge\":  {Region: \"us-west-1\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.488},\n\t\t\"m5ad.4xlarge\":  {Region: \"us-west-1\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.976},\n\t\t\"m5ad.8xlarge\":  {Region: \"us-west-1\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.952},\n\t\t\"m5ad.12xlarge\": {Region: \"us-west-1\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.928},\n\t\t\"m5ad.16xlarge\": {Region: \"us-west-1\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.904},\n\t\t\"m5ad.24xlarge\": {Region: \"us-west-1\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.856},\n\t\t\"m5d.large\":     {Region: \"us-west-1\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"m5d.xlarge\":    {Region: \"us-west-1\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"m5d.2xlarge\":   {Region: \"us-west-1\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"m5d.4xlarge\":   {Region: \"us-west-1\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"m5d.8xlarge\":   {Region: \"us-west-1\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"m5d.12xlarge\":  {Region: \"us-west-1\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.192},\n\t\t\"m5d.16xlarge\":  {Region: \"us-west-1\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"m5d.24xlarge\":  {Region: \"us-west-1\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"m5d.metal\":     {Region: \"us-west-1\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.384},\n\t\t\"m5zn.large\":    {Region: \"us-west-1\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1927},\n\t\t\"m5zn.xlarge\":   {Region: \"us-west-1\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3854},\n\t\t\"m5zn.2xlarge\":  {Region: \"us-west-1\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.7708},\n\t\t\"m5zn.3xlarge\":  {Region: \"us-west-1\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.1562},\n\t\t\"m5zn.6xlarge\":  {Region: \"us-west-1\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.3124},\n\t\t\"m5zn.12xlarge\": {Region: \"us-west-1\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.6248},\n\t\t\"m5zn.metal\":    {Region: \"us-west-1\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.6248},\n\t\t\"m6g.medium\":    {Region: \"us-west-1\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0448},\n\t\t\"m6g.large\":     {Region: \"us-west-1\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0896},\n\t\t\"m6g.xlarge\":    {Region: \"us-west-1\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1792},\n\t\t\"m6g.2xlarge\":   {Region: \"us-west-1\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3584},\n\t\t\"m6g.4xlarge\":   {Region: \"us-west-1\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7168},\n\t\t\"m6g.8xlarge\":   {Region: \"us-west-1\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.4336},\n\t\t\"m6g.12xlarge\":  {Region: \"us-west-1\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.1504},\n\t\t\"m6g.16xlarge\":  {Region: \"us-west-1\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8672},\n\t\t\"m6g.metal\":     {Region: \"us-west-1\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8672},\n\t\t\"m6gd.medium\":   {Region: \"us-west-1\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.053},\n\t\t\"m6gd.large\":    {Region: \"us-west-1\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.106},\n\t\t\"m6gd.xlarge\":   {Region: \"us-west-1\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.212},\n\t\t\"m6gd.2xlarge\":  {Region: \"us-west-1\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.424},\n\t\t\"m6gd.4xlarge\":  {Region: \"us-west-1\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.848},\n\t\t\"m6gd.8xlarge\":  {Region: \"us-west-1\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.696},\n\t\t\"m6gd.12xlarge\": {Region: \"us-west-1\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.544},\n\t\t\"m6gd.16xlarge\": {Region: \"us-west-1\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.392},\n\t\t\"m6gd.metal\":    {Region: \"us-west-1\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.392},\n\t\t\"m6i.large\":     {Region: \"us-west-1\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.112},\n\t\t\"m6i.xlarge\":    {Region: \"us-west-1\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.224},\n\t\t\"m6i.2xlarge\":   {Region: \"us-west-1\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.448},\n\t\t\"m6i.4xlarge\":   {Region: \"us-west-1\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.896},\n\t\t\"m6i.8xlarge\":   {Region: \"us-west-1\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.792},\n\t\t\"m6i.12xlarge\":  {Region: \"us-west-1\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"m6i.16xlarge\":  {Region: \"us-west-1\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.584},\n\t\t\"m6i.24xlarge\":  {Region: \"us-west-1\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.376},\n\t\t\"m6i.32xlarge\":  {Region: \"us-west-1\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.168},\n\t\t\"m6i.metal\":     {Region: \"us-west-1\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 7.168},\n\t\t\"r3.large\":      {Region: \"us-west-1\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.185},\n\t\t\"r3.xlarge\":     {Region: \"us-west-1\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.371},\n\t\t\"r3.2xlarge\":    {Region: \"us-west-1\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.741},\n\t\t\"r3.4xlarge\":    {Region: \"us-west-1\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.482},\n\t\t\"r3.8xlarge\":    {Region: \"us-west-1\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.964},\n\t\t\"r4.large\":      {Region: \"us-west-1\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1482},\n\t\t\"r4.xlarge\":     {Region: \"us-west-1\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2964},\n\t\t\"r4.2xlarge\":    {Region: \"us-west-1\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.5928},\n\t\t\"r4.4xlarge\":    {Region: \"us-west-1\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.1856},\n\t\t\"r4.8xlarge\":    {Region: \"us-west-1\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.3712},\n\t\t\"r4.16xlarge\":   {Region: \"us-west-1\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.7424},\n\t\t\"r5.large\":      {Region: \"us-west-1\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.14},\n\t\t\"r5.xlarge\":     {Region: \"us-west-1\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.28},\n\t\t\"r5.2xlarge\":    {Region: \"us-west-1\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.56},\n\t\t\"r5.4xlarge\":    {Region: \"us-west-1\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.12},\n\t\t\"r5.8xlarge\":    {Region: \"us-west-1\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.24},\n\t\t\"r5.12xlarge\":   {Region: \"us-west-1\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.36},\n\t\t\"r5.16xlarge\":   {Region: \"us-west-1\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.48},\n\t\t\"r5.24xlarge\":   {Region: \"us-west-1\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.72},\n\t\t\"r5.metal\":      {Region: \"us-west-1\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.72},\n\t\t\"r5a.large\":     {Region: \"us-west-1\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"r5a.xlarge\":    {Region: \"us-west-1\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"r5a.2xlarge\":   {Region: \"us-west-1\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"r5a.4xlarge\":   {Region: \"us-west-1\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"r5a.8xlarge\":   {Region: \"us-west-1\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"r5a.12xlarge\":  {Region: \"us-west-1\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"r5a.16xlarge\":  {Region: \"us-west-1\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r5a.24xlarge\":  {Region: \"us-west-1\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r5ad.large\":    {Region: \"us-west-1\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.148},\n\t\t\"r5ad.xlarge\":   {Region: \"us-west-1\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.296},\n\t\t\"r5ad.2xlarge\":  {Region: \"us-west-1\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.592},\n\t\t\"r5ad.4xlarge\":  {Region: \"us-west-1\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.184},\n\t\t\"r5ad.8xlarge\":  {Region: \"us-west-1\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.368},\n\t\t\"r5ad.12xlarge\": {Region: \"us-west-1\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.552},\n\t\t\"r5ad.16xlarge\": {Region: \"us-west-1\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.736},\n\t\t\"r5ad.24xlarge\": {Region: \"us-west-1\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.104},\n\t\t\"r5d.large\":     {Region: \"us-west-1\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.162},\n\t\t\"r5d.xlarge\":    {Region: \"us-west-1\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.324},\n\t\t\"r5d.2xlarge\":   {Region: \"us-west-1\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.648},\n\t\t\"r5d.4xlarge\":   {Region: \"us-west-1\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.296},\n\t\t\"r5d.8xlarge\":   {Region: \"us-west-1\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.592},\n\t\t\"r5d.12xlarge\":  {Region: \"us-west-1\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"r5d.16xlarge\":  {Region: \"us-west-1\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.184},\n\t\t\"r5d.24xlarge\":  {Region: \"us-west-1\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.776},\n\t\t\"r5d.metal\":     {Region: \"us-west-1\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.776},\n\t\t\"r5n.large\":     {Region: \"us-west-1\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.169},\n\t\t\"r5n.xlarge\":    {Region: \"us-west-1\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.338},\n\t\t\"r5n.2xlarge\":   {Region: \"us-west-1\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.676},\n\t\t\"r5n.4xlarge\":   {Region: \"us-west-1\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.352},\n\t\t\"r5n.8xlarge\":   {Region: \"us-west-1\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.704},\n\t\t\"r5n.12xlarge\":  {Region: \"us-west-1\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.056},\n\t\t\"r5n.16xlarge\":  {Region: \"us-west-1\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.408},\n\t\t\"r5n.24xlarge\":  {Region: \"us-west-1\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.112},\n\t\t\"r5n.metal\":     {Region: \"us-west-1\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.112},\n\t\t\"r6g.medium\":    {Region: \"us-west-1\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.056},\n\t\t\"r6g.large\":     {Region: \"us-west-1\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.112},\n\t\t\"r6g.xlarge\":    {Region: \"us-west-1\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.224},\n\t\t\"r6g.2xlarge\":   {Region: \"us-west-1\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.448},\n\t\t\"r6g.4xlarge\":   {Region: \"us-west-1\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.896},\n\t\t\"r6g.8xlarge\":   {Region: \"us-west-1\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.792},\n\t\t\"r6g.12xlarge\":  {Region: \"us-west-1\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.688},\n\t\t\"r6g.16xlarge\":  {Region: \"us-west-1\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.584},\n\t\t\"r6g.metal\":     {Region: \"us-west-1\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.584},\n\t\t\"r6gd.medium\":   {Region: \"us-west-1\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.065},\n\t\t\"r6gd.large\":    {Region: \"us-west-1\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"r6gd.xlarge\":   {Region: \"us-west-1\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.26},\n\t\t\"r6gd.2xlarge\":  {Region: \"us-west-1\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"r6gd.4xlarge\":  {Region: \"us-west-1\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.04},\n\t\t\"r6gd.8xlarge\":  {Region: \"us-west-1\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.08},\n\t\t\"r6gd.12xlarge\": {Region: \"us-west-1\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.12},\n\t\t\"r6gd.16xlarge\": {Region: \"us-west-1\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.16},\n\t\t\"r6gd.metal\":    {Region: \"us-west-1\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.16},\n\t\t\"r6i.large\":     {Region: \"us-west-1\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.14},\n\t\t\"r6i.xlarge\":    {Region: \"us-west-1\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.28},\n\t\t\"r6i.2xlarge\":   {Region: \"us-west-1\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.56},\n\t\t\"r6i.4xlarge\":   {Region: \"us-west-1\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.12},\n\t\t\"r6i.8xlarge\":   {Region: \"us-west-1\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.24},\n\t\t\"r6i.12xlarge\":  {Region: \"us-west-1\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.36},\n\t\t\"r6i.16xlarge\":  {Region: \"us-west-1\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.48},\n\t\t\"r6i.24xlarge\":  {Region: \"us-west-1\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.72},\n\t\t\"r6i.32xlarge\":  {Region: \"us-west-1\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.96},\n\t\t\"r6i.metal\":     {Region: \"us-west-1\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.96},\n\t\t\"t1.micro\":      {Region: \"us-west-1\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.025},\n\t\t\"t2.nano\":       {Region: \"us-west-1\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0069},\n\t\t\"t2.micro\":      {Region: \"us-west-1\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0138},\n\t\t\"t2.small\":      {Region: \"us-west-1\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0276},\n\t\t\"t2.medium\":     {Region: \"us-west-1\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0552},\n\t\t\"t2.large\":      {Region: \"us-west-1\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1104},\n\t\t\"t2.xlarge\":     {Region: \"us-west-1\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2208},\n\t\t\"t2.2xlarge\":    {Region: \"us-west-1\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4416},\n\t\t\"t3.nano\":       {Region: \"us-west-1\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0062},\n\t\t\"t3.micro\":      {Region: \"us-west-1\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0124},\n\t\t\"t3.small\":      {Region: \"us-west-1\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0248},\n\t\t\"t3.medium\":     {Region: \"us-west-1\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0496},\n\t\t\"t3.large\":      {Region: \"us-west-1\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0992},\n\t\t\"t3.xlarge\":     {Region: \"us-west-1\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1984},\n\t\t\"t3.2xlarge\":    {Region: \"us-west-1\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3968},\n\t\t\"t3a.nano\":      {Region: \"us-west-1\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0056},\n\t\t\"t3a.micro\":     {Region: \"us-west-1\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0112},\n\t\t\"t3a.small\":     {Region: \"us-west-1\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0223},\n\t\t\"t3a.medium\":    {Region: \"us-west-1\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0446},\n\t\t\"t3a.large\":     {Region: \"us-west-1\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0893},\n\t\t\"t3a.xlarge\":    {Region: \"us-west-1\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1786},\n\t\t\"t3a.2xlarge\":   {Region: \"us-west-1\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3571},\n\t\t\"t4g.nano\":      {Region: \"us-west-1\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.005},\n\t\t\"t4g.micro\":     {Region: \"us-west-1\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.01},\n\t\t\"t4g.small\":     {Region: \"us-west-1\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.02},\n\t\t\"t4g.medium\":    {Region: \"us-west-1\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.04},\n\t\t\"t4g.large\":     {Region: \"us-west-1\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.08},\n\t\t\"t4g.xlarge\":    {Region: \"us-west-1\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.16},\n\t\t\"t4g.2xlarge\":   {Region: \"us-west-1\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.32},\n\t\t\"z1d.large\":     {Region: \"us-west-1\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.211},\n\t\t\"z1d.xlarge\":    {Region: \"us-west-1\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.422},\n\t\t\"z1d.2xlarge\":   {Region: \"us-west-1\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.844},\n\t\t\"z1d.3xlarge\":   {Region: \"us-west-1\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.266},\n\t\t\"z1d.6xlarge\":   {Region: \"us-west-1\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.532},\n\t\t\"z1d.12xlarge\":  {Region: \"us-west-1\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.064},\n\t\t\"z1d.metal\":     {Region: \"us-west-1\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.064},\n\t},\n\t\"us-west-2\": {\n\t\t\"a1.medium\":         {Region: \"us-west-2\", Type: \"a1.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0255},\n\t\t\"a1.large\":          {Region: \"us-west-2\", Type: \"a1.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.051},\n\t\t\"a1.xlarge\":         {Region: \"us-west-2\", Type: \"a1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.102},\n\t\t\"a1.2xlarge\":        {Region: \"us-west-2\", Type: \"a1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.204},\n\t\t\"a1.4xlarge\":        {Region: \"us-west-2\", Type: \"a1.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"a1.metal\":          {Region: \"us-west-2\", Type: \"a1.metal\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.408},\n\t\t\"c1.medium\":         {Region: \"us-west-2\", Type: \"c1.medium\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.13},\n\t\t\"c1.xlarge\":         {Region: \"us-west-2\", Type: \"c1.xlarge\", Memory: kresource.MustParse(\"7168Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.52},\n\t\t\"c3.large\":          {Region: \"us-west-2\", Type: \"c3.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.105},\n\t\t\"c3.xlarge\":         {Region: \"us-west-2\", Type: \"c3.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.21},\n\t\t\"c3.2xlarge\":        {Region: \"us-west-2\", Type: \"c3.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.42},\n\t\t\"c3.4xlarge\":        {Region: \"us-west-2\", Type: \"c3.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.84},\n\t\t\"c3.8xlarge\":        {Region: \"us-west-2\", Type: \"c3.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.68},\n\t\t\"c4.large\":          {Region: \"us-west-2\", Type: \"c4.large\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"c4.xlarge\":         {Region: \"us-west-2\", Type: \"c4.xlarge\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.199},\n\t\t\"c4.2xlarge\":        {Region: \"us-west-2\", Type: \"c4.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.398},\n\t\t\"c4.4xlarge\":        {Region: \"us-west-2\", Type: \"c4.4xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.796},\n\t\t\"c4.8xlarge\":        {Region: \"us-west-2\", Type: \"c4.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.591},\n\t\t\"c5.large\":          {Region: \"us-west-2\", Type: \"c5.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c5.xlarge\":         {Region: \"us-west-2\", Type: \"c5.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c5.2xlarge\":        {Region: \"us-west-2\", Type: \"c5.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c5.4xlarge\":        {Region: \"us-west-2\", Type: \"c5.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c5.9xlarge\":        {Region: \"us-west-2\", Type: \"c5.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.53},\n\t\t\"c5.12xlarge\":       {Region: \"us-west-2\", Type: \"c5.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c5.18xlarge\":       {Region: \"us-west-2\", Type: \"c5.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.06},\n\t\t\"c5.24xlarge\":       {Region: \"us-west-2\", Type: \"c5.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c5.metal\":          {Region: \"us-west-2\", Type: \"c5.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c5a.large\":         {Region: \"us-west-2\", Type: \"c5a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.077},\n\t\t\"c5a.xlarge\":        {Region: \"us-west-2\", Type: \"c5a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"c5a.2xlarge\":       {Region: \"us-west-2\", Type: \"c5a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"c5a.4xlarge\":       {Region: \"us-west-2\", Type: \"c5a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"c5a.8xlarge\":       {Region: \"us-west-2\", Type: \"c5a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.232},\n\t\t\"c5a.12xlarge\":      {Region: \"us-west-2\", Type: \"c5a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.848},\n\t\t\"c5a.16xlarge\":      {Region: \"us-west-2\", Type: \"c5a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"c5a.24xlarge\":      {Region: \"us-west-2\", Type: \"c5a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.696},\n\t\t\"c5ad.large\":        {Region: \"us-west-2\", Type: \"c5ad.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"c5ad.xlarge\":       {Region: \"us-west-2\", Type: \"c5ad.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"c5ad.2xlarge\":      {Region: \"us-west-2\", Type: \"c5ad.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"c5ad.4xlarge\":      {Region: \"us-west-2\", Type: \"c5ad.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"c5ad.8xlarge\":      {Region: \"us-west-2\", Type: \"c5ad.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"c5ad.12xlarge\":     {Region: \"us-west-2\", Type: \"c5ad.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"c5ad.16xlarge\":     {Region: \"us-west-2\", Type: \"c5ad.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"c5ad.24xlarge\":     {Region: \"us-west-2\", Type: \"c5ad.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"c5d.large\":         {Region: \"us-west-2\", Type: \"c5d.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"c5d.xlarge\":        {Region: \"us-west-2\", Type: \"c5d.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"c5d.2xlarge\":       {Region: \"us-west-2\", Type: \"c5d.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"c5d.4xlarge\":       {Region: \"us-west-2\", Type: \"c5d.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"c5d.9xlarge\":       {Region: \"us-west-2\", Type: \"c5d.9xlarge\", Memory: kresource.MustParse(\"73728Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.728},\n\t\t\"c5d.12xlarge\":      {Region: \"us-west-2\", Type: \"c5d.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"c5d.18xlarge\":      {Region: \"us-west-2\", Type: \"c5d.18xlarge\", Memory: kresource.MustParse(\"147456Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"c5d.24xlarge\":      {Region: \"us-west-2\", Type: \"c5d.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5d.metal\":         {Region: \"us-west-2\", Type: \"c5d.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"c5n.large\":         {Region: \"us-west-2\", Type: \"c5n.large\", Memory: kresource.MustParse(\"5376Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.108},\n\t\t\"c5n.xlarge\":        {Region: \"us-west-2\", Type: \"c5n.xlarge\", Memory: kresource.MustParse(\"10752Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.216},\n\t\t\"c5n.2xlarge\":       {Region: \"us-west-2\", Type: \"c5n.2xlarge\", Memory: kresource.MustParse(\"21504Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.432},\n\t\t\"c5n.4xlarge\":       {Region: \"us-west-2\", Type: \"c5n.4xlarge\", Memory: kresource.MustParse(\"43008Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.864},\n\t\t\"c5n.9xlarge\":       {Region: \"us-west-2\", Type: \"c5n.9xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 1.944},\n\t\t\"c5n.18xlarge\":      {Region: \"us-west-2\", Type: \"c5n.18xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c5n.metal\":         {Region: \"us-west-2\", Type: \"c5n.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"72\"), GPU: 0, Inf: 0, Price: 3.888},\n\t\t\"c6a.large\":         {Region: \"us-west-2\", Type: \"c6a.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0765},\n\t\t\"c6a.xlarge\":        {Region: \"us-west-2\", Type: \"c6a.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.153},\n\t\t\"c6a.2xlarge\":       {Region: \"us-west-2\", Type: \"c6a.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.306},\n\t\t\"c6a.4xlarge\":       {Region: \"us-west-2\", Type: \"c6a.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.612},\n\t\t\"c6a.8xlarge\":       {Region: \"us-west-2\", Type: \"c6a.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.224},\n\t\t\"c6a.12xlarge\":      {Region: \"us-west-2\", Type: \"c6a.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.836},\n\t\t\"c6a.16xlarge\":      {Region: \"us-west-2\", Type: \"c6a.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.448},\n\t\t\"c6a.24xlarge\":      {Region: \"us-west-2\", Type: \"c6a.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 3.672},\n\t\t\"c6a.32xlarge\":      {Region: \"us-west-2\", Type: \"c6a.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 4.896},\n\t\t\"c6a.48xlarge\":      {Region: \"us-west-2\", Type: \"c6a.48xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"c6a.metal\":         {Region: \"us-west-2\", Type: \"c6a.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 7.344},\n\t\t\"c6g.medium\":        {Region: \"us-west-2\", Type: \"c6g.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.034},\n\t\t\"c6g.large\":         {Region: \"us-west-2\", Type: \"c6g.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.068},\n\t\t\"c6g.xlarge\":        {Region: \"us-west-2\", Type: \"c6g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"c6g.2xlarge\":       {Region: \"us-west-2\", Type: \"c6g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"c6g.4xlarge\":       {Region: \"us-west-2\", Type: \"c6g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"c6g.8xlarge\":       {Region: \"us-west-2\", Type: \"c6g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"c6g.12xlarge\":      {Region: \"us-west-2\", Type: \"c6g.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.632},\n\t\t\"c6g.16xlarge\":      {Region: \"us-west-2\", Type: \"c6g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"c6g.metal\":         {Region: \"us-west-2\", Type: \"c6g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"c6gd.medium\":       {Region: \"us-west-2\", Type: \"c6gd.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0384},\n\t\t\"c6gd.large\":        {Region: \"us-west-2\", Type: \"c6gd.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0768},\n\t\t\"c6gd.xlarge\":       {Region: \"us-west-2\", Type: \"c6gd.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1536},\n\t\t\"c6gd.2xlarge\":      {Region: \"us-west-2\", Type: \"c6gd.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3072},\n\t\t\"c6gd.4xlarge\":      {Region: \"us-west-2\", Type: \"c6gd.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6144},\n\t\t\"c6gd.8xlarge\":      {Region: \"us-west-2\", Type: \"c6gd.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.2288},\n\t\t\"c6gd.12xlarge\":     {Region: \"us-west-2\", Type: \"c6gd.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.8432},\n\t\t\"c6gd.16xlarge\":     {Region: \"us-west-2\", Type: \"c6gd.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.4576},\n\t\t\"c6gd.metal\":        {Region: \"us-west-2\", Type: \"c6gd.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.4576},\n\t\t\"c6gn.medium\":       {Region: \"us-west-2\", Type: \"c6gn.medium\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0432},\n\t\t\"c6gn.large\":        {Region: \"us-west-2\", Type: \"c6gn.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0864},\n\t\t\"c6gn.xlarge\":       {Region: \"us-west-2\", Type: \"c6gn.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1728},\n\t\t\"c6gn.2xlarge\":      {Region: \"us-west-2\", Type: \"c6gn.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3456},\n\t\t\"c6gn.4xlarge\":      {Region: \"us-west-2\", Type: \"c6gn.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6912},\n\t\t\"c6gn.8xlarge\":      {Region: \"us-west-2\", Type: \"c6gn.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3824},\n\t\t\"c6gn.12xlarge\":     {Region: \"us-west-2\", Type: \"c6gn.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0736},\n\t\t\"c6gn.16xlarge\":     {Region: \"us-west-2\", Type: \"c6gn.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7648},\n\t\t\"c6i.large\":         {Region: \"us-west-2\", Type: \"c6i.large\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.085},\n\t\t\"c6i.xlarge\":        {Region: \"us-west-2\", Type: \"c6i.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.17},\n\t\t\"c6i.2xlarge\":       {Region: \"us-west-2\", Type: \"c6i.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.34},\n\t\t\"c6i.4xlarge\":       {Region: \"us-west-2\", Type: \"c6i.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.68},\n\t\t\"c6i.8xlarge\":       {Region: \"us-west-2\", Type: \"c6i.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.36},\n\t\t\"c6i.12xlarge\":      {Region: \"us-west-2\", Type: \"c6i.12xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.04},\n\t\t\"c6i.16xlarge\":      {Region: \"us-west-2\", Type: \"c6i.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.72},\n\t\t\"c6i.24xlarge\":      {Region: \"us-west-2\", Type: \"c6i.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.08},\n\t\t\"c6i.32xlarge\":      {Region: \"us-west-2\", Type: \"c6i.32xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.44},\n\t\t\"c6i.metal\":         {Region: \"us-west-2\", Type: \"c6i.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.44},\n\t\t\"cc2.8xlarge\":       {Region: \"us-west-2\", Type: \"cc2.8xlarge\", Memory: kresource.MustParse(\"61952Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"cr1.8xlarge\":       {Region: \"us-west-2\", Type: \"cr1.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.5},\n\t\t\"d2.xlarge\":         {Region: \"us-west-2\", Type: \"d2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.69},\n\t\t\"d2.2xlarge\":        {Region: \"us-west-2\", Type: \"d2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.38},\n\t\t\"d2.4xlarge\":        {Region: \"us-west-2\", Type: \"d2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.76},\n\t\t\"d2.8xlarge\":        {Region: \"us-west-2\", Type: \"d2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"36\"), GPU: 0, Inf: 0, Price: 5.52},\n\t\t\"d3.xlarge\":         {Region: \"us-west-2\", Type: \"d3.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.499},\n\t\t\"d3.2xlarge\":        {Region: \"us-west-2\", Type: \"d3.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.999},\n\t\t\"d3.4xlarge\":        {Region: \"us-west-2\", Type: \"d3.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.998},\n\t\t\"d3.8xlarge\":        {Region: \"us-west-2\", Type: \"d3.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 3.99552},\n\t\t\"d3en.xlarge\":       {Region: \"us-west-2\", Type: \"d3en.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.526},\n\t\t\"d3en.2xlarge\":      {Region: \"us-west-2\", Type: \"d3en.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.051},\n\t\t\"d3en.4xlarge\":      {Region: \"us-west-2\", Type: \"d3en.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.103},\n\t\t\"d3en.6xlarge\":      {Region: \"us-west-2\", Type: \"d3en.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 3.154},\n\t\t\"d3en.8xlarge\":      {Region: \"us-west-2\", Type: \"d3en.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.20576},\n\t\t\"d3en.12xlarge\":     {Region: \"us-west-2\", Type: \"d3en.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 6.30864},\n\t\t\"dl1.24xlarge\":      {Region: \"us-west-2\", Type: \"dl1.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 13.10904},\n\t\t\"f1.2xlarge\":        {Region: \"us-west-2\", Type: \"f1.2xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.65},\n\t\t\"f1.4xlarge\":        {Region: \"us-west-2\", Type: \"f1.4xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.3},\n\t\t\"f1.16xlarge\":       {Region: \"us-west-2\", Type: \"f1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.2},\n\t\t\"g2.2xlarge\":        {Region: \"us-west-2\", Type: \"g2.2xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.65},\n\t\t\"g2.8xlarge\":        {Region: \"us-west-2\", Type: \"g2.8xlarge\", Memory: kresource.MustParse(\"61440Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 2.6},\n\t\t\"g3.4xlarge\":        {Region: \"us-west-2\", Type: \"g3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.14},\n\t\t\"g3.8xlarge\":        {Region: \"us-west-2\", Type: \"g3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 2.28},\n\t\t\"g3.16xlarge\":       {Region: \"us-west-2\", Type: \"g3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 4.56},\n\t\t\"g3s.xlarge\":        {Region: \"us-west-2\", Type: \"g3s.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.75},\n\t\t\"g4ad.xlarge\":       {Region: \"us-west-2\", Type: \"g4ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.37853},\n\t\t\"g4ad.2xlarge\":      {Region: \"us-west-2\", Type: \"g4ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.54117},\n\t\t\"g4ad.4xlarge\":      {Region: \"us-west-2\", Type: \"g4ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 0.867},\n\t\t\"g4ad.8xlarge\":      {Region: \"us-west-2\", Type: \"g4ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 2, Inf: 0, Price: 1.734},\n\t\t\"g4ad.16xlarge\":     {Region: \"us-west-2\", Type: \"g4ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 4, Inf: 0, Price: 3.468},\n\t\t\"g4dn.xlarge\":       {Region: \"us-west-2\", Type: \"g4dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.526},\n\t\t\"g4dn.2xlarge\":      {Region: \"us-west-2\", Type: \"g4dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.752},\n\t\t\"g4dn.4xlarge\":      {Region: \"us-west-2\", Type: \"g4dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.204},\n\t\t\"g4dn.8xlarge\":      {Region: \"us-west-2\", Type: \"g4dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.176},\n\t\t\"g4dn.12xlarge\":     {Region: \"us-west-2\", Type: \"g4dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 3.912},\n\t\t\"g4dn.16xlarge\":     {Region: \"us-west-2\", Type: \"g4dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.352},\n\t\t\"g4dn.metal\":        {Region: \"us-west-2\", Type: \"g4dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 7.824},\n\t\t\"g5.xlarge\":         {Region: \"us-west-2\", Type: \"g5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 1.006},\n\t\t\"g5.2xlarge\":        {Region: \"us-west-2\", Type: \"g5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 1.212},\n\t\t\"g5.4xlarge\":        {Region: \"us-west-2\", Type: \"g5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 1.624},\n\t\t\"g5.8xlarge\":        {Region: \"us-west-2\", Type: \"g5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 2.448},\n\t\t\"g5.12xlarge\":       {Region: \"us-west-2\", Type: \"g5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 4, Inf: 0, Price: 5.672},\n\t\t\"g5.16xlarge\":       {Region: \"us-west-2\", Type: \"g5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 1, Inf: 0, Price: 4.096},\n\t\t\"g5.24xlarge\":       {Region: \"us-west-2\", Type: \"g5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 4, Inf: 0, Price: 8.144},\n\t\t\"g5.48xlarge\":       {Region: \"us-west-2\", Type: \"g5.48xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 8, Inf: 0, Price: 16.288},\n\t\t\"g5g.xlarge\":        {Region: \"us-west-2\", Type: \"g5g.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.42},\n\t\t\"g5g.2xlarge\":       {Region: \"us-west-2\", Type: \"g5g.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 0.556},\n\t\t\"g5g.4xlarge\":       {Region: \"us-west-2\", Type: \"g5g.4xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 1, Inf: 0, Price: 0.828},\n\t\t\"g5g.8xlarge\":       {Region: \"us-west-2\", Type: \"g5g.8xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 1, Inf: 0, Price: 1.372},\n\t\t\"g5g.16xlarge\":      {Region: \"us-west-2\", Type: \"g5g.16xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 2.744},\n\t\t\"g5g.metal\":         {Region: \"us-west-2\", Type: \"g5g.metal\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 2, Inf: 0, Price: 2.744},\n\t\t\"h1.2xlarge\":        {Region: \"us-west-2\", Type: \"h1.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.468},\n\t\t\"h1.4xlarge\":        {Region: \"us-west-2\", Type: \"h1.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.936},\n\t\t\"h1.8xlarge\":        {Region: \"us-west-2\", Type: \"h1.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.872},\n\t\t\"h1.16xlarge\":       {Region: \"us-west-2\", Type: \"h1.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.744},\n\t\t\"hs1.8xlarge\":       {Region: \"us-west-2\", Type: \"hs1.8xlarge\", Memory: kresource.MustParse(\"119808Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 4.6},\n\t\t\"i2.xlarge\":         {Region: \"us-west-2\", Type: \"i2.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.853},\n\t\t\"i2.2xlarge\":        {Region: \"us-west-2\", Type: \"i2.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.705},\n\t\t\"i2.4xlarge\":        {Region: \"us-west-2\", Type: \"i2.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.41},\n\t\t\"i2.8xlarge\":        {Region: \"us-west-2\", Type: \"i2.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.82},\n\t\t\"i3.large\":          {Region: \"us-west-2\", Type: \"i3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.156},\n\t\t\"i3.xlarge\":         {Region: \"us-west-2\", Type: \"i3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.312},\n\t\t\"i3.2xlarge\":        {Region: \"us-west-2\", Type: \"i3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.624},\n\t\t\"i3.4xlarge\":        {Region: \"us-west-2\", Type: \"i3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.248},\n\t\t\"i3.8xlarge\":        {Region: \"us-west-2\", Type: \"i3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.496},\n\t\t\"i3.16xlarge\":       {Region: \"us-west-2\", Type: \"i3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"i3.metal\":          {Region: \"us-west-2\", Type: \"i3.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.992},\n\t\t\"i3en.large\":        {Region: \"us-west-2\", Type: \"i3en.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"i3en.xlarge\":       {Region: \"us-west-2\", Type: \"i3en.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"i3en.2xlarge\":      {Region: \"us-west-2\", Type: \"i3en.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"i3en.3xlarge\":      {Region: \"us-west-2\", Type: \"i3en.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.356},\n\t\t\"i3en.6xlarge\":      {Region: \"us-west-2\", Type: \"i3en.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"i3en.12xlarge\":     {Region: \"us-west-2\", Type: \"i3en.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"i3en.24xlarge\":     {Region: \"us-west-2\", Type: \"i3en.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.848},\n\t\t\"i3en.metal\":        {Region: \"us-west-2\", Type: \"i3en.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 10.848},\n\t\t\"im4gn.large\":       {Region: \"us-west-2\", Type: \"im4gn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1819},\n\t\t\"im4gn.xlarge\":      {Region: \"us-west-2\", Type: \"im4gn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.36379},\n\t\t\"im4gn.2xlarge\":     {Region: \"us-west-2\", Type: \"im4gn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.72758},\n\t\t\"im4gn.4xlarge\":     {Region: \"us-west-2\", Type: \"im4gn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.45517},\n\t\t\"im4gn.8xlarge\":     {Region: \"us-west-2\", Type: \"im4gn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.91034},\n\t\t\"im4gn.16xlarge\":    {Region: \"us-west-2\", Type: \"im4gn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.82067},\n\t\t\"inf1.xlarge\":       {Region: \"us-west-2\", Type: \"inf1.xlarge\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 1, Price: 0.228},\n\t\t\"inf1.2xlarge\":      {Region: \"us-west-2\", Type: \"inf1.2xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 1, Price: 0.362},\n\t\t\"inf1.6xlarge\":      {Region: \"us-west-2\", Type: \"inf1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 4, Price: 1.18},\n\t\t\"inf1.24xlarge\":     {Region: \"us-west-2\", Type: \"inf1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 16, Price: 4.721},\n\t\t\"is4gen.medium\":     {Region: \"us-west-2\", Type: \"is4gen.medium\", Memory: kresource.MustParse(\"6144Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.14408},\n\t\t\"is4gen.large\":      {Region: \"us-west-2\", Type: \"is4gen.large\", Memory: kresource.MustParse(\"12288Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.28815},\n\t\t\"is4gen.xlarge\":     {Region: \"us-west-2\", Type: \"is4gen.xlarge\", Memory: kresource.MustParse(\"24576Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.5763},\n\t\t\"is4gen.2xlarge\":    {Region: \"us-west-2\", Type: \"is4gen.2xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.1526},\n\t\t\"is4gen.4xlarge\":    {Region: \"us-west-2\", Type: \"is4gen.4xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 2.3052},\n\t\t\"is4gen.8xlarge\":    {Region: \"us-west-2\", Type: \"is4gen.8xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 4.6104},\n\t\t\"m1.small\":          {Region: \"us-west-2\", Type: \"m1.small\", Memory: kresource.MustParse(\"1740Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.044},\n\t\t\"m1.medium\":         {Region: \"us-west-2\", Type: \"m1.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.087},\n\t\t\"m1.large\":          {Region: \"us-west-2\", Type: \"m1.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.175},\n\t\t\"m1.xlarge\":         {Region: \"us-west-2\", Type: \"m1.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.35},\n\t\t\"m2.xlarge\":         {Region: \"us-west-2\", Type: \"m2.xlarge\", Memory: kresource.MustParse(\"17510Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.245},\n\t\t\"m2.2xlarge\":        {Region: \"us-west-2\", Type: \"m2.2xlarge\", Memory: kresource.MustParse(\"35020Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.49},\n\t\t\"m2.4xlarge\":        {Region: \"us-west-2\", Type: \"m2.4xlarge\", Memory: kresource.MustParse(\"70041Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.98},\n\t\t\"m3.medium\":         {Region: \"us-west-2\", Type: \"m3.medium\", Memory: kresource.MustParse(\"3840Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.067},\n\t\t\"m3.large\":          {Region: \"us-west-2\", Type: \"m3.large\", Memory: kresource.MustParse(\"7680Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"m3.xlarge\":         {Region: \"us-west-2\", Type: \"m3.xlarge\", Memory: kresource.MustParse(\"15360Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"m3.2xlarge\":        {Region: \"us-west-2\", Type: \"m3.2xlarge\", Memory: kresource.MustParse(\"30720Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"m4.large\":          {Region: \"us-west-2\", Type: \"m4.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1},\n\t\t\"m4.xlarge\":         {Region: \"us-west-2\", Type: \"m4.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2},\n\t\t\"m4.2xlarge\":        {Region: \"us-west-2\", Type: \"m4.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4},\n\t\t\"m4.4xlarge\":        {Region: \"us-west-2\", Type: \"m4.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8},\n\t\t\"m4.10xlarge\":       {Region: \"us-west-2\", Type: \"m4.10xlarge\", Memory: kresource.MustParse(\"163840Mi\"), CPU: kresource.MustParse(\"40\"), GPU: 0, Inf: 0, Price: 2.0},\n\t\t\"m4.16xlarge\":       {Region: \"us-west-2\", Type: \"m4.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2},\n\t\t\"m5.large\":          {Region: \"us-west-2\", Type: \"m5.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m5.xlarge\":         {Region: \"us-west-2\", Type: \"m5.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m5.2xlarge\":        {Region: \"us-west-2\", Type: \"m5.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m5.4xlarge\":        {Region: \"us-west-2\", Type: \"m5.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m5.8xlarge\":        {Region: \"us-west-2\", Type: \"m5.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m5.12xlarge\":       {Region: \"us-west-2\", Type: \"m5.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m5.16xlarge\":       {Region: \"us-west-2\", Type: \"m5.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m5.24xlarge\":       {Region: \"us-west-2\", Type: \"m5.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m5.metal\":          {Region: \"us-west-2\", Type: \"m5.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m5a.large\":         {Region: \"us-west-2\", Type: \"m5a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.086},\n\t\t\"m5a.xlarge\":        {Region: \"us-west-2\", Type: \"m5a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.172},\n\t\t\"m5a.2xlarge\":       {Region: \"us-west-2\", Type: \"m5a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.344},\n\t\t\"m5a.4xlarge\":       {Region: \"us-west-2\", Type: \"m5a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.688},\n\t\t\"m5a.8xlarge\":       {Region: \"us-west-2\", Type: \"m5a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.376},\n\t\t\"m5a.12xlarge\":      {Region: \"us-west-2\", Type: \"m5a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.064},\n\t\t\"m5a.16xlarge\":      {Region: \"us-west-2\", Type: \"m5a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.752},\n\t\t\"m5a.24xlarge\":      {Region: \"us-west-2\", Type: \"m5a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.128},\n\t\t\"m5ad.large\":        {Region: \"us-west-2\", Type: \"m5ad.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.103},\n\t\t\"m5ad.xlarge\":       {Region: \"us-west-2\", Type: \"m5ad.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.206},\n\t\t\"m5ad.2xlarge\":      {Region: \"us-west-2\", Type: \"m5ad.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.412},\n\t\t\"m5ad.4xlarge\":      {Region: \"us-west-2\", Type: \"m5ad.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.824},\n\t\t\"m5ad.8xlarge\":      {Region: \"us-west-2\", Type: \"m5ad.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.648},\n\t\t\"m5ad.12xlarge\":     {Region: \"us-west-2\", Type: \"m5ad.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.472},\n\t\t\"m5ad.16xlarge\":     {Region: \"us-west-2\", Type: \"m5ad.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.296},\n\t\t\"m5ad.24xlarge\":     {Region: \"us-west-2\", Type: \"m5ad.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.944},\n\t\t\"m5d.large\":         {Region: \"us-west-2\", Type: \"m5d.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"m5d.xlarge\":        {Region: \"us-west-2\", Type: \"m5d.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"m5d.2xlarge\":       {Region: \"us-west-2\", Type: \"m5d.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"m5d.4xlarge\":       {Region: \"us-west-2\", Type: \"m5d.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"m5d.8xlarge\":       {Region: \"us-west-2\", Type: \"m5d.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"m5d.12xlarge\":      {Region: \"us-west-2\", Type: \"m5d.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"m5d.16xlarge\":      {Region: \"us-west-2\", Type: \"m5d.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"m5d.24xlarge\":      {Region: \"us-west-2\", Type: \"m5d.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"m5d.metal\":         {Region: \"us-west-2\", Type: \"m5d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"m5dn.large\":        {Region: \"us-west-2\", Type: \"m5dn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.136},\n\t\t\"m5dn.xlarge\":       {Region: \"us-west-2\", Type: \"m5dn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.272},\n\t\t\"m5dn.2xlarge\":      {Region: \"us-west-2\", Type: \"m5dn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.544},\n\t\t\"m5dn.4xlarge\":      {Region: \"us-west-2\", Type: \"m5dn.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.088},\n\t\t\"m5dn.8xlarge\":      {Region: \"us-west-2\", Type: \"m5dn.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.176},\n\t\t\"m5dn.12xlarge\":     {Region: \"us-west-2\", Type: \"m5dn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.264},\n\t\t\"m5dn.16xlarge\":     {Region: \"us-west-2\", Type: \"m5dn.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.352},\n\t\t\"m5dn.24xlarge\":     {Region: \"us-west-2\", Type: \"m5dn.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"m5dn.metal\":        {Region: \"us-west-2\", Type: \"m5dn.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.528},\n\t\t\"m5n.large\":         {Region: \"us-west-2\", Type: \"m5n.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.119},\n\t\t\"m5n.xlarge\":        {Region: \"us-west-2\", Type: \"m5n.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.238},\n\t\t\"m5n.2xlarge\":       {Region: \"us-west-2\", Type: \"m5n.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.476},\n\t\t\"m5n.4xlarge\":       {Region: \"us-west-2\", Type: \"m5n.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.952},\n\t\t\"m5n.8xlarge\":       {Region: \"us-west-2\", Type: \"m5n.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.904},\n\t\t\"m5n.12xlarge\":      {Region: \"us-west-2\", Type: \"m5n.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.856},\n\t\t\"m5n.16xlarge\":      {Region: \"us-west-2\", Type: \"m5n.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.808},\n\t\t\"m5n.24xlarge\":      {Region: \"us-west-2\", Type: \"m5n.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.712},\n\t\t\"m5n.metal\":         {Region: \"us-west-2\", Type: \"m5n.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.712},\n\t\t\"m5zn.large\":        {Region: \"us-west-2\", Type: \"m5zn.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1652},\n\t\t\"m5zn.xlarge\":       {Region: \"us-west-2\", Type: \"m5zn.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.3303},\n\t\t\"m5zn.2xlarge\":      {Region: \"us-west-2\", Type: \"m5zn.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.6607},\n\t\t\"m5zn.3xlarge\":      {Region: \"us-west-2\", Type: \"m5zn.3xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 0.991},\n\t\t\"m5zn.6xlarge\":      {Region: \"us-west-2\", Type: \"m5zn.6xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 1.982},\n\t\t\"m5zn.12xlarge\":     {Region: \"us-west-2\", Type: \"m5zn.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.9641},\n\t\t\"m5zn.metal\":        {Region: \"us-west-2\", Type: \"m5zn.metal\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.9641},\n\t\t\"m6a.large\":         {Region: \"us-west-2\", Type: \"m6a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0864},\n\t\t\"m6a.xlarge\":        {Region: \"us-west-2\", Type: \"m6a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1728},\n\t\t\"m6a.2xlarge\":       {Region: \"us-west-2\", Type: \"m6a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3456},\n\t\t\"m6a.4xlarge\":       {Region: \"us-west-2\", Type: \"m6a.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.6912},\n\t\t\"m6a.8xlarge\":       {Region: \"us-west-2\", Type: \"m6a.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.3824},\n\t\t\"m6a.12xlarge\":      {Region: \"us-west-2\", Type: \"m6a.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.0736},\n\t\t\"m6a.16xlarge\":      {Region: \"us-west-2\", Type: \"m6a.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.7648},\n\t\t\"m6a.24xlarge\":      {Region: \"us-west-2\", Type: \"m6a.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.1472},\n\t\t\"m6a.32xlarge\":      {Region: \"us-west-2\", Type: \"m6a.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 5.5296},\n\t\t\"m6a.48xlarge\":      {Region: \"us-west-2\", Type: \"m6a.48xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 8.2944},\n\t\t\"m6a.metal\":         {Region: \"us-west-2\", Type: \"m6a.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"192\"), GPU: 0, Inf: 0, Price: 8.2944},\n\t\t\"m6g.medium\":        {Region: \"us-west-2\", Type: \"m6g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0385},\n\t\t\"m6g.large\":         {Region: \"us-west-2\", Type: \"m6g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.077},\n\t\t\"m6g.xlarge\":        {Region: \"us-west-2\", Type: \"m6g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.154},\n\t\t\"m6g.2xlarge\":       {Region: \"us-west-2\", Type: \"m6g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.308},\n\t\t\"m6g.4xlarge\":       {Region: \"us-west-2\", Type: \"m6g.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.616},\n\t\t\"m6g.8xlarge\":       {Region: \"us-west-2\", Type: \"m6g.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.232},\n\t\t\"m6g.12xlarge\":      {Region: \"us-west-2\", Type: \"m6g.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 1.848},\n\t\t\"m6g.16xlarge\":      {Region: \"us-west-2\", Type: \"m6g.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"m6g.metal\":         {Region: \"us-west-2\", Type: \"m6g.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.464},\n\t\t\"m6gd.medium\":       {Region: \"us-west-2\", Type: \"m6gd.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0452},\n\t\t\"m6gd.large\":        {Region: \"us-west-2\", Type: \"m6gd.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0904},\n\t\t\"m6gd.xlarge\":       {Region: \"us-west-2\", Type: \"m6gd.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1808},\n\t\t\"m6gd.2xlarge\":      {Region: \"us-west-2\", Type: \"m6gd.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3616},\n\t\t\"m6gd.4xlarge\":      {Region: \"us-west-2\", Type: \"m6gd.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.7232},\n\t\t\"m6gd.8xlarge\":      {Region: \"us-west-2\", Type: \"m6gd.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.4464},\n\t\t\"m6gd.12xlarge\":     {Region: \"us-west-2\", Type: \"m6gd.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.1696},\n\t\t\"m6gd.16xlarge\":     {Region: \"us-west-2\", Type: \"m6gd.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8928},\n\t\t\"m6gd.metal\":        {Region: \"us-west-2\", Type: \"m6gd.metal\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 2.8928},\n\t\t\"m6i.large\":         {Region: \"us-west-2\", Type: \"m6i.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.096},\n\t\t\"m6i.xlarge\":        {Region: \"us-west-2\", Type: \"m6i.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.192},\n\t\t\"m6i.2xlarge\":       {Region: \"us-west-2\", Type: \"m6i.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.384},\n\t\t\"m6i.4xlarge\":       {Region: \"us-west-2\", Type: \"m6i.4xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.768},\n\t\t\"m6i.8xlarge\":       {Region: \"us-west-2\", Type: \"m6i.8xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.536},\n\t\t\"m6i.12xlarge\":      {Region: \"us-west-2\", Type: \"m6i.12xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"m6i.16xlarge\":      {Region: \"us-west-2\", Type: \"m6i.16xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.072},\n\t\t\"m6i.24xlarge\":      {Region: \"us-west-2\", Type: \"m6i.24xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"m6i.32xlarge\":      {Region: \"us-west-2\", Type: \"m6i.32xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.144},\n\t\t\"m6i.metal\":         {Region: \"us-west-2\", Type: \"m6i.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 6.144},\n\t\t\"p2.xlarge\":         {Region: \"us-west-2\", Type: \"p2.xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 1, Inf: 0, Price: 0.9},\n\t\t\"p2.8xlarge\":        {Region: \"us-west-2\", Type: \"p2.8xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 8, Inf: 0, Price: 7.2},\n\t\t\"p2.16xlarge\":       {Region: \"us-west-2\", Type: \"p2.16xlarge\", Memory: kresource.MustParse(\"749568Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 16, Inf: 0, Price: 14.4},\n\t\t\"p3.2xlarge\":        {Region: \"us-west-2\", Type: \"p3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 1, Inf: 0, Price: 3.06},\n\t\t\"p3.8xlarge\":        {Region: \"us-west-2\", Type: \"p3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 4, Inf: 0, Price: 12.24},\n\t\t\"p3.16xlarge\":       {Region: \"us-west-2\", Type: \"p3.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 8, Inf: 0, Price: 24.48},\n\t\t\"p3dn.24xlarge\":     {Region: \"us-west-2\", Type: \"p3dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 31.212},\n\t\t\"p4d.24xlarge\":      {Region: \"us-west-2\", Type: \"p4d.24xlarge\", Memory: kresource.MustParse(\"1179648Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 8, Inf: 0, Price: 32.7726},\n\t\t\"r3.large\":          {Region: \"us-west-2\", Type: \"r3.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.166},\n\t\t\"r3.xlarge\":         {Region: \"us-west-2\", Type: \"r3.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.333},\n\t\t\"r3.2xlarge\":        {Region: \"us-west-2\", Type: \"r3.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.665},\n\t\t\"r3.4xlarge\":        {Region: \"us-west-2\", Type: \"r3.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.33},\n\t\t\"r3.8xlarge\":        {Region: \"us-west-2\", Type: \"r3.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.66},\n\t\t\"r4.large\":          {Region: \"us-west-2\", Type: \"r4.large\", Memory: kresource.MustParse(\"15616Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.133},\n\t\t\"r4.xlarge\":         {Region: \"us-west-2\", Type: \"r4.xlarge\", Memory: kresource.MustParse(\"31232Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.266},\n\t\t\"r4.2xlarge\":        {Region: \"us-west-2\", Type: \"r4.2xlarge\", Memory: kresource.MustParse(\"62464Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.532},\n\t\t\"r4.4xlarge\":        {Region: \"us-west-2\", Type: \"r4.4xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.064},\n\t\t\"r4.8xlarge\":        {Region: \"us-west-2\", Type: \"r4.8xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.128},\n\t\t\"r4.16xlarge\":       {Region: \"us-west-2\", Type: \"r4.16xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.256},\n\t\t\"r5.large\":          {Region: \"us-west-2\", Type: \"r5.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"r5.xlarge\":         {Region: \"us-west-2\", Type: \"r5.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"r5.2xlarge\":        {Region: \"us-west-2\", Type: \"r5.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"r5.4xlarge\":        {Region: \"us-west-2\", Type: \"r5.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"r5.8xlarge\":        {Region: \"us-west-2\", Type: \"r5.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"r5.12xlarge\":       {Region: \"us-west-2\", Type: \"r5.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"r5.16xlarge\":       {Region: \"us-west-2\", Type: \"r5.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r5.24xlarge\":       {Region: \"us-west-2\", Type: \"r5.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r5.metal\":          {Region: \"us-west-2\", Type: \"r5.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r5a.large\":         {Region: \"us-west-2\", Type: \"r5a.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.113},\n\t\t\"r5a.xlarge\":        {Region: \"us-west-2\", Type: \"r5a.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.226},\n\t\t\"r5a.2xlarge\":       {Region: \"us-west-2\", Type: \"r5a.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.452},\n\t\t\"r5a.4xlarge\":       {Region: \"us-west-2\", Type: \"r5a.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.904},\n\t\t\"r5a.8xlarge\":       {Region: \"us-west-2\", Type: \"r5a.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.808},\n\t\t\"r5a.12xlarge\":      {Region: \"us-west-2\", Type: \"r5a.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.712},\n\t\t\"r5a.16xlarge\":      {Region: \"us-west-2\", Type: \"r5a.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.616},\n\t\t\"r5a.24xlarge\":      {Region: \"us-west-2\", Type: \"r5a.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.424},\n\t\t\"r5ad.large\":        {Region: \"us-west-2\", Type: \"r5ad.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.131},\n\t\t\"r5ad.xlarge\":       {Region: \"us-west-2\", Type: \"r5ad.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.262},\n\t\t\"r5ad.2xlarge\":      {Region: \"us-west-2\", Type: \"r5ad.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.524},\n\t\t\"r5ad.4xlarge\":      {Region: \"us-west-2\", Type: \"r5ad.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.048},\n\t\t\"r5ad.8xlarge\":      {Region: \"us-west-2\", Type: \"r5ad.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.096},\n\t\t\"r5ad.12xlarge\":     {Region: \"us-west-2\", Type: \"r5ad.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.144},\n\t\t\"r5ad.16xlarge\":     {Region: \"us-west-2\", Type: \"r5ad.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.192},\n\t\t\"r5ad.24xlarge\":     {Region: \"us-west-2\", Type: \"r5ad.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.288},\n\t\t\"r5b.large\":         {Region: \"us-west-2\", Type: \"r5b.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"r5b.xlarge\":        {Region: \"us-west-2\", Type: \"r5b.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"r5b.2xlarge\":       {Region: \"us-west-2\", Type: \"r5b.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"r5b.4xlarge\":       {Region: \"us-west-2\", Type: \"r5b.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"r5b.8xlarge\":       {Region: \"us-west-2\", Type: \"r5b.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.384},\n\t\t\"r5b.12xlarge\":      {Region: \"us-west-2\", Type: \"r5b.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"r5b.16xlarge\":      {Region: \"us-west-2\", Type: \"r5b.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.768},\n\t\t\"r5b.24xlarge\":      {Region: \"us-west-2\", Type: \"r5b.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5b.metal\":         {Region: \"us-west-2\", Type: \"r5b.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5d.large\":         {Region: \"us-west-2\", Type: \"r5d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.144},\n\t\t\"r5d.xlarge\":        {Region: \"us-west-2\", Type: \"r5d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.288},\n\t\t\"r5d.2xlarge\":       {Region: \"us-west-2\", Type: \"r5d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.576},\n\t\t\"r5d.4xlarge\":       {Region: \"us-west-2\", Type: \"r5d.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.152},\n\t\t\"r5d.8xlarge\":       {Region: \"us-west-2\", Type: \"r5d.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.304},\n\t\t\"r5d.12xlarge\":      {Region: \"us-west-2\", Type: \"r5d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.456},\n\t\t\"r5d.16xlarge\":      {Region: \"us-west-2\", Type: \"r5d.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.608},\n\t\t\"r5d.24xlarge\":      {Region: \"us-west-2\", Type: \"r5d.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.912},\n\t\t\"r5d.metal\":         {Region: \"us-west-2\", Type: \"r5d.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.912},\n\t\t\"r5dn.large\":        {Region: \"us-west-2\", Type: \"r5dn.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"r5dn.xlarge\":       {Region: \"us-west-2\", Type: \"r5dn.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"r5dn.2xlarge\":      {Region: \"us-west-2\", Type: \"r5dn.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"r5dn.4xlarge\":      {Region: \"us-west-2\", Type: \"r5dn.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"r5dn.8xlarge\":      {Region: \"us-west-2\", Type: \"r5dn.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"r5dn.12xlarge\":     {Region: \"us-west-2\", Type: \"r5dn.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"r5dn.16xlarge\":     {Region: \"us-west-2\", Type: \"r5dn.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"r5dn.24xlarge\":     {Region: \"us-west-2\", Type: \"r5dn.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5dn.metal\":        {Region: \"us-west-2\", Type: \"r5dn.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 8.016},\n\t\t\"r5n.large\":         {Region: \"us-west-2\", Type: \"r5n.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.149},\n\t\t\"r5n.xlarge\":        {Region: \"us-west-2\", Type: \"r5n.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.298},\n\t\t\"r5n.2xlarge\":       {Region: \"us-west-2\", Type: \"r5n.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.596},\n\t\t\"r5n.4xlarge\":       {Region: \"us-west-2\", Type: \"r5n.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.192},\n\t\t\"r5n.8xlarge\":       {Region: \"us-west-2\", Type: \"r5n.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.384},\n\t\t\"r5n.12xlarge\":      {Region: \"us-west-2\", Type: \"r5n.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.576},\n\t\t\"r5n.16xlarge\":      {Region: \"us-west-2\", Type: \"r5n.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.768},\n\t\t\"r5n.24xlarge\":      {Region: \"us-west-2\", Type: \"r5n.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r5n.metal\":         {Region: \"us-west-2\", Type: \"r5n.metal\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 7.152},\n\t\t\"r6g.medium\":        {Region: \"us-west-2\", Type: \"r6g.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0504},\n\t\t\"r6g.large\":         {Region: \"us-west-2\", Type: \"r6g.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1008},\n\t\t\"r6g.xlarge\":        {Region: \"us-west-2\", Type: \"r6g.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2016},\n\t\t\"r6g.2xlarge\":       {Region: \"us-west-2\", Type: \"r6g.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4032},\n\t\t\"r6g.4xlarge\":       {Region: \"us-west-2\", Type: \"r6g.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.8064},\n\t\t\"r6g.8xlarge\":       {Region: \"us-west-2\", Type: \"r6g.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.6128},\n\t\t\"r6g.12xlarge\":      {Region: \"us-west-2\", Type: \"r6g.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.4192},\n\t\t\"r6g.16xlarge\":      {Region: \"us-west-2\", Type: \"r6g.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2256},\n\t\t\"r6g.metal\":         {Region: \"us-west-2\", Type: \"r6g.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.2256},\n\t\t\"r6gd.medium\":       {Region: \"us-west-2\", Type: \"r6gd.medium\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0576},\n\t\t\"r6gd.large\":        {Region: \"us-west-2\", Type: \"r6gd.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.1152},\n\t\t\"r6gd.xlarge\":       {Region: \"us-west-2\", Type: \"r6gd.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.2304},\n\t\t\"r6gd.2xlarge\":      {Region: \"us-west-2\", Type: \"r6gd.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.4608},\n\t\t\"r6gd.4xlarge\":      {Region: \"us-west-2\", Type: \"r6gd.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 0.9216},\n\t\t\"r6gd.8xlarge\":      {Region: \"us-west-2\", Type: \"r6gd.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 1.8432},\n\t\t\"r6gd.12xlarge\":     {Region: \"us-west-2\", Type: \"r6gd.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 2.7648},\n\t\t\"r6gd.16xlarge\":     {Region: \"us-west-2\", Type: \"r6gd.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.6864},\n\t\t\"r6gd.metal\":        {Region: \"us-west-2\", Type: \"r6gd.metal\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 3.6864},\n\t\t\"r6i.large\":         {Region: \"us-west-2\", Type: \"r6i.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.126},\n\t\t\"r6i.xlarge\":        {Region: \"us-west-2\", Type: \"r6i.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.252},\n\t\t\"r6i.2xlarge\":       {Region: \"us-west-2\", Type: \"r6i.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.504},\n\t\t\"r6i.4xlarge\":       {Region: \"us-west-2\", Type: \"r6i.4xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.008},\n\t\t\"r6i.8xlarge\":       {Region: \"us-west-2\", Type: \"r6i.8xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.016},\n\t\t\"r6i.12xlarge\":      {Region: \"us-west-2\", Type: \"r6i.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 3.024},\n\t\t\"r6i.16xlarge\":      {Region: \"us-west-2\", Type: \"r6i.16xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 4.032},\n\t\t\"r6i.24xlarge\":      {Region: \"us-west-2\", Type: \"r6i.24xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 6.048},\n\t\t\"r6i.32xlarge\":      {Region: \"us-west-2\", Type: \"r6i.32xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"r6i.metal\":         {Region: \"us-west-2\", Type: \"r6i.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 8.064},\n\t\t\"t1.micro\":          {Region: \"us-west-2\", Type: \"t1.micro\", Memory: kresource.MustParse(\"627Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.02},\n\t\t\"t2.nano\":           {Region: \"us-west-2\", Type: \"t2.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0058},\n\t\t\"t2.micro\":          {Region: \"us-west-2\", Type: \"t2.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0116},\n\t\t\"t2.small\":          {Region: \"us-west-2\", Type: \"t2.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.023},\n\t\t\"t2.medium\":         {Region: \"us-west-2\", Type: \"t2.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0464},\n\t\t\"t2.large\":          {Region: \"us-west-2\", Type: \"t2.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0928},\n\t\t\"t2.xlarge\":         {Region: \"us-west-2\", Type: \"t2.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1856},\n\t\t\"t2.2xlarge\":        {Region: \"us-west-2\", Type: \"t2.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3712},\n\t\t\"t3.nano\":           {Region: \"us-west-2\", Type: \"t3.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0052},\n\t\t\"t3.micro\":          {Region: \"us-west-2\", Type: \"t3.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0104},\n\t\t\"t3.small\":          {Region: \"us-west-2\", Type: \"t3.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0208},\n\t\t\"t3.medium\":         {Region: \"us-west-2\", Type: \"t3.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0416},\n\t\t\"t3.large\":          {Region: \"us-west-2\", Type: \"t3.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0832},\n\t\t\"t3.xlarge\":         {Region: \"us-west-2\", Type: \"t3.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1664},\n\t\t\"t3.2xlarge\":        {Region: \"us-west-2\", Type: \"t3.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3328},\n\t\t\"t3a.nano\":          {Region: \"us-west-2\", Type: \"t3a.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0047},\n\t\t\"t3a.micro\":         {Region: \"us-west-2\", Type: \"t3a.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0094},\n\t\t\"t3a.small\":         {Region: \"us-west-2\", Type: \"t3a.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0188},\n\t\t\"t3a.medium\":        {Region: \"us-west-2\", Type: \"t3a.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0376},\n\t\t\"t3a.large\":         {Region: \"us-west-2\", Type: \"t3a.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0752},\n\t\t\"t3a.xlarge\":        {Region: \"us-west-2\", Type: \"t3a.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1504},\n\t\t\"t3a.2xlarge\":       {Region: \"us-west-2\", Type: \"t3a.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.3008},\n\t\t\"t4g.nano\":          {Region: \"us-west-2\", Type: \"t4g.nano\", Memory: kresource.MustParse(\"512Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0042},\n\t\t\"t4g.micro\":         {Region: \"us-west-2\", Type: \"t4g.micro\", Memory: kresource.MustParse(\"1024Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0084},\n\t\t\"t4g.small\":         {Region: \"us-west-2\", Type: \"t4g.small\", Memory: kresource.MustParse(\"2048Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0168},\n\t\t\"t4g.medium\":        {Region: \"us-west-2\", Type: \"t4g.medium\", Memory: kresource.MustParse(\"4096Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0336},\n\t\t\"t4g.large\":         {Region: \"us-west-2\", Type: \"t4g.large\", Memory: kresource.MustParse(\"8192Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.0672},\n\t\t\"t4g.xlarge\":        {Region: \"us-west-2\", Type: \"t4g.xlarge\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.1344},\n\t\t\"t4g.2xlarge\":       {Region: \"us-west-2\", Type: \"t4g.2xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.2688},\n\t\t\"u-12tb1.112xlarge\": {Region: \"us-west-2\", Type: \"u-12tb1.112xlarge\", Memory: kresource.MustParse(\"12582912Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 109.2},\n\t\t\"u-3tb1.56xlarge\":   {Region: \"us-west-2\", Type: \"u-3tb1.56xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 27.3},\n\t\t\"u-6tb1.56xlarge\":   {Region: \"us-west-2\", Type: \"u-6tb1.56xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"224\"), GPU: 0, Inf: 0, Price: 46.40391},\n\t\t\"u-6tb1.112xlarge\":  {Region: \"us-west-2\", Type: \"u-6tb1.112xlarge\", Memory: kresource.MustParse(\"6291456Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 54.6},\n\t\t\"u-9tb1.112xlarge\":  {Region: \"us-west-2\", Type: \"u-9tb1.112xlarge\", Memory: kresource.MustParse(\"9437184Mi\"), CPU: kresource.MustParse(\"448\"), GPU: 0, Inf: 0, Price: 81.9},\n\t\t\"vt1.3xlarge\":       {Region: \"us-west-2\", Type: \"vt1.3xlarge\", Memory: kresource.MustParse(\"24576Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 0.65},\n\t\t\"vt1.6xlarge\":       {Region: \"us-west-2\", Type: \"vt1.6xlarge\", Memory: kresource.MustParse(\"49152Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 1.3},\n\t\t\"vt1.24xlarge\":      {Region: \"us-west-2\", Type: \"vt1.24xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 5.2},\n\t\t\"x1.16xlarge\":       {Region: \"us-west-2\", Type: \"x1.16xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 6.669},\n\t\t\"x1.32xlarge\":       {Region: \"us-west-2\", Type: \"x1.32xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 13.338},\n\t\t\"x1e.xlarge\":        {Region: \"us-west-2\", Type: \"x1e.xlarge\", Memory: kresource.MustParse(\"124928Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.834},\n\t\t\"x1e.2xlarge\":       {Region: \"us-west-2\", Type: \"x1e.2xlarge\", Memory: kresource.MustParse(\"249856Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.668},\n\t\t\"x1e.4xlarge\":       {Region: \"us-west-2\", Type: \"x1e.4xlarge\", Memory: kresource.MustParse(\"499712Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"x1e.8xlarge\":       {Region: \"us-west-2\", Type: \"x1e.8xlarge\", Memory: kresource.MustParse(\"999424Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.672},\n\t\t\"x1e.16xlarge\":      {Region: \"us-west-2\", Type: \"x1e.16xlarge\", Memory: kresource.MustParse(\"1998848Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.344},\n\t\t\"x1e.32xlarge\":      {Region: \"us-west-2\", Type: \"x1e.32xlarge\", Memory: kresource.MustParse(\"3997696Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 26.688},\n\t\t\"x2gd.medium\":       {Region: \"us-west-2\", Type: \"x2gd.medium\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"1\"), GPU: 0, Inf: 0, Price: 0.0835},\n\t\t\"x2gd.large\":        {Region: \"us-west-2\", Type: \"x2gd.large\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.167},\n\t\t\"x2gd.xlarge\":       {Region: \"us-west-2\", Type: \"x2gd.xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.334},\n\t\t\"x2gd.2xlarge\":      {Region: \"us-west-2\", Type: \"x2gd.2xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.668},\n\t\t\"x2gd.4xlarge\":      {Region: \"us-west-2\", Type: \"x2gd.4xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 1.336},\n\t\t\"x2gd.8xlarge\":      {Region: \"us-west-2\", Type: \"x2gd.8xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 2.672},\n\t\t\"x2gd.12xlarge\":     {Region: \"us-west-2\", Type: \"x2gd.12xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.008},\n\t\t\"x2gd.16xlarge\":     {Region: \"us-west-2\", Type: \"x2gd.16xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"x2gd.metal\":        {Region: \"us-west-2\", Type: \"x2gd.metal\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 5.344},\n\t\t\"x2iedn.xlarge\":     {Region: \"us-west-2\", Type: \"x2iedn.xlarge\", Memory: kresource.MustParse(\"131072Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.83363},\n\t\t\"x2iedn.2xlarge\":    {Region: \"us-west-2\", Type: \"x2iedn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.66725},\n\t\t\"x2iedn.4xlarge\":    {Region: \"us-west-2\", Type: \"x2iedn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.3345},\n\t\t\"x2iedn.8xlarge\":    {Region: \"us-west-2\", Type: \"x2iedn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.669},\n\t\t\"x2iedn.16xlarge\":   {Region: \"us-west-2\", Type: \"x2iedn.16xlarge\", Memory: kresource.MustParse(\"2097152Mi\"), CPU: kresource.MustParse(\"64\"), GPU: 0, Inf: 0, Price: 13.338},\n\t\t\"x2iedn.24xlarge\":   {Region: \"us-west-2\", Type: \"x2iedn.24xlarge\", Memory: kresource.MustParse(\"3145728Mi\"), CPU: kresource.MustParse(\"96\"), GPU: 0, Inf: 0, Price: 20.007},\n\t\t\"x2iedn.32xlarge\":   {Region: \"us-west-2\", Type: \"x2iedn.32xlarge\", Memory: kresource.MustParse(\"4194304Mi\"), CPU: kresource.MustParse(\"128\"), GPU: 0, Inf: 0, Price: 26.676},\n\t\t\"x2iezn.2xlarge\":    {Region: \"us-west-2\", Type: \"x2iezn.2xlarge\", Memory: kresource.MustParse(\"262144Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 1.668},\n\t\t\"x2iezn.4xlarge\":    {Region: \"us-west-2\", Type: \"x2iezn.4xlarge\", Memory: kresource.MustParse(\"524288Mi\"), CPU: kresource.MustParse(\"16\"), GPU: 0, Inf: 0, Price: 3.336},\n\t\t\"x2iezn.6xlarge\":    {Region: \"us-west-2\", Type: \"x2iezn.6xlarge\", Memory: kresource.MustParse(\"786432Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 5.004},\n\t\t\"x2iezn.8xlarge\":    {Region: \"us-west-2\", Type: \"x2iezn.8xlarge\", Memory: kresource.MustParse(\"1048576Mi\"), CPU: kresource.MustParse(\"32\"), GPU: 0, Inf: 0, Price: 6.672},\n\t\t\"x2iezn.12xlarge\":   {Region: \"us-west-2\", Type: \"x2iezn.12xlarge\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 10.008},\n\t\t\"x2iezn.metal\":      {Region: \"us-west-2\", Type: \"x2iezn.metal\", Memory: kresource.MustParse(\"1572864Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 10.008},\n\t\t\"z1d.large\":         {Region: \"us-west-2\", Type: \"z1d.large\", Memory: kresource.MustParse(\"16384Mi\"), CPU: kresource.MustParse(\"2\"), GPU: 0, Inf: 0, Price: 0.186},\n\t\t\"z1d.xlarge\":        {Region: \"us-west-2\", Type: \"z1d.xlarge\", Memory: kresource.MustParse(\"32768Mi\"), CPU: kresource.MustParse(\"4\"), GPU: 0, Inf: 0, Price: 0.372},\n\t\t\"z1d.2xlarge\":       {Region: \"us-west-2\", Type: \"z1d.2xlarge\", Memory: kresource.MustParse(\"65536Mi\"), CPU: kresource.MustParse(\"8\"), GPU: 0, Inf: 0, Price: 0.744},\n\t\t\"z1d.3xlarge\":       {Region: \"us-west-2\", Type: \"z1d.3xlarge\", Memory: kresource.MustParse(\"98304Mi\"), CPU: kresource.MustParse(\"12\"), GPU: 0, Inf: 0, Price: 1.116},\n\t\t\"z1d.6xlarge\":       {Region: \"us-west-2\", Type: \"z1d.6xlarge\", Memory: kresource.MustParse(\"196608Mi\"), CPU: kresource.MustParse(\"24\"), GPU: 0, Inf: 0, Price: 2.232},\n\t\t\"z1d.12xlarge\":      {Region: \"us-west-2\", Type: \"z1d.12xlarge\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.464},\n\t\t\"z1d.metal\":         {Region: \"us-west-2\", Type: \"z1d.metal\", Memory: kresource.MustParse(\"393216Mi\"), CPU: kresource.MustParse(\"48\"), GPU: 0, Inf: 0, Price: 4.464},\n\t},\n}\n\n// region -> NLB metadata\nvar NLBMetadatas = map[string]NLBMetadata{\n\t\"af-south-1\":     {Region: \"af-south-1\", Price: 0.029988},\n\t\"ap-east-1\":      {Region: \"ap-east-1\", Price: 0.0277},\n\t\"ap-northeast-1\": {Region: \"ap-northeast-1\", Price: 0.0243},\n\t\"ap-northeast-2\": {Region: \"ap-northeast-2\", Price: 0.0225},\n\t\"ap-northeast-3\": {Region: \"ap-northeast-3\", Price: 0.0243},\n\t\"ap-south-1\":     {Region: \"ap-south-1\", Price: 0.0239},\n\t\"ap-southeast-1\": {Region: \"ap-southeast-1\", Price: 0.0252},\n\t\"ap-southeast-2\": {Region: \"ap-southeast-2\", Price: 0.0252},\n\t\"ca-central-1\":   {Region: \"ca-central-1\", Price: 0.02475},\n\t\"eu-central-1\":   {Region: \"eu-central-1\", Price: 0.027},\n\t\"eu-north-1\":     {Region: \"eu-north-1\", Price: 0.02394},\n\t\"eu-south-1\":     {Region: \"eu-south-1\", Price: 0.02646},\n\t\"eu-west-1\":      {Region: \"eu-west-1\", Price: 0.0252},\n\t\"eu-west-2\":      {Region: \"eu-west-2\", Price: 0.02646},\n\t\"eu-west-3\":      {Region: \"eu-west-3\", Price: 0.02646},\n\t\"me-south-1\":     {Region: \"me-south-1\", Price: 0.02772},\n\t\"sa-east-1\":      {Region: \"sa-east-1\", Price: 0.034},\n\t\"us-east-1\":      {Region: \"us-east-1\", Price: 0.0225},\n\t\"us-east-2\":      {Region: \"us-east-2\", Price: 0.0225},\n\t\"us-gov-east-1\":  {Region: \"us-gov-east-1\", Price: 0.032},\n\t\"us-gov-west-1\":  {Region: \"us-gov-west-1\", Price: 0.032},\n\t\"us-west-1\":      {Region: \"us-west-1\", Price: 0.0252},\n\t\"us-west-2\":      {Region: \"us-west-2\", Price: 0.0225},\n}\n\n// region -> ELB metadata\nvar ELBMetadatas = map[string]ELBMetadata{\n\t\"af-south-1\":     {Region: \"af-south-1\", Price: 0.03332},\n\t\"ap-east-1\":      {Region: \"ap-east-1\", Price: 0.0308},\n\t\"ap-northeast-1\": {Region: \"ap-northeast-1\", Price: 0.027},\n\t\"ap-northeast-2\": {Region: \"ap-northeast-2\", Price: 0.025},\n\t\"ap-northeast-3\": {Region: \"ap-northeast-3\", Price: 0.027},\n\t\"ap-south-1\":     {Region: \"ap-south-1\", Price: 0.0266},\n\t\"ap-southeast-1\": {Region: \"ap-southeast-1\", Price: 0.028},\n\t\"ap-southeast-2\": {Region: \"ap-southeast-2\", Price: 0.028},\n\t\"ca-central-1\":   {Region: \"ca-central-1\", Price: 0.0275},\n\t\"eu-central-1\":   {Region: \"eu-central-1\", Price: 0.03},\n\t\"eu-north-1\":     {Region: \"eu-north-1\", Price: 0.0266},\n\t\"eu-south-1\":     {Region: \"eu-south-1\", Price: 0.0294},\n\t\"eu-west-1\":      {Region: \"eu-west-1\", Price: 0.028},\n\t\"eu-west-2\":      {Region: \"eu-west-2\", Price: 0.0294},\n\t\"eu-west-3\":      {Region: \"eu-west-3\", Price: 0.0294},\n\t\"me-south-1\":     {Region: \"me-south-1\", Price: 0.0308},\n\t\"sa-east-1\":      {Region: \"sa-east-1\", Price: 0.034},\n\t\"us-east-1\":      {Region: \"us-east-1\", Price: 0.025},\n\t\"us-east-2\":      {Region: \"us-east-2\", Price: 0.025},\n\t\"us-gov-east-1\":  {Region: \"us-gov-east-1\", Price: 0.032},\n\t\"us-gov-west-1\":  {Region: \"us-gov-west-1\", Price: 0.032},\n\t\"us-west-1\":      {Region: \"us-west-1\", Price: 0.028},\n\t\"us-west-2\":      {Region: \"us-west-2\", Price: 0.025},\n}\n\n// region -> NAT metadata\nvar NATMetadatas = map[string]NATMetadata{\n\t\"af-south-1\":     {Region: \"af-south-1\", Price: 0.057},\n\t\"ap-east-1\":      {Region: \"ap-east-1\", Price: 0.065},\n\t\"ap-northeast-1\": {Region: \"ap-northeast-1\", Price: 0.062},\n\t\"ap-northeast-2\": {Region: \"ap-northeast-2\", Price: 0.059},\n\t\"ap-northeast-3\": {Region: \"ap-northeast-3\", Price: 0.062},\n\t\"ap-south-1\":     {Region: \"ap-south-1\", Price: 0.056},\n\t\"ap-southeast-1\": {Region: \"ap-southeast-1\", Price: 0.059},\n\t\"ap-southeast-2\": {Region: \"ap-southeast-2\", Price: 0.059},\n\t\"ca-central-1\":   {Region: \"ca-central-1\", Price: 0.05},\n\t\"eu-central-1\":   {Region: \"eu-central-1\", Price: 0.052},\n\t\"eu-north-1\":     {Region: \"eu-north-1\", Price: 0.046},\n\t\"eu-south-1\":     {Region: \"eu-south-1\", Price: 0.05},\n\t\"eu-west-1\":      {Region: \"eu-west-1\", Price: 0.048},\n\t\"eu-west-2\":      {Region: \"eu-west-2\", Price: 0.05},\n\t\"eu-west-3\":      {Region: \"eu-west-3\", Price: 0.05},\n\t\"me-south-1\":     {Region: \"me-south-1\", Price: 0.0528},\n\t\"sa-east-1\":      {Region: \"sa-east-1\", Price: 0.093},\n\t\"us-east-1\":      {Region: \"us-east-1\", Price: 0.045},\n\t\"us-east-2\":      {Region: \"us-east-2\", Price: 0.045},\n\t\"us-gov-east-1\":  {Region: \"us-gov-east-1\", Price: 0.054},\n\t\"us-gov-west-1\":  {Region: \"us-gov-west-1\", Price: 0.054},\n\t\"us-west-1\":      {Region: \"us-west-1\", Price: 0.048},\n\t\"us-west-2\":      {Region: \"us-west-2\", Price: 0.045},\n}\n\n// region -> EBS metadata\nvar EBSMetadatas = map[string]map[string]EBSMetadata{\n\t\"af-south-1\": {\n\t\t\"gp2\": {Region: \"af-south-1\", Type: \"gp2\", PriceGB: 0.1309, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"af-south-1\", Type: \"gp3\", PriceGB: 0.1047, PriceIOPS: 0.0065000000, PriceThroughput: 0.0536576, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"af-south-1\", Type: \"io1\", PriceGB: 0.16422, PriceIOPS: 0.0856800000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"af-south-1\", Type: \"sc1\", PriceGB: 0.019992, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"af-south-1\", Type: \"st1\", PriceGB: 0.0595, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"ap-east-1\": {\n\t\t\"gp2\": {Region: \"ap-east-1\", Type: \"gp2\", PriceGB: 0.132, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"ap-east-1\", Type: \"gp3\", PriceGB: 0.1056, PriceIOPS: 0.0066000000, PriceThroughput: 0.0540672, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"ap-east-1\", Type: \"io1\", PriceGB: 0.1518, PriceIOPS: 0.0792000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"ap-east-1\", Type: \"io2\", PriceGB: 0.1518, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"ap-east-1\", Type: \"sc1\", PriceGB: 0.0198, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"ap-east-1\", Type: \"st1\", PriceGB: 0.0594, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"ap-northeast-1\": {\n\t\t\"gp2\": {Region: \"ap-northeast-1\", Type: \"gp2\", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"ap-northeast-1\", Type: \"gp3\", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"ap-northeast-1\", Type: \"io1\", PriceGB: 0.142, PriceIOPS: 0.0740000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"ap-northeast-1\", Type: \"io2\", PriceGB: 0.142, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"ap-northeast-1\", Type: \"sc1\", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"ap-northeast-1\", Type: \"st1\", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"ap-northeast-2\": {\n\t\t\"gp2\": {Region: \"ap-northeast-2\", Type: \"gp2\", PriceGB: 0.114, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"ap-northeast-2\", Type: \"gp3\", PriceGB: 0.0912, PriceIOPS: 0.0057000000, PriceThroughput: 0.046694400000000004, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"ap-northeast-2\", Type: \"io1\", PriceGB: 0.1278, PriceIOPS: 0.0666000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"ap-northeast-2\", Type: \"io2\", PriceGB: 0.1278, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"ap-northeast-2\", Type: \"sc1\", PriceGB: 0.0174, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"ap-northeast-2\", Type: \"st1\", PriceGB: 0.051, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"ap-northeast-3\": {\n\t\t\"gp2\": {Region: \"ap-northeast-3\", Type: \"gp2\", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"ap-northeast-3\", Type: \"gp3\", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"ap-northeast-3\", Type: \"io1\", PriceGB: 0.142, PriceIOPS: 0.0740000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"ap-northeast-3\", Type: \"sc1\", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"ap-northeast-3\", Type: \"st1\", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"ap-south-1\": {\n\t\t\"gp2\": {Region: \"ap-south-1\", Type: \"gp2\", PriceGB: 0.114, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"ap-south-1\", Type: \"gp3\", PriceGB: 0.0912, PriceIOPS: 0.0057000000, PriceThroughput: 0.046694400000000004, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"ap-south-1\", Type: \"io1\", PriceGB: 0.131, PriceIOPS: 0.0680000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"ap-south-1\", Type: \"io2\", PriceGB: 0.131, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"ap-south-1\", Type: \"sc1\", PriceGB: 0.0174, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"ap-south-1\", Type: \"st1\", PriceGB: 0.051, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"ap-southeast-1\": {\n\t\t\"gp2\": {Region: \"ap-southeast-1\", Type: \"gp2\", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"ap-southeast-1\", Type: \"gp3\", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"ap-southeast-1\", Type: \"io1\", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"ap-southeast-1\", Type: \"io2\", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"ap-southeast-1\", Type: \"sc1\", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"ap-southeast-1\", Type: \"st1\", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"ap-southeast-2\": {\n\t\t\"gp2\": {Region: \"ap-southeast-2\", Type: \"gp2\", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"ap-southeast-2\", Type: \"gp3\", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"ap-southeast-2\", Type: \"io1\", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"ap-southeast-2\", Type: \"io2\", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"ap-southeast-2\", Type: \"sc1\", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"ap-southeast-2\", Type: \"st1\", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"ca-central-1\": {\n\t\t\"gp2\": {Region: \"ca-central-1\", Type: \"gp2\", PriceGB: 0.11, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"ca-central-1\", Type: \"gp3\", PriceGB: 0.088, PriceIOPS: 0.0055000000, PriceThroughput: 0.045056, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"ca-central-1\", Type: \"io1\", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"ca-central-1\", Type: \"io2\", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"ca-central-1\", Type: \"sc1\", PriceGB: 0.0168, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"ca-central-1\", Type: \"st1\", PriceGB: 0.05, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"eu-central-1\": {\n\t\t\"gp2\": {Region: \"eu-central-1\", Type: \"gp2\", PriceGB: 0.119, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"eu-central-1\", Type: \"gp3\", PriceGB: 0.0952, PriceIOPS: 0.0060000000, PriceThroughput: 0.048742400000000005, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"eu-central-1\", Type: \"io1\", PriceGB: 0.149, PriceIOPS: 0.0780000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"eu-central-1\", Type: \"io2\", PriceGB: 0.149, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"eu-central-1\", Type: \"sc1\", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"eu-central-1\", Type: \"st1\", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"eu-north-1\": {\n\t\t\"gp2\": {Region: \"eu-north-1\", Type: \"gp2\", PriceGB: 0.1045, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"eu-north-1\", Type: \"gp3\", PriceGB: 0.0836, PriceIOPS: 0.0052000000, PriceThroughput: 0.0428032, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"eu-north-1\", Type: \"io1\", PriceGB: 0.1311, PriceIOPS: 0.0684000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"eu-north-1\", Type: \"io2\", PriceGB: 0.1311, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"eu-north-1\", Type: \"sc1\", PriceGB: 0.01596, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"eu-north-1\", Type: \"st1\", PriceGB: 0.0475, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"eu-south-1\": {\n\t\t\"gp2\": {Region: \"eu-south-1\", Type: \"gp2\", PriceGB: 0.1155, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"eu-south-1\", Type: \"gp3\", PriceGB: 0.0924, PriceIOPS: 0.0058000000, PriceThroughput: 0.0473088, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"eu-south-1\", Type: \"io1\", PriceGB: 0.1449, PriceIOPS: 0.0756000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"eu-south-1\", Type: \"sc1\", PriceGB: 0.01764, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"eu-south-1\", Type: \"st1\", PriceGB: 0.0525, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"eu-west-1\": {\n\t\t\"gp2\": {Region: \"eu-west-1\", Type: \"gp2\", PriceGB: 0.11, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"eu-west-1\", Type: \"gp3\", PriceGB: 0.088, PriceIOPS: 0.0055000000, PriceThroughput: 0.045056, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"eu-west-1\", Type: \"io1\", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"eu-west-1\", Type: \"io2\", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"eu-west-1\", Type: \"sc1\", PriceGB: 0.0168, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"eu-west-1\", Type: \"st1\", PriceGB: 0.05, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"eu-west-2\": {\n\t\t\"gp2\": {Region: \"eu-west-2\", Type: \"gp2\", PriceGB: 0.116, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"eu-west-2\", Type: \"gp3\", PriceGB: 0.0928, PriceIOPS: 0.0058000000, PriceThroughput: 0.047513599999999996, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"eu-west-2\", Type: \"io1\", PriceGB: 0.145, PriceIOPS: 0.0760000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"eu-west-2\", Type: \"io2\", PriceGB: 0.145, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"eu-west-2\", Type: \"sc1\", PriceGB: 0.0174, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"eu-west-2\", Type: \"st1\", PriceGB: 0.053, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"eu-west-3\": {\n\t\t\"gp2\": {Region: \"eu-west-3\", Type: \"gp2\", PriceGB: 0.116, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"eu-west-3\", Type: \"gp3\", PriceGB: 0.0928, PriceIOPS: 0.0058000000, PriceThroughput: 0.047513599999999996, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"eu-west-3\", Type: \"io1\", PriceGB: 0.145, PriceIOPS: 0.0760000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"eu-west-3\", Type: \"sc1\", PriceGB: 0.0174, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"eu-west-3\", Type: \"st1\", PriceGB: 0.053, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"me-south-1\": {\n\t\t\"gp2\": {Region: \"me-south-1\", Type: \"gp2\", PriceGB: 0.121, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"me-south-1\", Type: \"gp3\", PriceGB: 0.0968, PriceIOPS: 0.0061000000, PriceThroughput: 0.0495616, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"me-south-1\", Type: \"io1\", PriceGB: 0.1518, PriceIOPS: 0.0792000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"me-south-1\", Type: \"io2\", PriceGB: 0.1518, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"me-south-1\", Type: \"sc1\", PriceGB: 0.01848, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"me-south-1\", Type: \"st1\", PriceGB: 0.055, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"sa-east-1\": {\n\t\t\"gp2\": {Region: \"sa-east-1\", Type: \"gp2\", PriceGB: 0.19, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"sa-east-1\", Type: \"gp3\", PriceGB: 0.152, PriceIOPS: 0.0095000000, PriceThroughput: 0.077824, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"sa-east-1\", Type: \"io1\", PriceGB: 0.238, PriceIOPS: 0.0910000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"sa-east-1\", Type: \"sc1\", PriceGB: 0.0288, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"sa-east-1\", Type: \"st1\", PriceGB: 0.086, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"us-east-1\": {\n\t\t\"gp2\": {Region: \"us-east-1\", Type: \"gp2\", PriceGB: 0.1, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"us-east-1\", Type: \"gp3\", PriceGB: 0.08, PriceIOPS: 0.0050000000, PriceThroughput: 0.04096, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"us-east-1\", Type: \"io1\", PriceGB: 0.125, PriceIOPS: 0.0650000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"us-east-1\", Type: \"io2\", PriceGB: 0.125, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"us-east-1\", Type: \"sc1\", PriceGB: 0.015, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"us-east-1\", Type: \"st1\", PriceGB: 0.045, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"us-east-2\": {\n\t\t\"gp2\": {Region: \"us-east-2\", Type: \"gp2\", PriceGB: 0.1, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"us-east-2\", Type: \"gp3\", PriceGB: 0.08, PriceIOPS: 0.0050000000, PriceThroughput: 0.04096, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"us-east-2\", Type: \"io1\", PriceGB: 0.125, PriceIOPS: 0.0650000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"us-east-2\", Type: \"io2\", PriceGB: 0.125, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"us-east-2\", Type: \"sc1\", PriceGB: 0.015, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"us-east-2\", Type: \"st1\", PriceGB: 0.045, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"us-gov-east-1\": {\n\t\t\"gp2\": {Region: \"us-gov-east-1\", Type: \"gp2\", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"us-gov-east-1\", Type: \"gp3\", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"us-gov-east-1\", Type: \"io1\", PriceGB: 0.15, PriceIOPS: 0.0780000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"us-gov-east-1\", Type: \"sc1\", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"us-gov-east-1\", Type: \"st1\", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"us-gov-west-1\": {\n\t\t\"gp2\": {Region: \"us-gov-west-1\", Type: \"gp2\", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"us-gov-west-1\", Type: \"gp3\", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"us-gov-west-1\", Type: \"io1\", PriceGB: 0.15, PriceIOPS: 0.0780000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"us-gov-west-1\", Type: \"sc1\", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"us-gov-west-1\", Type: \"st1\", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"us-west-1\": {\n\t\t\"gp2\": {Region: \"us-west-1\", Type: \"gp2\", PriceGB: 0.12, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"us-west-1\", Type: \"gp3\", PriceGB: 0.096, PriceIOPS: 0.0060000000, PriceThroughput: 0.049152, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"us-west-1\", Type: \"io1\", PriceGB: 0.138, PriceIOPS: 0.0720000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"us-west-1\", Type: \"io2\", PriceGB: 0.138, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"us-west-1\", Type: \"sc1\", PriceGB: 0.018, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"us-west-1\", Type: \"st1\", PriceGB: 0.054, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n\t\"us-west-2\": {\n\t\t\"gp2\": {Region: \"us-west-2\", Type: \"gp2\", PriceGB: 0.1, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"gp3\": {Region: \"us-west-2\", Type: \"gp3\", PriceGB: 0.08, PriceIOPS: 0.0050000000, PriceThroughput: 0.04096, IOPSConfigurable: true, ThroughputConfigurable: true},\n\t\t\"io1\": {Region: \"us-west-2\", Type: \"io1\", PriceGB: 0.125, PriceIOPS: 0.0650000000, PriceThroughput: 0, IOPSConfigurable: true, ThroughputConfigurable: false},\n\t\t\"io2\": {Region: \"us-west-2\", Type: \"io2\", PriceGB: 0.125, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"sc1\": {Region: \"us-west-2\", Type: \"sc1\", PriceGB: 0.015, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t\t\"st1\": {Region: \"us-west-2\", Type: \"st1\", PriceGB: 0.045, PriceIOPS: 0, PriceThroughput: 0, IOPSConfigurable: false, ThroughputConfigurable: false},\n\t},\n}\n\n// region -> EKS price\nvar EKSPrices = map[string]float64{\n\t\"af-south-1\":     0.1,\n\t\"ap-east-1\":      0.1,\n\t\"ap-northeast-1\": 0.1,\n\t\"ap-northeast-2\": 0.1,\n\t\"ap-northeast-3\": 0.1,\n\t\"ap-south-1\":     0.1,\n\t\"ap-southeast-1\": 0.1,\n\t\"ap-southeast-2\": 0.1,\n\t\"ca-central-1\":   0.1,\n\t\"eu-central-1\":   0.1,\n\t\"eu-north-1\":     0.1,\n\t\"eu-south-1\":     0.1,\n\t\"eu-west-1\":      0.1,\n\t\"eu-west-2\":      0.1,\n\t\"eu-west-3\":      0.1,\n\t\"me-south-1\":     0.1,\n\t\"sa-east-1\":      0.1,\n\t\"us-east-1\":      0.1,\n\t\"us-east-2\":      0.1,\n\t\"us-gov-east-1\":  0.1,\n\t\"us-gov-west-1\":  0.1,\n\t\"us-west-1\":      0.1,\n\t\"us-west-2\":      0.1,\n}\n"
  },
  {
    "path": "pkg/lib/aws/s3.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"bytes\"\n\t\"crypto/md5\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n\t\"github.com/aws/aws-sdk-go/aws/endpoints\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/aws/aws-sdk-go/service/s3/s3manager\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/msgpack\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst DefaultS3Region string = endpoints.UsWest2RegionID\n\nvar S3Regions strset.Set\n\nfunc init() {\n\tresolver := endpoints.DefaultResolver()\n\tpartitions := resolver.(endpoints.EnumPartitions).Partitions()\n\n\tS3Regions = strset.New()\n\n\tfor _, p := range partitions {\n\t\tif p.ID() == endpoints.AwsPartitionID || p.ID() == endpoints.AwsCnPartitionID {\n\t\t\tfor id := range p.Regions() {\n\t\t\t\tS3Regions.Add(id)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc S3Path(bucket string, key string) string {\n\treturn \"s3://\" + filepath.Join(bucket, key)\n}\n\nfunc JoinS3Path(paths ...string) string {\n\tif len(paths) == 0 {\n\t\treturn \"\"\n\t}\n\tpaths[0] = paths[0][5:]\n\treturn \"s3://\" + filepath.Join(paths...)\n}\n\nfunc SplitS3Path(s3Path string) (string, string, error) {\n\tif !IsValidS3Path(s3Path) {\n\t\treturn \"\", \"\", ErrorInvalidS3Path(s3Path)\n\t}\n\tfullPath := s3Path[len(\"s3://\"):]\n\tslashIndex := strings.Index(fullPath, \"/\")\n\tif slashIndex == -1 {\n\t\treturn fullPath, \"\", nil\n\t}\n\tbucket := fullPath[0:slashIndex]\n\tkey := fullPath[slashIndex+1:]\n\n\treturn bucket, key, nil\n}\n\nfunc SplitS3aPath(s3aPath string) (string, string, error) {\n\tif !IsValidS3aPath(s3aPath) {\n\t\treturn \"\", \"\", ErrorInvalidS3aPath(s3aPath)\n\t}\n\tfullPath := s3aPath[len(\"s3a://\"):]\n\tslashIndex := strings.Index(fullPath, \"/\")\n\tbucket := fullPath[0:slashIndex]\n\tkey := fullPath[slashIndex+1:]\n\n\treturn bucket, key, nil\n}\n\nfunc IsValidS3Path(s3Path string) bool {\n\tif !strings.HasPrefix(s3Path, \"s3://\") {\n\t\treturn false\n\t}\n\tparts := strings.Split(s3Path[5:], \"/\")\n\tif len(parts) == 0 {\n\t\treturn false\n\t}\n\tif parts[0] == \"\" {\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc IsValidS3aPath(s3aPath string) bool {\n\tif !strings.HasPrefix(s3aPath, \"s3a://\") {\n\t\treturn false\n\t}\n\tparts := strings.Split(s3aPath[6:], \"/\")\n\tif len(parts) < 2 {\n\t\treturn false\n\t}\n\tif parts[0] == \"\" || parts[1] == \"\" {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// List all S3 objects that are \"depth\" levels or deeper than the given \"s3Path\".\n// Setting depth to 1 effectively translates to listing the objects one level or deeper than the given prefix (aka listing the directory contents).\n//\n// 1st returned value is the list of paths found at level <depth> or deeper.\n// 2nd returned value is the list of paths found at all levels.\nfunc (c *Client) GetNLevelsDeepFromS3Path(s3Path string, depth int, includeDirObjects bool, maxResults *int64) ([]string, []string, error) {\n\tpaths := strset.New()\n\n\t_, key, err := SplitS3Path(s3Path)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tallS3Objects, err := c.ListS3PathDir(s3Path, includeDirObjects, maxResults, nil)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tallPaths := ConvertS3ObjectsToKeys(allS3Objects...)\n\n\tkeySplit := slices.RemoveEmpties(strings.Split(key, \"/\"))\n\tfor _, path := range allPaths {\n\t\tpathSplit := slices.RemoveEmpties(strings.Split(path, \"/\"))\n\t\tif len(pathSplit)-len(keySplit) >= depth {\n\t\t\tcomputedPath := strings.Join(pathSplit[:len(keySplit)+depth], \"/\")\n\t\t\tpaths.Add(computedPath)\n\t\t}\n\t}\n\n\treturn paths.Slice(), allPaths, nil\n}\n\n// Efficient way to list all top-level directory prefixes on a bucket.\n//\n// Warning: when using this function, make sure the \"~\" character isn't used in any of the S3 objects.\nfunc (c *Client) ListS3TopLevelDirs(bucket string) ([]string, error) {\n\t// find first top-level directory\n\ts3Objects, err := c.ListS3Prefix(bucket, \"\", false, pointer.Int64(1), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdirs := []string{}\n\tfor _, prefix := range ConvertS3ObjectsToKeys(s3Objects...) {\n\t\tdirs = append(dirs, files.GetTopLevelDirectory(prefix))\n\t}\n\n\t// detect all remaining top-level dirs\n\tfor {\n\t\tif len(dirs) == 0 {\n\t\t\tbreak\n\t\t}\n\t\tpreviousDir := dirs[len(dirs)-1]\n\t\ts3Objects, err := c.ListS3Prefix(\n\t\t\tbucket,\n\t\t\t\"\",\n\t\t\ttrue,\n\t\t\tpointer.Int64(1),\n\t\t\tpointer.String(filepath.Join(previousDir, \"~~~\")),\n\t\t)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif len(ConvertS3ObjectsToKeys(s3Objects...)) == 0 {\n\t\t\tbreak\n\t\t}\n\t\tfor _, prefix := range ConvertS3ObjectsToKeys(s3Objects...) {\n\t\t\tdirs = append(dirs, files.GetTopLevelDirectory(prefix))\n\t\t}\n\t}\n\n\treturn dirs, nil\n}\n\nfunc ConvertS3ObjectsToKeys(s3Objects ...*s3.Object) []string {\n\tpaths := make([]string, 0, len(s3Objects))\n\tfor _, object := range s3Objects {\n\t\tif object != nil {\n\t\t\tpaths = append(paths, *object.Key)\n\t\t}\n\t}\n\treturn paths\n}\n\nfunc GetBucketRegionFromS3Path(s3Path string) (string, error) {\n\tbucket, _, err := SplitS3Path(s3Path)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn GetBucketRegion(bucket)\n}\n\nfunc GetBucketRegion(bucket string) (string, error) {\n\tsess := session.Must(session.NewSession()) // credentials are not necessary for this request, and will not be used\n\tregion, err := s3manager.GetBucketRegion(aws.BackgroundContext(), sess, bucket, endpoints.UsWest2RegionID)\n\tif err != nil {\n\t\treturn \"\", ErrorBucketNotFound(bucket)\n\t}\n\treturn region, nil\n}\n\nfunc (c *Client) IsS3PathFile(s3Path string, s3Paths ...string) (bool, error) {\n\tallS3Paths := append(s3Paths, s3Path)\n\tfor _, s3Path := range allS3Paths {\n\t\tbucket, prefix, err := SplitS3Path(s3Path)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\texists, err := c.IsS3File(bucket, prefix)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\tif !exists {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) IsS3File(bucket string, fileKey string, fileKeys ...string) (bool, error) {\n\tallFileKeys := append(fileKeys, fileKey)\n\tfor _, key := range allFileKeys {\n\t\t_, err := c.S3().HeadObject(&s3.HeadObjectInput{\n\t\t\tBucket: aws.String(bucket),\n\t\t\tKey:    aws.String(key),\n\t\t})\n\n\t\tif IsNotFoundErr(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\tif err != nil {\n\t\t\treturn false, errors.Wrap(err, S3Path(bucket, key))\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) IsS3PathPrefix(s3Path string, s3Paths ...string) (bool, error) {\n\tallS3Paths := append(s3Paths, s3Path)\n\tfor _, s3Path := range allS3Paths {\n\t\tbucket, prefix, err := SplitS3Path(s3Path)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\texists, err := c.IsS3Prefix(bucket, prefix)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\tif !exists {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) IsS3Prefix(bucket string, prefix string, prefixes ...string) (bool, error) {\n\tallPrefixes := append(prefixes, prefix)\n\tfor _, prefix := range allPrefixes {\n\t\tout, err := c.S3().ListObjectsV2(&s3.ListObjectsV2Input{\n\t\t\tBucket:  aws.String(bucket),\n\t\t\tPrefix:  aws.String(prefix),\n\t\t\tMaxKeys: aws.Int64(1),\n\t\t})\n\n\t\tif err != nil {\n\t\t\treturn false, errors.Wrap(err, S3Path(bucket, prefix))\n\t\t}\n\n\t\tif *out.KeyCount == 0 {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) IsS3PathDir(s3Path string, s3Paths ...string) (bool, error) {\n\tallS3Paths := append(s3Paths, s3Path)\n\tfor _, s3Path := range allS3Paths {\n\t\tbucket, prefix, err := SplitS3Path(s3Path)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\texists, err := c.IsS3Dir(bucket, prefix)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\tif !exists {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) IsS3Dir(bucket string, dirPath string, dirPaths ...string) (bool, error) {\n\tfullDirPaths := make([]string, len(dirPaths)+1)\n\tallDirPaths := append(dirPaths, dirPath)\n\tfor i, dirPath := range allDirPaths {\n\t\tfullDirPaths[i] = s.EnsureSuffix(dirPath, \"/\")\n\t}\n\treturn c.IsS3Prefix(bucket, fullDirPaths[0], fullDirPaths[1:]...)\n}\n\n// Checks bucket existence and accessibility with credentials\nfunc (c *Client) DoesBucketExist(bucket string) (bool, error) {\n\t_, err := c.S3().HeadBucket(&s3.HeadBucketInput{\n\t\tBucket: aws.String(bucket),\n\t})\n\tif err != nil {\n\t\tif aerr, ok := err.(awserr.Error); ok {\n\t\t\tswitch aerr.Code() {\n\t\t\tcase \"NotFound\":\n\t\t\t\treturn false, nil\n\t\t\tcase \"Forbidden\":\n\t\t\t\treturn false, ErrorBucketInaccessible(bucket)\n\t\t\t}\n\t\t}\n\t\treturn false, errors.Wrap(err, \"bucket \"+bucket)\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) CreateBucket(bucket string) error {\n\tvar bucketConfiguration *s3.CreateBucketConfiguration\n\tif c.Region != \"us-east-1\" {\n\t\tbucketConfiguration = &s3.CreateBucketConfiguration{\n\t\t\tLocationConstraint: aws.String(c.Region),\n\t\t}\n\t}\n\t_, err := c.S3().CreateBucket(&s3.CreateBucketInput{\n\t\tBucket:                    aws.String(bucket),\n\t\tCreateBucketConfiguration: bucketConfiguration,\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"creating bucket \"+bucket)\n\t}\n\treturn nil\n}\n\nfunc (c *Client) EnableBucketEncryption(bucket string) error {\n\t_, err := c.S3().PutBucketEncryption(&s3.PutBucketEncryptionInput{\n\t\tBucket: aws.String(bucket),\n\t\tServerSideEncryptionConfiguration: &s3.ServerSideEncryptionConfiguration{\n\t\t\tRules: []*s3.ServerSideEncryptionRule{\n\t\t\t\t{\n\t\t\t\t\tApplyServerSideEncryptionByDefault: &s3.ServerSideEncryptionByDefault{\n\t\t\t\t\t\tSSEAlgorithm: pointer.String(\"AES256\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"enabling encryption for bucket \"+bucket)\n\t}\n\treturn nil\n}\n\nfunc (c *Client) UploadReaderToS3(data io.Reader, bucket string, key string) error {\n\t_, err := c.S3Uploader().Upload(&s3manager.UploadInput{\n\t\tBucket:             aws.String(bucket),\n\t\tKey:                aws.String(key),\n\t\tBody:               data,\n\t\tACL:                aws.String(\"private\"),\n\t\tContentDisposition: aws.String(\"attachment\"),\n\t})\n\n\tif err != nil {\n\t\treturn errors.Wrap(err, S3Path(bucket, key))\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) UploadFileToS3(path string, bucket string, key string) error {\n\tfile, err := files.Open(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer file.Close()\n\treturn c.UploadReaderToS3(file, bucket, key)\n}\n\nfunc (c *Client) UploadBytesToS3(data []byte, bucket string, key string) error {\n\treturn c.UploadReaderToS3(bytes.NewReader(data), bucket, key)\n}\n\nfunc (c *Client) UploadStringToS3(str string, bucket string, key string) error {\n\treturn c.UploadReaderToS3(strings.NewReader(str), bucket, key)\n}\n\nfunc (c *Client) UploadJSONToS3(obj interface{}, bucket string, key string) error {\n\tjsonBytes, err := json.Marshal(obj)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn c.UploadBytesToS3(jsonBytes, bucket, key)\n}\n\nfunc (c *Client) UploadMsgpackToS3(obj interface{}, bucket string, key string) error {\n\tmsgpackBytes, err := msgpack.Marshal(obj)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn c.UploadBytesToS3(msgpackBytes, bucket, key)\n}\n\nfunc (c *Client) CreateEmptyS3File(bucket string, key string) error {\n\treturn c.UploadReaderToS3(bytes.NewReader(nil), bucket, key)\n}\n\nfunc (c *Client) UploadDirToS3(localDirPath string, bucket string, s3Dir string, ignoreFns ...files.IgnoreFn) error {\n\tlocalRelPaths, err := files.ListDirRecursive(localDirPath, true, ignoreFns...)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, localRelPath := range localRelPaths {\n\t\tlocalPath := filepath.Join(localDirPath, localRelPath)\n\t\tkey := filepath.Join(s3Dir, localRelPath)\n\t\tif err := c.UploadFileToS3(localPath, bucket, key); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// returned io.ReadCloser should be closed by the caller\nfunc (c *Client) ReadReaderFromS3(bucket string, key string) (io.ReadCloser, error) {\n\t// for reading into memory, s3.S3.GetObject() seems faster than s3manager.Downloader.Download() with aws.NewWriteAtBuffer([]byte{})\n\tresponse, err := c.S3().GetObject(&s3.GetObjectInput{\n\t\tKey:    aws.String(key),\n\t\tBucket: aws.String(bucket),\n\t})\n\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, S3Path(bucket, key))\n\t}\n\n\treturn response.Body, nil\n}\n\nfunc (c *Client) ReadBufferFromS3(bucket string, key string) (*bytes.Buffer, error) {\n\treader, err := c.ReadReaderFromS3(bucket, key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer reader.Close()\n\n\tbuf := new(bytes.Buffer)\n\tbuf.ReadFrom(reader)\n\n\treturn buf, nil\n}\n\nfunc (c *Client) ReadBytesFromS3(bucket string, key string) ([]byte, error) {\n\tbuf, err := c.ReadBufferFromS3(bucket, key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn buf.Bytes(), nil\n}\n\nfunc (c *Client) ReadStringFromS3(bucket string, key string) (string, error) {\n\tbuf, err := c.ReadBufferFromS3(bucket, key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn buf.String(), nil\n}\n\nfunc (c *Client) ReadJSONFromS3(objPtr interface{}, bucket string, key string) error {\n\tjsonBytes, err := c.ReadBytesFromS3(bucket, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn errors.Wrap(json.Unmarshal(jsonBytes, objPtr), S3Path(bucket, key))\n}\n\nfunc (c *Client) ReadMsgpackFromS3(objPtr interface{}, bucket string, key string) error {\n\tmsgpackBytes, err := c.ReadBytesFromS3(bucket, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn errors.Wrap(msgpack.Unmarshal(msgpackBytes, objPtr), S3Path(bucket, key))\n}\n\nfunc (c *Client) ReadStringFromS3Path(s3Path string) (string, error) {\n\tbucket, key, err := SplitS3Path(s3Path)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn c.ReadStringFromS3(bucket, key)\n}\n\nfunc (c *Client) ReadBytesFromS3Path(s3Path string) ([]byte, error) {\n\tbucket, key, err := SplitS3Path(s3Path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.ReadBytesFromS3(bucket, key)\n}\n\nfunc (c *Client) ReadMsgpackFromS3Path(objPtr interface{}, s3Path string) error {\n\tbucket, key, err := SplitS3Path(s3Path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn c.ReadMsgpackFromS3(objPtr, bucket, key)\n}\n\n// overwrites existing file\nfunc (c *Client) DownloadFileFromS3(bucket string, key string, localPath string) error {\n\tfile, err := files.Create(localPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer file.Close()\n\n\t// for downloading files, s3manager.Downloader.Download() is faster than s3.S3.GetObject()\n\t_, err = c.S3Downloader().Download(file, &s3.GetObjectInput{\n\t\tBucket: aws.String(bucket),\n\t\tKey:    aws.String(key),\n\t})\n\n\tif err != nil {\n\t\treturn errors.Wrap(err, S3Path(bucket, key))\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) DownloadDirFromS3(bucket string, s3Dir string, localDirPath string, shouldTrimDirPrefix bool, maxFiles *int64) error {\n\tprefix := s.EnsureSuffix(s3Dir, \"/\")\n\treturn c.DownloadPrefixFromS3(bucket, prefix, localDirPath, shouldTrimDirPrefix, maxFiles)\n}\n\n// if shouldTrimDirPrefix is true, the directory path of prefix will be trimmed when downloading files\n//\n// example:\n// s3: [test/dir/1.txt, test/dir2/2.txt, test/directions.txt]\n// localDirPath: ~/downloads\n//\n// shouldTrimDirPrefix = true\n//   prefix: \"test/dir\"\n//   result: [~/downloads/dir/1.txt, ~/downloads/dir2/1.txt, ~/downloads/directions.txt]\n//\n//   prefix: \"test/dir/\"\n//   result: [~/downloads/1.txt]\n//\n// shouldTrimDirPrefix = false\n//   prefix: \"test/dir\"\n//   result: [~/downloads/test/dir/1.txt, ~/downloads/test/dir2/1.txt, ~/downloads/test/directions.txt]\n//\n//   prefix: \"test/dir/\"\n//   result: [~/downloads/test/dir/1.txt]\nfunc (c *Client) DownloadPrefixFromS3(bucket string, prefix string, localDirPath string, shouldTrimDirPrefix bool, maxFiles *int64) error {\n\tif _, err := files.CreateDirIfMissing(localDirPath); err != nil {\n\t\treturn err\n\t}\n\tcreatedDirs := strset.New(localDirPath)\n\n\tvar trimPrefix string\n\tif shouldTrimDirPrefix {\n\t\tlastIndex := strings.LastIndex(prefix, \"/\")\n\t\tif lastIndex == -1 {\n\t\t\ttrimPrefix = \"\"\n\t\t} else {\n\t\t\ttrimPrefix = prefix[:lastIndex+1]\n\t\t}\n\t}\n\n\terr := c.S3Iterator(bucket, prefix, true, maxFiles, nil, func(object *s3.Object) (bool, error) {\n\t\tlocalRelPath := *object.Key\n\t\tif shouldTrimDirPrefix {\n\t\t\tlocalRelPath = strings.TrimPrefix(localRelPath, trimPrefix)\n\t\t}\n\n\t\tlocalPath := filepath.Join(localDirPath, localRelPath)\n\n\t\t// check for directory objects\n\t\tif strings.HasSuffix(*object.Key, \"/\") {\n\t\t\tif !createdDirs.Has(localPath) {\n\t\t\t\tif _, err := files.CreateDirIfMissing(localPath); err != nil {\n\t\t\t\t\treturn false, err\n\t\t\t\t}\n\t\t\t\tcreatedDirs.Add(localPath)\n\t\t\t}\n\t\t\treturn true, nil\n\t\t}\n\n\t\tlocalDir := filepath.Dir(localPath)\n\t\tif !createdDirs.Has(localDir) {\n\t\t\tif _, err := files.CreateDirIfMissing(localDir); err != nil {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\t\tcreatedDirs.Add(localDir)\n\t\t}\n\n\t\tif err := c.DownloadFileFromS3(bucket, *object.Key, localPath); err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\treturn true, nil\n\t})\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) S3FileIterator(bucket string, s3Obj *s3.Object, partSize int, fn func(buffer io.ReadCloser, isLastPart bool) (bool, error)) error {\n\tsize := int(*s3Obj.Size)\n\n\titers := size / partSize\n\tif size%partSize != 0 {\n\t\titers++\n\t}\n\n\tfor i := 0; i < iters; i++ {\n\t\tmin := i * (partSize)\n\t\tmax := (i + 1) * (partSize)\n\t\tif max > size {\n\t\t\tmax = size\n\t\t}\n\t\tmax--\n\n\t\tbyteRange := fmt.Sprintf(\"bytes=%d-%d\", min, max)\n\t\tobj, err := c.S3().GetObject(&s3.GetObjectInput{\n\t\t\tBucket: aws.String(bucket),\n\t\t\tKey:    s3Obj.Key,\n\t\t\tRange:  aws.String(byteRange), // use range instead of part numbers because only files uploaded using multipart have parts\n\t\t})\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, S3Path(bucket, *s3Obj.Key), \"range \"+byteRange)\n\t\t}\n\n\t\tisLastChunk := i+1 == iters\n\t\tshouldContinue, err := fn(obj.Body, isLastChunk)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, S3Path(bucket, *s3Obj.Key))\n\t\t}\n\n\t\tif !shouldContinue {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) ListS3Dir(bucket string, s3Dir string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) {\n\tprefix := s.EnsureSuffix(s3Dir, \"/\")\n\treturn c.ListS3Prefix(bucket, prefix, includeDirObjects, maxResults, startAfter)\n}\n\nfunc (c *Client) ListS3PathDir(s3DirPath string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) {\n\ts3Path := s.EnsureSuffix(s3DirPath, \"/\")\n\treturn c.ListS3PathPrefix(s3Path, includeDirObjects, maxResults, startAfter)\n}\n\n// This behaves like you'd expect `ls` to behave on a local filesystem\n// \"directory\" names will be returned even if S3 directory objects don't exist\nfunc (c *Client) ListS3DirOneLevel(bucket string, s3Dir string, maxResults *int64, startAfter *string) ([]string, error) {\n\ts3Dir = s.EnsureSuffix(s3Dir, \"/\")\n\n\tallNames := strset.New()\n\n\terr := c.S3Iterator(bucket, s3Dir, true, nil, startAfter, func(object *s3.Object) (bool, error) {\n\t\trelativePath := strings.TrimPrefix(*object.Key, s3Dir)\n\t\toneLevelPath := strings.Split(relativePath, \"/\")[0]\n\t\tallNames.Add(oneLevelPath)\n\n\t\tif maxResults != nil && int64(len(allNames)) >= *maxResults {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn true, nil\n\t})\n\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, S3Path(bucket, s3Dir))\n\t}\n\n\treturn allNames.SliceSorted(), nil\n}\n\nfunc (c *Client) ListS3Prefix(bucket string, prefix string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) {\n\tvar allObjects []*s3.Object\n\n\terr := c.S3BatchIterator(bucket, prefix, includeDirObjects, maxResults, startAfter, func(objects []*s3.Object) (bool, error) {\n\t\tallObjects = append(allObjects, objects...)\n\t\treturn true, nil\n\t})\n\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, S3Path(bucket, prefix))\n\t}\n\n\treturn allObjects, nil\n}\n\nfunc (c *Client) ListS3PathPrefix(s3Path string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) {\n\tbucket, prefix, err := SplitS3Path(s3Path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.ListS3Prefix(bucket, prefix, includeDirObjects, maxResults, startAfter)\n}\n\nfunc (c *Client) DeleteS3File(bucket string, key string) error {\n\t_, err := c.S3().DeleteObject(\n\t\t&s3.DeleteObjectInput{\n\t\t\tBucket: aws.String(bucket),\n\t\t\tKey:    aws.String(key),\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\treturn nil\n}\n\nfunc (c *Client) DeleteS3Dir(bucket string, s3Dir string, continueIfFailure bool) error {\n\tprefix := s.EnsureSuffix(s3Dir, \"/\")\n\treturn c.DeleteS3Prefix(bucket, prefix, continueIfFailure)\n}\n\nfunc (c *Client) DeleteS3Prefix(bucket string, prefix string, continueIfFailure bool) error {\n\t// This implementation is confirmed to be considerably faster than using s3manager.NewDeleteListIterator() + s3manager.NewBatchDeleteWithClient()\n\terr := c.S3BatchIterator(bucket, prefix, true, nil, nil, func(objects []*s3.Object) (bool, error) {\n\t\tdeleteObjects := make([]*s3.ObjectIdentifier, len(objects))\n\t\tfor i, object := range objects {\n\t\t\tdeleteObjects[i] = &s3.ObjectIdentifier{Key: object.Key}\n\t\t}\n\n\t\t_, err := c.S3().DeleteObjects(&s3.DeleteObjectsInput{\n\t\t\tBucket: aws.String(bucket),\n\t\t\tDelete: &s3.Delete{\n\t\t\t\tObjects: deleteObjects,\n\t\t\t\tQuiet:   aws.Bool(true),\n\t\t\t},\n\t\t})\n\n\t\tif err != nil {\n\t\t\terr := errors.Wrap(err, S3Path(bucket, prefix))\n\t\t\tif !continueIfFailure {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\t\treturn true, err\n\t\t}\n\n\t\treturn true, nil\n\t})\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) HashS3Dir(bucket string, prefix string, maxResults *int64, startAfter *string) (string, error) {\n\tmd5Hash := md5.New()\n\n\terr := c.S3BatchIterator(bucket, prefix, true, maxResults, startAfter, func(objects []*s3.Object) (bool, error) {\n\t\tvar subErr error\n\t\tfor _, object := range objects {\n\t\t\tio.WriteString(md5Hash, *object.ETag)\n\t\t}\n\t\treturn true, subErr\n\t})\n\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn hex.EncodeToString(md5Hash.Sum(nil)), nil\n}\n\n// Directory objects are empty objects ending in \"/\". They are not guaranteed to exists, and there may or may not be files \"in\" the directory\nfunc (c *Client) S3Iterator(bucket string, prefix string, includeDirObjects bool, maxResults *int64, startAfter *string, fn func(*s3.Object) (bool, error)) error {\n\terr := c.S3BatchIterator(bucket, prefix, includeDirObjects, maxResults, startAfter, func(objects []*s3.Object) (bool, error) {\n\t\tvar subErr error\n\t\tfor _, object := range objects {\n\t\t\tshouldContinue, newSubErr := fn(object)\n\t\t\tif newSubErr != nil {\n\t\t\t\tsubErr = newSubErr\n\t\t\t}\n\t\t\tif !shouldContinue {\n\t\t\t\treturn false, subErr\n\t\t\t}\n\t\t}\n\t\treturn true, subErr\n\t})\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// The return value of fn([]*s3.Object) (bool, error) should be whether to continue iterating, and an error (if any occurred)\n// Directory objects are empty objects ending in \"/\". They are not guaranteed to exists, and there may or may not be files \"in\" the directory\nfunc (c *Client) S3BatchIterator(bucket string, prefix string, includeDirObjects bool, maxResults *int64, startAfter *string, fn func([]*s3.Object) (bool, error)) error {\n\tvar maxResultsRemaining *int64\n\tif maxResults != nil {\n\t\tmaxResultsRemaining = pointer.Int64(*maxResults)\n\t}\n\n\tlistObjectsInput := &s3.ListObjectsV2Input{\n\t\tBucket:     aws.String(bucket),\n\t\tPrefix:     aws.String(prefix),\n\t\tMaxKeys:    maxResultsRemaining,\n\t\tStartAfter: startAfter,\n\t}\n\n\tvar numSeen int64\n\tvar subErr error\n\n\terr := c.S3().ListObjectsV2Pages(listObjectsInput,\n\t\tfunc(listObjectsOutput *s3.ListObjectsV2Output, lastPage bool) bool {\n\t\t\tobjects := listObjectsOutput.Contents\n\n\t\t\t// filter directory objects (https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating)\n\t\t\tif !includeDirObjects {\n\t\t\t\tfiltered := objects[:0]\n\t\t\t\tfor _, object := range objects {\n\t\t\t\t\tif !strings.HasSuffix(*object.Key, \"/\") {\n\t\t\t\t\t\tfiltered = append(filtered, object)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tobjects = filtered\n\t\t\t}\n\n\t\t\tif len(objects) == 0 {\n\t\t\t\treturn true\n\t\t\t}\n\n\t\t\tshouldContinue, newSubErr := fn(objects)\n\t\t\tif newSubErr != nil {\n\t\t\t\tsubErr = newSubErr\n\t\t\t}\n\t\t\tif !shouldContinue {\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tnumSeen += int64(len(objects))\n\n\t\t\tif maxResults != nil {\n\t\t\t\tif numSeen >= *maxResults {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\t*maxResultsRemaining = *maxResults - numSeen\n\t\t\t}\n\n\t\t\treturn true\n\t\t})\n\n\tif subErr != nil {\n\t\treturn subErr\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) TagBucket(bucket string, tagMap map[string]string) error {\n\tvar tagSet []*s3.Tag\n\tfor key, value := range tagMap {\n\t\ttagSet = append(tagSet, &s3.Tag{\n\t\t\tKey:   aws.String(key),\n\t\t\tValue: aws.String(value),\n\t\t})\n\t}\n\n\t_, err := c.S3().PutBucketTagging(\n\t\t&s3.PutBucketTaggingInput{\n\t\t\tBucket: aws.String(bucket),\n\t\t\tTagging: &s3.Tagging{\n\t\t\t\tTagSet: tagSet,\n\t\t\t},\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to add tags to bucket\", bucket)\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) SetLifecycleRules(bucket string, rules []s3.LifecycleRule) error {\n\tpointerRules := []*s3.LifecycleRule{}\n\tfor i := range rules {\n\t\tpointerRules = append(pointerRules, &rules[i])\n\t}\n\t_, err := c.S3().PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{\n\t\tBucket: pointer.String(bucket),\n\t\tLifecycleConfiguration: &s3.BucketLifecycleConfiguration{\n\t\t\tRules: pointerRules,\n\t\t},\n\t})\n\n\treturn errors.WithStack(err)\n}\n\nfunc (c *Client) GetLifecycleRules(bucket string) ([]s3.LifecycleRule, error) {\n\tlifecycleOutput, err := c.S3().GetBucketLifecycleConfiguration(&s3.GetBucketLifecycleConfigurationInput{\n\t\tBucket: pointer.String(bucket),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif lifecycleOutput == nil {\n\t\treturn nil, nil\n\t}\n\n\ts3Rules := []s3.LifecycleRule{}\n\tfor _, rule := range lifecycleOutput.Rules {\n\t\tif rule != nil {\n\t\t\ts3Rules = append(s3Rules, *rule)\n\t\t}\n\t}\n\n\treturn s3Rules, nil\n}\n\nfunc (c *Client) DeleteLifecycleRules(bucket string) error {\n\t_, err := c.S3().DeleteBucketLifecycle(&s3.DeleteBucketLifecycleInput{\n\t\tBucket: pointer.String(bucket),\n\t})\n\treturn errors.WithStack(err)\n}\n"
  },
  {
    "path": "pkg/lib/aws/servicequotas.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/servicequotas\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\nvar _standardInstanceFamilies = strset.New(\"a\", \"c\", \"d\", \"h\", \"i\", \"m\", \"r\", \"t\", \"z\")\nvar _knownInstanceFamilies = strset.Union(_standardInstanceFamilies, strset.New(\"p\", \"g\", \"inf\", \"x\", \"f\", \"mac\", \"vt\", \"dl\", \"trn\", \"im\", \"is\", \"hpc\"))\n\nconst (\n\t// 11 inbound rules\n\t_baseInboundRulesForNodeGroup = 11\n\t_inboundRulesPerAZ            = 8\n\t// ClusterSharedNodeSecurityGroup, ControlPlaneSecurityGroup, eks-cluster-sg-<cluster-name>, and operator security group\n\t_baseNumberOfSecurityGroups = 4\n)\n\ntype InstanceTypeRequests struct {\n\tInstanceType              string\n\tRequiredOnDemandInstances int64\n\tRequiredSpotInstances     int64\n}\n\ntype instanceClassRequest struct {\n\tInstanceClass string\n\n\tInstanceTypes strset.Set // the instance class (e.g. \"standard\") can have multiple instance types\n\n\tRequiredOnDemandCPUs int64\n\tRequiredSpotCPUs     int64\n\n\tOnDemandCPUQuota  *int64\n\tOnDemandQuotaCode string\n\n\tSpotCPUQuota  *int64\n\tSpotQuotaCode string\n}\n\nfunc (c *Client) VerifyInstanceQuota(instances []InstanceTypeRequests) error {\n\tinstanceClassRequests := []instanceClassRequest{}\n\tfor _, instance := range instances {\n\t\tif instance.RequiredOnDemandInstances == 0 && instance.RequiredSpotInstances == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tparsedType, err := ParseInstanceType(instance.InstanceType)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\t// Allow the instance if we don't recognize the type\n\t\tif !_knownInstanceFamilies.Has(parsedType.Family) {\n\t\t\tcontinue\n\t\t}\n\n\t\tinstanceClass := parsedType.Family\n\t\tif _standardInstanceFamilies.Has(parsedType.Family) {\n\t\t\tinstanceClass = \"standard\"\n\t\t}\n\n\t\tcpusPerInstance := InstanceMetadatas[c.Region][instance.InstanceType].CPU\n\n\t\tinstanceClassFound := false\n\t\tfor idx, r := range instanceClassRequests {\n\t\t\tif r.InstanceClass == instanceClass {\n\t\t\t\tinstanceClassRequests[idx].InstanceTypes.Add(instance.InstanceType)\n\t\t\t\tinstanceClassRequests[idx].RequiredOnDemandCPUs += instance.RequiredOnDemandInstances * cpusPerInstance.Value()\n\t\t\t\tinstanceClassRequests[idx].RequiredSpotCPUs += instance.RequiredSpotInstances * cpusPerInstance.Value()\n\n\t\t\t\tinstanceClassFound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif !instanceClassFound {\n\t\t\tinstanceClassRequests = append(instanceClassRequests, instanceClassRequest{\n\t\t\t\tInstanceClass:        instanceClass,\n\t\t\t\tInstanceTypes:        strset.New(instance.InstanceType),\n\t\t\t\tRequiredOnDemandCPUs: instance.RequiredOnDemandInstances * cpusPerInstance.Value(),\n\t\t\t\tRequiredSpotCPUs:     instance.RequiredSpotInstances * cpusPerInstance.Value(),\n\t\t\t})\n\t\t}\n\t}\n\n\terr := c.ServiceQuotas().ListServiceQuotasPages(\n\t\t&servicequotas.ListServiceQuotasInput{\n\t\t\tServiceCode: aws.String(\"ec2\"),\n\t\t},\n\t\tfunc(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool {\n\t\t\tif page == nil {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tfor _, quota := range page.Quotas {\n\t\t\t\tif quota == nil || quota.UsageMetric == nil || len(quota.UsageMetric.MetricDimensions) == 0 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tmetricClass, ok := quota.UsageMetric.MetricDimensions[\"Class\"]\n\t\t\t\tif !ok || metricClass == nil || !(strings.HasSuffix(*metricClass, \"/OnDemand\") || strings.HasSuffix(*metricClass, \"/Spot\")) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tfor idx, r := range instanceClassRequests {\n\t\t\t\t\t// quota is specified in number of vCPU permitted per family\n\t\t\t\t\tif strings.ToLower(*metricClass) == r.InstanceClass+\"/ondemand\" {\n\t\t\t\t\t\tinstanceClassRequests[idx].OnDemandCPUQuota = pointer.Int64(int64(*quota.Value))\n\t\t\t\t\t\tinstanceClassRequests[idx].OnDemandQuotaCode = *quota.QuotaCode\n\t\t\t\t\t} else if strings.ToLower(*metricClass) == r.InstanceClass+\"/spot\" {\n\t\t\t\t\t\tinstanceClassRequests[idx].SpotCPUQuota = pointer.Int64(int64(*quota.Value))\n\t\t\t\t\t\tinstanceClassRequests[idx].SpotQuotaCode = *quota.QuotaCode\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\tfor _, r := range instanceClassRequests {\n\t\tif r.OnDemandCPUQuota != nil && *r.OnDemandCPUQuota < r.RequiredOnDemandCPUs {\n\t\t\treturn ErrorInsufficientInstanceQuota(r.InstanceTypes.Slice(), \"on-demand\", c.Region, r.RequiredOnDemandCPUs, *r.OnDemandCPUQuota, r.OnDemandQuotaCode)\n\t\t}\n\t\tif r.SpotCPUQuota != nil && *r.SpotCPUQuota < r.RequiredSpotCPUs {\n\t\t\treturn ErrorInsufficientInstanceQuota(r.InstanceTypes.Slice(), \"spot\", c.Region, r.RequiredSpotCPUs, *r.SpotCPUQuota, r.SpotQuotaCode)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) ListServiceQuotas(quotaCodes []string, serviceCodes []string) (map[string]int, error) {\n\tdesiredQuotaCodes := strset.New(quotaCodes...)\n\tquotaCodeToValueMap := map[string]int{}\n\n\tfor _, serviceCode := range serviceCodes {\n\t\terr := c.ServiceQuotas().ListServiceQuotasPages(\n\t\t\t&servicequotas.ListServiceQuotasInput{\n\t\t\t\tServiceCode: aws.String(serviceCode),\n\t\t\t},\n\t\t\tfunc(page *servicequotas.ListServiceQuotasOutput, lastPage bool) bool {\n\t\t\t\tif page == nil {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\tfor _, quota := range page.Quotas {\n\t\t\t\t\tif quota == nil || quota.QuotaCode == nil || quota.Value == nil {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tif desiredQuotaCodes.Has(*quota.QuotaCode) {\n\t\t\t\t\t\tquotaCodeToValueMap[*quota.QuotaCode] = int(*quota.Value)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn true\n\t\t\t},\n\t\t)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, serviceCode)\n\t\t}\n\t}\n\n\treturn quotaCodeToValueMap, nil\n}\n\nfunc (c *Client) VerifyInternetGatewayQuota(internetGatewayQuota int, requiredInternetGateways int) error {\n\tinternetGatewaysInUse, err := c.ListInternetGateways()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tadditionalQuotaRequired := len(internetGatewaysInUse) + requiredInternetGateways - internetGatewayQuota\n\n\tif additionalQuotaRequired > 0 {\n\t\treturn ErrorInternetGatewayLimitExceeded(internetGatewayQuota, additionalQuotaRequired, c.Region)\n\t}\n\treturn nil\n}\n\nfunc (c *Client) VerifyNATGatewayQuota(natGatewayQuota int, availabilityZones strset.Set, highlyAvailableNATGateway bool) error {\n\t// get NAT GW in use per selected AZ\n\tnatGateways, err := c.DescribeNATGateways()\n\tif err != nil {\n\t\treturn err\n\t}\n\tsubnets, err := c.DescribeSubnets()\n\tif err != nil {\n\t\treturn err\n\t}\n\tazToGatewaysInUse := map[string]int{}\n\tfor _, natGateway := range natGateways {\n\t\tif natGateway.SubnetId == nil {\n\t\t\tcontinue\n\t\t}\n\t\tfor _, subnet := range subnets {\n\t\t\tif subnet.SubnetId == nil || subnet.AvailabilityZone == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !availabilityZones.Has(*subnet.AvailabilityZone) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif *subnet.SubnetId == *natGateway.SubnetId {\n\t\t\t\tazToGatewaysInUse[*subnet.AvailabilityZone]++\n\t\t\t}\n\t\t}\n\t}\n\t// check NAT GW quota\n\tnumOfExhaustedNATGatewayAZs := 0\n\tazsWithQuotaDeficit := []string{}\n\tfor az, numActiveGatewaysOnAZ := range azToGatewaysInUse {\n\t\t// -1 comes from the NAT gateway we require per AZ\n\t\tazDeficit := natGatewayQuota - numActiveGatewaysOnAZ - 1\n\t\tif azDeficit < 0 {\n\t\t\tnumOfExhaustedNATGatewayAZs++\n\t\t\tazsWithQuotaDeficit = append(azsWithQuotaDeficit, az)\n\t\t}\n\t}\n\tif (highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs > 0) || (!highlyAvailableNATGateway && numOfExhaustedNATGatewayAZs == len(availabilityZones)) {\n\t\treturn ErrorNATGatewayLimitExceeded(natGatewayQuota, 1, azsWithQuotaDeficit, c.Region)\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) VerifyEIPQuota(eipQuota int, availabilityZones strset.Set, highlyAvailableNATGateway bool) error {\n\telasticIPsInUse, err := c.ListElasticIPs()\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar requiredElasticIPs int\n\tif highlyAvailableNATGateway {\n\t\trequiredElasticIPs = len(availabilityZones)\n\t} else {\n\t\trequiredElasticIPs = 1\n\t}\n\n\tadditionalQuotaRequired := len(elasticIPsInUse) + requiredElasticIPs - eipQuota\n\n\tif additionalQuotaRequired > 0 {\n\t\treturn ErrorEIPLimitExceeded(eipQuota, additionalQuotaRequired, c.Region)\n\t}\n\n\treturn nil\n}\n\nfunc (c *Client) VerifyVPCQuota(vpcQuota int, requiredVPCs int) error {\n\tvpcs, err := c.DescribeVpcs()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tadditionalQuotaRequired := len(vpcs) + requiredVPCs - vpcQuota\n\n\tif additionalQuotaRequired > 0 {\n\t\treturn ErrorVPCLimitExceeded(vpcQuota, additionalQuotaRequired, c.Region)\n\t}\n\treturn nil\n}\n\nfunc (c *Client) VerifySecurityGroupQuota(securifyGroupsQuota int, numNodeGroups int, clusterAlreadyExists bool) error {\n\trequiredSecurityGroups := requiredSecurityGroups(numNodeGroups, clusterAlreadyExists)\n\tsgs, err := c.DescribeSecurityGroups()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tadditionalQuotaRequired := len(sgs) + requiredSecurityGroups - securifyGroupsQuota\n\n\tif additionalQuotaRequired > 0 {\n\t\treturn ErrorSecurityGroupLimitExceeded(securifyGroupsQuota, additionalQuotaRequired, c.Region)\n\n\t}\n\treturn nil\n}\n\nfunc (c *Client) VerifySecurityGroupRulesQuota(\n\tsecurifyGroupRulesQuota int,\n\tavailabilityZones strset.Set,\n\tnumNodeGroups int,\n\tlongestCIDRWhiteList int) error {\n\n\t// check rules quota for nodegroup SGs\n\trequiredRulesForSG := requiredRulesForNodeGroupSecurityGroup(len(availabilityZones), longestCIDRWhiteList)\n\tif requiredRulesForSG > securifyGroupRulesQuota {\n\t\tadditionalQuotaRequired := requiredRulesForSG - securifyGroupRulesQuota\n\t\treturn ErrorSecurityGroupRulesExceeded(securifyGroupRulesQuota, additionalQuotaRequired, c.Region)\n\t}\n\n\t// check rules quota for control plane SG\n\trequiredRulesForCPSG := requiredRulesForControlPlaneSecurityGroup(numNodeGroups)\n\tif requiredRulesForCPSG > securifyGroupRulesQuota {\n\t\tadditionalQuotaRequired := requiredRulesForCPSG - securifyGroupRulesQuota\n\t\treturn ErrorSecurityGroupRulesExceeded(securifyGroupRulesQuota, additionalQuotaRequired, c.Region)\n\t}\n\treturn nil\n}\n\nfunc requiredRulesForNodeGroupSecurityGroup(numAZs, whitelistLength int) int {\n\twhitelistRuleCount := 0\n\tif whitelistLength == 1 {\n\t\twhitelistRuleCount = 1\n\t} else if whitelistLength > 1 {\n\t\twhitelistRuleCount = 1 + 5*(whitelistLength-1)\n\t}\n\n\treturn _baseInboundRulesForNodeGroup + numAZs*_inboundRulesPerAZ + whitelistRuleCount\n}\n\nfunc requiredRulesForControlPlaneSecurityGroup(numNodeGroups int) int {\n\t// +2 for the operator and prometheus node groups\n\t// this is the number of outbound rules (there are half as many inbound rules, so that is not the limiting factor)\n\treturn 2 * (numNodeGroups + 2)\n}\n\nfunc requiredSecurityGroups(numNodeGroups int, clusterAlreadyExists bool) int {\n\tif clusterAlreadyExists {\n\t\treturn numNodeGroups\n\t}\n\t// each node group requires a security group\n\treturn _baseNumberOfSecurityGroups + numNodeGroups\n}\n"
  },
  {
    "path": "pkg/lib/aws/sqs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nfunc (c *Client) GetAllQueueAttributes(queueURL string) (map[string]string, error) {\n\toutput, err := c.SQS().GetQueueAttributes(&sqs.GetQueueAttributesInput{\n\t\tQueueUrl:       aws.String(queueURL),\n\t\tAttributeNames: aws.StringSlice([]string{\"All\"}),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"unable to get queue attributes\", queueURL)\n\t}\n\n\treturn aws.StringValueMap(output.Attributes), nil\n}\n\nfunc (c *Client) ListQueuesByQueueNamePrefix(queueNamePrefix string) ([]string, error) {\n\tvar queueURLs []string\n\n\terr := c.SQS().ListQueuesPages(&sqs.ListQueuesInput{\n\t\tQueueNamePrefix: aws.String(queueNamePrefix),\n\t}, func(output *sqs.ListQueuesOutput, lastPage bool) bool {\n\t\tqueueURLs = append(queueURLs, aws.StringValueSlice(output.QueueUrls)...)\n\t\treturn true\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn queueURLs, nil\n}\n\nfunc (c *Client) DoesQueueExist(queueName string) (bool, error) {\n\t_, err := c.SQS().GetQueueUrl(&sqs.GetQueueUrlInput{\n\t\tQueueName: aws.String(queueName),\n\t})\n\tif err != nil {\n\t\tif IsErrCode(err, sqs.ErrCodeQueueDoesNotExist) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.Wrap(err, \"failed to check if queue exists\", queueName)\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) DeleteQueuesWithPrefix(queueNamePrefix string) (int, error) {\n\tvar numDeleted int\n\tvar deleteError error\n\n\terr := c.SQS().ListQueuesPages(&sqs.ListQueuesInput{\n\t\tQueueNamePrefix: aws.String(queueNamePrefix),\n\t}, func(output *sqs.ListQueuesOutput, lastPage bool) bool {\n\t\tfor _, queueURL := range output.QueueUrls {\n\t\t\t_, err := c.SQS().DeleteQueue(&sqs.DeleteQueueInput{ // best effort delete\n\t\t\t\tQueueUrl: queueURL,\n\t\t\t})\n\n\t\t\tnumDeleted++\n\n\t\t\tif deleteError != nil {\n\t\t\t\tdeleteError = err\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\n\tif err != nil {\n\t\treturn 0, errors.WithStack(err)\n\t}\n\n\tif deleteError != nil {\n\t\treturn 0, errors.WithStack(deleteError)\n\t}\n\n\treturn numDeleted, nil\n}\n"
  },
  {
    "path": "pkg/lib/aws/sts.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage aws\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/xml\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n\t\"github.com/aws/aws-sdk-go/aws/request\"\n\t\"github.com/aws/aws-sdk-go/private/protocol/query\"\n\t\"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil\"\n\t\"github.com/aws/aws-sdk-go/service/sts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/hash\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n)\n\n// Returns account ID, whether the credentials were valid, any other error that occurred\n// Ignores cache, so will re-run on every call to this method\nfunc (c *Client) CheckCredentials() (string, string, error) {\n\tresponse, err := c.STS().GetCallerIdentity(nil)\n\tif err != nil {\n\t\treturn \"\", \"\", ErrorInvalidAWSCredentials(err)\n\t}\n\n\tc.accountID = response.Account\n\tc.hashedAccountID = pointer.String(hash.String(*c.accountID))\n\n\treturn *c.accountID, *c.hashedAccountID, nil\n}\n\n// Only re-checks the credentials if they have never been checked (so will not catch e.g. credentials expiring or getting revoked)\nfunc (c *Client) GetCachedAccountID() (string, string, error) {\n\tif c.accountID == nil || c.hashedAccountID == nil {\n\t\tif _, _, err := c.CheckCredentials(); err != nil {\n\t\t\treturn \"\", \"\", err\n\t\t}\n\t}\n\treturn *c.accountID, *c.hashedAccountID, nil\n}\n\ntype awsRequest struct {\n\tHeader        http.Header\n\tURL           string\n\tMethod        string\n\tHost          string\n\tBody          string\n\tContentLength int64\n}\n\nfunc (c *Client) IdentityRequestAsHeader() (string, error) {\n\treq, _ := c.STS().GetCallerIdentityRequest(nil)\n\n\terr := req.Sign()\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(err)\n\t}\n\n\treqBody, err := ioutil.ReadAll(req.HTTPRequest.Body)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(err)\n\t}\n\n\tsignedRequestArtifacts := awsRequest{\n\t\tHeader:        req.HTTPRequest.Header,\n\t\tURL:           req.HTTPRequest.URL.String(),\n\t\tMethod:        req.HTTPRequest.Method,\n\t\tHost:          req.HTTPRequest.Host,\n\t\tBody:          string(reqBody),\n\t\tContentLength: req.HTTPRequest.ContentLength,\n\t}\n\tjsonSignedRequestArtifacts, err := libjson.Marshal(signedRequestArtifacts)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn base64.RawURLEncoding.EncodeToString(jsonSignedRequestArtifacts), nil\n}\n\n// ExecuteIdentityRequestFromHeader executes identity request marshalled from header and returns account id if successful\nfunc ExecuteIdentityRequestFromHeader(indentityRequestheader string) (string, error) {\n\tjsonObj, err := base64.RawURLEncoding.DecodeString(indentityRequestheader)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(err)\n\t}\n\n\tsignedRequestArtifacts := awsRequest{}\n\terr = libjson.Unmarshal(jsonObj, &signedRequestArtifacts)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\thttpClient := http.Client{}\n\n\turl, err := url.Parse(signedRequestArtifacts.URL)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(err)\n\t}\n\n\treq := http.Request{\n\t\tHeader:        signedRequestArtifacts.Header,\n\t\tMethod:        signedRequestArtifacts.Method,\n\t\tURL:           url,\n\t\tBody:          ioutil.NopCloser(strings.NewReader(signedRequestArtifacts.Body)),\n\t\tContentLength: signedRequestArtifacts.ContentLength,\n\t\tHost:          signedRequestArtifacts.Host,\n\t}\n\n\tresp, err := httpClient.Do(&req)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(err)\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode >= 400 {\n\t\tawsReq := request.Request{HTTPResponse: resp}\n\t\tquery.UnmarshalError(&awsReq)\n\t\treturn \"\", errors.WithStack(awsReq.Error)\n\t}\n\n\tdecoder := xml.NewDecoder(resp.Body)\n\n\tresult := sts.GetCallerIdentityOutput{}\n\terr = xmlutil.UnmarshalXML(&result, decoder, \"GetCallerIdentityResult\")\n\tif err != nil {\n\t\treturn \"\", awserr.NewRequestFailure(\n\t\t\tawserr.New(request.ErrCodeSerialization, \"failed decoding Query response\", err),\n\t\t\tresp.StatusCode,\n\t\t\tresp.Header.Get(\"X-Amzn-Requestid\"),\n\t\t)\n\t}\n\tif result.Account == nil {\n\t\treturn \"\", errors.ErrorUnexpected(\"GetCallerIdentityResult xml parsing failed\")\n\t}\n\n\treturn *result.Account, nil\n}\n"
  },
  {
    "path": "pkg/lib/cast/interface.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cast\n\nimport (\n\t\"encoding/json\"\n\t\"reflect\"\n)\n\nfunc InterfaceToInt8(in interface{}) (int8, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToInt(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn casted, true\n\tcase int16:\n\t\tif val := int8(casted); int16(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int32:\n\t\tif val := int8(casted); int32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int:\n\t\tif val := int8(casted); int(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int64:\n\t\tif val := int8(casted); int64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToInt8Downcast(in interface{}) (int8, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToIntOrFloat(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn casted, true\n\tcase int16:\n\t\tif val := int8(casted); int16(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int32:\n\t\tif val := int8(casted); int32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int:\n\t\tif val := int8(casted); int(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int64:\n\t\tif val := int8(casted); int64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float32:\n\t\tif val := int8(casted); float32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float64:\n\t\tif val := int8(casted); float64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToInt16(in interface{}) (int16, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToInt(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn int16(casted), true\n\tcase int16:\n\t\treturn casted, true\n\tcase int32:\n\t\tif val := int16(casted); int32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int:\n\t\tif val := int16(casted); int(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int64:\n\t\tif val := int16(casted); int64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToInt16Downcast(in interface{}) (int16, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToIntOrFloat(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn int16(casted), true\n\tcase int16:\n\t\treturn casted, true\n\tcase int32:\n\t\tif val := int16(casted); int32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int:\n\t\tif val := int16(casted); int(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int64:\n\t\tif val := int16(casted); int64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float32:\n\t\tif val := int16(casted); float32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float64:\n\t\tif val := int16(casted); float64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToInt32(in interface{}) (int32, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToInt(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn int32(casted), true\n\tcase int16:\n\t\treturn int32(casted), true\n\tcase int32:\n\t\treturn casted, true\n\tcase int:\n\t\tif val := int32(casted); int(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int64:\n\t\tif val := int32(casted); int64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToInt32Downcast(in interface{}) (int32, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToIntOrFloat(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn int32(casted), true\n\tcase int16:\n\t\treturn int32(casted), true\n\tcase int32:\n\t\treturn casted, true\n\tcase int:\n\t\tif val := int32(casted); int(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase int64:\n\t\tif val := int32(casted); int64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float32:\n\t\tif val := int32(casted); float32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float64:\n\t\tif val := int32(casted); float64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToInt(in interface{}) (int, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToInt(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn int(casted), true\n\tcase int16:\n\t\treturn int(casted), true\n\tcase int32:\n\t\treturn int(casted), true\n\tcase int:\n\t\treturn casted, true\n\tcase int64:\n\t\tif val := int(casted); int64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToIntDowncast(in interface{}) (int, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToIntOrFloat(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn int(casted), true\n\tcase int16:\n\t\treturn int(casted), true\n\tcase int32:\n\t\treturn int(casted), true\n\tcase int:\n\t\treturn casted, true\n\tcase int64:\n\t\tif val := int(casted); int64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float32:\n\t\tif val := int(casted); float32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float64:\n\t\tif val := int(casted); float64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToInt64(in interface{}) (int64, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToInt(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn int64(casted), true\n\tcase int16:\n\t\treturn int64(casted), true\n\tcase int32:\n\t\treturn int64(casted), true\n\tcase int:\n\t\treturn int64(casted), true\n\tcase int64:\n\t\treturn casted, true\n\t}\n\treturn 0, false\n}\n\nfunc InterfaceToInt64Downcast(in interface{}) (int64, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToIntOrFloat(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn int64(casted), true\n\tcase int16:\n\t\treturn int64(casted), true\n\tcase int32:\n\t\treturn int64(casted), true\n\tcase int:\n\t\treturn int64(casted), true\n\tcase int64:\n\t\treturn casted, true\n\tcase float32:\n\t\tif val := int64(casted); float32(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\tcase float64:\n\t\tif val := int64(casted); float64(val) == casted {\n\t\t\treturn val, true\n\t\t}\n\t}\n\treturn 0, false\n}\n\n// InterfaceToFloat32 will convert any int or float type\nfunc InterfaceToFloat32(in interface{}) (float32, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToIntOrFloat(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn float32(casted), true\n\tcase int16:\n\t\treturn float32(casted), true\n\tcase int32:\n\t\treturn float32(casted), true\n\tcase int:\n\t\treturn float32(casted), true\n\tcase int64:\n\t\treturn float32(casted), true\n\tcase float32:\n\t\treturn casted, true\n\tcase float64:\n\t\treturn float32(casted), true\n\t}\n\treturn 0, false\n}\n\n// InterfaceToFloat64 will convert any int or float type\nfunc InterfaceToFloat64(in interface{}) (float64, bool) {\n\tvar ok bool\n\tif in, ok = JSONNumberToIntOrFloat(in); !ok {\n\t\treturn 0, false\n\t}\n\n\tswitch casted := in.(type) {\n\tcase int8:\n\t\treturn float64(casted), true\n\tcase int16:\n\t\treturn float64(casted), true\n\tcase int32:\n\t\treturn float64(casted), true\n\tcase int:\n\t\treturn float64(casted), true\n\tcase int64:\n\t\treturn float64(casted), true\n\tcase float32:\n\t\treturn float64(casted), true\n\tcase float64:\n\t\treturn casted, true\n\t}\n\treturn 0, false\n}\n\nfunc JSONNumberToInt(in interface{}) (interface{}, bool) {\n\tnumber, ok := in.(json.Number)\n\tif !ok {\n\t\treturn in, true\n\t}\n\tinInt, err := number.Int64()\n\tif err == nil {\n\t\treturn inInt, true\n\t}\n\treturn nil, false\n}\n\nfunc JSONNumberToIntOrFloat(in interface{}) (interface{}, bool) {\n\tnumber, ok := in.(json.Number)\n\tif !ok {\n\t\treturn in, true\n\t}\n\tinInt, err := number.Int64()\n\tif err == nil {\n\t\treturn inInt, true\n\t}\n\tinFloat, err := number.Float64()\n\tif err == nil {\n\t\treturn inFloat, true\n\t}\n\treturn nil, false\n}\n\nfunc JSONNumber(in interface{}) interface{} {\n\tnumber, ok := in.(json.Number)\n\tif !ok {\n\t\treturn in\n\t}\n\tinInt, err := number.Int64()\n\tif err == nil {\n\t\treturn inInt\n\t}\n\tinFloat, err := number.Float64()\n\tif err == nil {\n\t\treturn inFloat\n\t}\n\treturn in // unexpected\n}\n\nfunc JSONNumbers(in []interface{}) []interface{} {\n\tcasted := make([]interface{}, len(in))\n\tfor i, element := range in {\n\t\tcasted[i] = JSONNumber(element)\n\t}\n\treturn casted\n}\n\nfunc InterfaceToInterfaceSlice(in interface{}) ([]interface{}, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif inSlice, ok := in.([]interface{}); ok {\n\t\treturn inSlice, true\n\t}\n\n\tif reflect.TypeOf(in).Kind() != reflect.Slice {\n\t\treturn nil, false\n\t}\n\n\tinVal := reflect.ValueOf(in)\n\tif inVal.IsNil() {\n\t\treturn nil, true\n\t}\n\n\tout := make([]interface{}, inVal.Len())\n\tfor i := 0; i < inVal.Len(); i++ {\n\t\tout[i] = inVal.Index(i).Interface()\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToIntSlice(in interface{}) ([]int, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif intSlice, ok := in.([]int); ok {\n\t\treturn intSlice, true\n\t}\n\n\tinSlice, ok := InterfaceToInterfaceSlice(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tout := make([]int, len(inSlice))\n\n\tfor i, elem := range inSlice {\n\t\tcasted, ok := InterfaceToInt(elem)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[i] = casted\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToInt32Slice(in interface{}) ([]int32, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif intSlice, ok := in.([]int32); ok {\n\t\treturn intSlice, true\n\t}\n\n\tinSlice, ok := InterfaceToInterfaceSlice(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tout := make([]int32, len(inSlice))\n\n\tfor i, elem := range inSlice {\n\t\tcasted, ok := InterfaceToInt32(elem)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[i] = casted\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToInt64Slice(in interface{}) ([]int64, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif intSlice, ok := in.([]int64); ok {\n\t\treturn intSlice, true\n\t}\n\n\tinSlice, ok := InterfaceToInterfaceSlice(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tout := make([]int64, len(inSlice))\n\n\tfor i, elem := range inSlice {\n\t\tcasted, ok := InterfaceToInt64(elem)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[i] = casted\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToFloat32Slice(in interface{}) ([]float32, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif floatSlice, ok := in.([]float32); ok {\n\t\treturn floatSlice, true\n\t}\n\n\tinSlice, ok := InterfaceToInterfaceSlice(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tout := make([]float32, len(inSlice))\n\n\tfor i, elem := range inSlice {\n\t\tcasted, ok := InterfaceToFloat32(elem)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[i] = casted\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToFloat64Slice(in interface{}) ([]float64, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif floatSlice, ok := in.([]float64); ok {\n\t\treturn floatSlice, true\n\t}\n\n\tinSlice, ok := InterfaceToInterfaceSlice(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tout := make([]float64, len(inSlice))\n\n\tfor i, elem := range inSlice {\n\t\tcasted, ok := InterfaceToFloat64(elem)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[i] = casted\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToStrSlice(in interface{}) ([]string, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif strSlice, ok := in.([]string); ok {\n\t\treturn strSlice, true\n\t}\n\n\tinSlice, ok := InterfaceToInterfaceSlice(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\n\tout := make([]string, len(inSlice))\n\n\tfor i, elem := range inSlice {\n\t\tcasted, ok := elem.(string)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[i] = casted\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToBoolSlice(in interface{}) ([]bool, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif boolSlice, ok := in.([]bool); ok {\n\t\treturn boolSlice, true\n\t}\n\n\tinSlice, ok := InterfaceToInterfaceSlice(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\n\tout := make([]bool, len(inSlice))\n\n\tfor i, elem := range inSlice {\n\t\tcasted, ok := elem.(bool)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[i] = casted\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToStrInterfaceMapSlice(in interface{}) ([]map[string]interface{}, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif strMapSlice, ok := in.([]map[string]interface{}); ok {\n\t\treturn strMapSlice, true\n\t}\n\n\tinSlice, ok := InterfaceToInterfaceSlice(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\n\tout := make([]map[string]interface{}, len(inSlice))\n\n\tfor i, elem := range inSlice {\n\t\tcasted, ok := InterfaceToStrInterfaceMap(elem)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[i] = casted\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToInterfaceInterfaceMap(in interface{}) (map[interface{}]interface{}, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif inMap, ok := in.(map[interface{}]interface{}); ok {\n\t\treturn inMap, true\n\t}\n\n\tif reflect.TypeOf(in).Kind() != reflect.Map {\n\t\treturn nil, false\n\t}\n\n\tinVal := reflect.ValueOf(in)\n\tif inVal.IsNil() {\n\t\treturn nil, true\n\t}\n\n\tout := make(map[interface{}]interface{}, inVal.Len())\n\tfor _, key := range inVal.MapKeys() {\n\t\tout[key.Interface()] = inVal.MapIndex(key).Interface()\n\t}\n\treturn out, true\n}\n\nfunc InterfaceToStrInterfaceMap(in interface{}) (map[string]interface{}, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif strMap, ok := in.(map[string]interface{}); ok {\n\t\treturn strMap, true\n\t}\n\n\tinMap, ok := InterfaceToInterfaceInterfaceMap(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\n\tout := map[string]interface{}{}\n\n\tfor key, value := range inMap {\n\t\tcasted, ok := key.(string)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[casted] = value\n\t}\n\treturn out, true\n}\n\n// Recursively casts interface->interface maps to string->interface maps\nfunc JSONMarshallable(in interface{}) (interface{}, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif inMap, ok := InterfaceToInterfaceInterfaceMap(in); ok {\n\t\tout := map[string]interface{}{}\n\t\tfor key, value := range inMap {\n\t\t\tcastedKey, ok := key.(string)\n\t\t\tif !ok {\n\t\t\t\treturn nil, false\n\t\t\t}\n\t\t\tcastedValue, ok := JSONMarshallable(value)\n\t\t\tif !ok {\n\t\t\t\treturn nil, false\n\t\t\t}\n\t\t\tout[castedKey] = castedValue\n\t\t}\n\t\treturn out, true\n\t} else if inSlice, ok := InterfaceToInterfaceSlice(in); ok {\n\t\tout := make([]interface{}, 0, len(inSlice))\n\t\tfor _, inValue := range inSlice {\n\t\t\tcastedInValue, ok := JSONMarshallable(inValue)\n\t\t\tif !ok {\n\t\t\t\treturn nil, false\n\t\t\t}\n\t\t\tout = append(out, castedInValue)\n\t\t}\n\t\treturn out, true\n\t}\n\treturn in, true\n}\n\nfunc InterfaceToStrStrMap(in interface{}) (map[string]string, bool) {\n\tif in == nil {\n\t\treturn nil, true\n\t}\n\n\tif strMap, ok := in.(map[string]string); ok {\n\t\treturn strMap, true\n\t}\n\n\tinMap, ok := InterfaceToInterfaceInterfaceMap(in)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\n\tout := map[string]string{}\n\n\tfor key, value := range inMap {\n\t\tcastedKey, ok := key.(string)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tcastedVal, ok := value.(string)\n\t\tif !ok {\n\t\t\treturn nil, false\n\t\t}\n\t\tout[castedKey] = castedVal\n\t}\n\treturn out, true\n}\n\nfunc StrMapToStrInterfaceMap(in map[string]string) map[string]interface{} {\n\tif in == nil {\n\t\treturn nil\n\t}\n\n\tout := map[string]interface{}{}\n\tfor k, v := range in {\n\t\tout[k] = v\n\t}\n\n\treturn out\n}\n\nfunc IsIntType(in interface{}) bool {\n\tswitch in := in.(type) {\n\tcase json.Number:\n\t\t_, err := in.Int64()\n\t\treturn err == nil\n\tcase int8:\n\t\treturn true\n\tcase int16:\n\t\treturn true\n\tcase int32:\n\t\treturn true\n\tcase int64:\n\t\treturn true\n\tcase int:\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc IsFloatType(in interface{}) bool {\n\tswitch in := in.(type) {\n\tcase json.Number:\n\t\t_, intErr := in.Int64()\n\t\t_, floatErr := in.Float64()\n\t\treturn floatErr == nil && intErr != nil\n\tcase float32:\n\t\treturn true\n\tcase float64:\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc IsNumericType(in interface{}) bool {\n\treturn IsIntType(in) || IsFloatType(in)\n}\n\nfunc IsScalarType(in interface{}) bool {\n\tif IsNumericType(in) {\n\t\treturn true\n\t}\n\n\tswitch in.(type) {\n\tcase string:\n\t\treturn true\n\tcase bool:\n\t\treturn true\n\t}\n\n\treturn false\n}\n\nfunc FlattenInterfaceSlices(in ...interface{}) []interface{} {\n\tvar result []interface{}\n\n\tfor _, item := range in {\n\t\tif item == nil {\n\t\t\tresult = append(result, nil)\n\t\t\tcontinue\n\t\t}\n\t\tif subItems, ok := InterfaceToInterfaceSlice(item); ok {\n\t\t\tif len(subItems) != 0 {\n\t\t\t\tresult = append(result, FlattenInterfaceSlices(subItems...)...)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tresult = append(result, item)\n\t}\n\n\treturn result\n}\n"
  },
  {
    "path": "pkg/lib/cast/interface_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cast\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestInterfaceToInterfaceSlice(t *testing.T) {\n\tslice1 := []string{\"test1\", \"test2\", \"test3\"}\n\tslice2 := []interface{}{\"test1\", \"test2\", \"test3\"}\n\tvar ok bool\n\n\t_, ok = InterfaceToInterfaceSlice(slice1)\n\trequire.True(t, ok)\n\n\t_, ok = InterfaceToInterfaceSlice(slice2)\n\trequire.True(t, ok)\n\n\t_, ok = InterfaceToStrSlice(slice1)\n\trequire.True(t, ok)\n\n\t_, ok = InterfaceToStrSlice(slice2)\n\trequire.True(t, ok)\n}\n\nfunc TestInterfaceToFloat64(t *testing.T) {\n\tvar out float64\n\tvar ok bool\n\n\tout, ok = InterfaceToFloat64(float64(1.1))\n\trequire.True(t, ok)\n\trequire.Equal(t, float64(1.1), out)\n\n\tout, ok = InterfaceToFloat64(float32(2.2))\n\trequire.True(t, ok)\n\trequire.Equal(t, float64(float32(2.2)), out)\n\n\tout, ok = InterfaceToFloat64(int(3))\n\trequire.True(t, ok)\n\trequire.Equal(t, float64(3), out)\n\n\t_, ok = InterfaceToFloat64(\"test\")\n\trequire.False(t, ok)\n}\n\nfunc TestInterfaceToIntDowncast(t *testing.T) {\n\tvar out int\n\tvar ok bool\n\n\tout, ok = InterfaceToIntDowncast(int(1))\n\trequire.True(t, ok)\n\trequire.Equal(t, int(1), out)\n\n\tout, ok = InterfaceToIntDowncast(float32(2))\n\trequire.True(t, ok)\n\trequire.Equal(t, int(2), out)\n\n\tout, ok = InterfaceToIntDowncast(float32(2.0))\n\trequire.True(t, ok)\n\trequire.Equal(t, int(2), out)\n\n\t_, ok = InterfaceToIntDowncast(float32(2.2))\n\trequire.False(t, ok)\n\n\tout, ok = InterfaceToIntDowncast(float64(3))\n\trequire.True(t, ok)\n\trequire.Equal(t, int(3), out)\n\n\tout, ok = InterfaceToIntDowncast(float64(3.0))\n\trequire.True(t, ok)\n\trequire.Equal(t, int(3), out)\n\n\t_, ok = InterfaceToIntDowncast(float64(3.3))\n\trequire.False(t, ok)\n\n\t_, ok = InterfaceToIntDowncast(\"test\")\n\trequire.False(t, ok)\n}\n\nfunc TestInterfaceToInt(t *testing.T) {\n\tvar out int\n\tvar ok bool\n\n\tout, ok = InterfaceToInt(int(1))\n\trequire.True(t, ok)\n\trequire.Equal(t, int(1), out)\n\n\t_, ok = InterfaceToInt(float32(2))\n\trequire.False(t, ok)\n\n\t_, ok = InterfaceToInt(\"test\")\n\trequire.False(t, ok)\n}\n\nfunc TestInterfaceToInt8Downcast(t *testing.T) {\n\tvar out int8\n\tvar ok bool\n\n\tout, ok = InterfaceToInt8Downcast(int(1))\n\trequire.True(t, ok)\n\trequire.Equal(t, int8(1), out)\n\n\tout, ok = InterfaceToInt8Downcast(float32(2))\n\trequire.True(t, ok)\n\trequire.Equal(t, int8(2), out)\n\n\tout, ok = InterfaceToInt8Downcast(float32(2.0))\n\trequire.True(t, ok)\n\trequire.Equal(t, int8(2), out)\n\n\t_, ok = InterfaceToInt8Downcast(float32(2.2))\n\trequire.False(t, ok)\n\n\tout, ok = InterfaceToInt8Downcast(float64(3))\n\trequire.True(t, ok)\n\trequire.Equal(t, int8(3), out)\n\n\tout, ok = InterfaceToInt8Downcast(float64(3.0))\n\trequire.True(t, ok)\n\trequire.Equal(t, int8(3), out)\n\n\t_, ok = InterfaceToInt8Downcast(float64(3.3))\n\trequire.False(t, ok)\n\n\t_, ok = InterfaceToInt8Downcast(\"test\")\n\trequire.False(t, ok)\n\n\t_, ok = InterfaceToInt8Downcast(int(999999))\n\trequire.False(t, ok)\n}\n\nfunc TestInterfaceToInt8(t *testing.T) {\n\tvar out int8\n\tvar ok bool\n\n\tout, ok = InterfaceToInt8(int(1))\n\trequire.True(t, ok)\n\trequire.Equal(t, int8(1), out)\n\n\t_, ok = InterfaceToInt8(float32(2))\n\trequire.False(t, ok)\n\n\t_, ok = InterfaceToInt8(\"test\")\n\trequire.False(t, ok)\n}\n\nfunc TestInterfaceToInterfaceInterfaceMap(t *testing.T) {\n\tvar ok bool\n\tvar in interface{}\n\tvar casted map[interface{}]interface{}\n\tvar expected map[interface{}]interface{}\n\n\tin = map[string]string{\"test\": \"str\"}\n\texpected = map[interface{}]interface{}{\"test\": \"str\"}\n\tcasted, ok = InterfaceToInterfaceInterfaceMap(in)\n\trequire.True(t, ok)\n\trequire.Equal(t, expected, casted)\n\n\tin = map[int]bool{2: true}\n\texpected = map[interface{}]interface{}{int(2): true}\n\tcasted, ok = InterfaceToInterfaceInterfaceMap(in)\n\trequire.True(t, ok)\n\trequire.Equal(t, expected, casted)\n\n\tin = map[interface{}]float32{\"test\": float32(2.2)}\n\texpected = map[interface{}]interface{}{\"test\": float32(2.2)}\n\tcasted, ok = InterfaceToInterfaceInterfaceMap(in)\n\trequire.True(t, ok)\n\trequire.Equal(t, expected, casted)\n}\n\nfunc TestJSONMarshallable(t *testing.T) {\n\tvar ok bool\n\tvar in interface{}\n\tvar casted interface{}\n\tvar expected interface{}\n\tvar err error\n\n\tin = map[string]interface{}{\"test\": map[interface{}]interface{}{\"testing\": []string{}}}\n\texpected = map[string]interface{}{\"test\": map[string]interface{}{\"testing\": []interface{}{}}}\n\tcasted, ok = JSONMarshallable(in)\n\trequire.True(t, ok)\n\trequire.Equal(t, expected, casted)\n\t_, err = json.Marshal(casted)\n\trequire.Equal(t, err, nil)\n\n\tin = map[string]interface{}{\"test\": map[interface{}]interface{}{1: []string{}}, \"slice\": []int{1}}\n\tcasted, ok = JSONMarshallable(in)\n\trequire.False(t, ok)\n\n\tin = map[string]interface{}{\"test\": map[interface{}]interface{}{\"1\": []string{}}, \"slice\": []int{1}}\n\texpected = map[string]interface{}{\"test\": map[string]interface{}{\"1\": []interface{}{}}, \"slice\": []interface{}{1}}\n\tcasted, ok = JSONMarshallable(in)\n\trequire.True(t, ok)\n\trequire.Equal(t, expected, casted)\n\t_, err = json.Marshal(casted)\n\trequire.Equal(t, err, nil)\n\n\tin = map[string]interface{}{\"test\": nil}\n\texpected = map[string]interface{}{\"test\": nil}\n\tcasted, ok = JSONMarshallable(in)\n\trequire.True(t, ok)\n\trequire.Equal(t, expected, casted)\n\t_, err = json.Marshal(casted)\n\trequire.Equal(t, err, nil)\n\n\tin = map[string]interface{}{\"slice\": []interface{}{1, \"1\", map[interface{}]interface{}{\"key\": false}}}\n\texpected = map[string]interface{}{\"slice\": []interface{}{1, \"1\", map[string]interface{}{\"key\": false}}}\n\tcasted, ok = JSONMarshallable(in)\n\trequire.True(t, ok)\n\trequire.Equal(t, expected, casted)\n\t_, err = json.Marshal(casted)\n\trequire.Equal(t, err, nil)\n\n\tin = map[string]interface{}{}\n\texpected = map[string]interface{}{}\n\tcasted, ok = JSONMarshallable(in)\n\trequire.True(t, ok)\n\trequire.Equal(t, expected, casted)\n\t_, err = json.Marshal(casted)\n\trequire.Equal(t, err, nil)\n}\n\nfunc TestFlattenInterfaceSlices(t *testing.T) {\n\texpected := []interface{}{\"a\", \"b\", \"c\"}\n\n\tin := []interface{}{\"a\", \"b\", \"c\"}\n\trequire.Equal(t, expected, FlattenInterfaceSlices(in))\n\n\tin2 := [][]interface{}{in}\n\trequire.Equal(t, expected, FlattenInterfaceSlices(in2))\n\n\tin3 := [][]interface{}{{\"a\"}, {\"b\", \"c\"}}\n\trequire.Equal(t, expected, FlattenInterfaceSlices(in3))\n\n\tin4 := [][]interface{}{{\"a\"}, {[]interface{}{\"b\"}, \"c\"}}\n\trequire.Equal(t, expected, FlattenInterfaceSlices(in4))\n}\n"
  },
  {
    "path": "pkg/lib/configreader/bool.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype BoolValidation struct {\n\tRequired              bool\n\tDefault               bool\n\tTreatNullAsFalse      bool // `<field>: ` and `<field>: null` will be read as `<field>: false`\n\tCantBeSpecifiedErrStr *string\n\tStrToBool             map[string]bool // lowercase\n}\n\nfunc Bool(inter interface{}, v *BoolValidation) (bool, error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsFalse {\n\t\t\treturn ValidateBoolProvided(false, v)\n\t\t}\n\t\treturn false, ErrorCannotBeNull(v.Required)\n\t}\n\tcasted, castOk := inter.(bool)\n\tif !castOk {\n\t\treturn false, ErrorInvalidPrimitiveType(inter, PrimTypeBool)\n\t}\n\treturn ValidateBoolProvided(casted, v)\n}\n\nfunc BoolFromInterfaceMap(key string, iMap map[string]interface{}, v *BoolValidation) (bool, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateBoolMissing(v)\n\t\tif err != nil {\n\t\t\treturn false, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Bool(inter, v)\n\tif err != nil {\n\t\treturn false, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc BoolFromStrMap(key string, sMap map[string]string, v *BoolValidation) (bool, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateBoolMissing(v)\n\t\tif err != nil {\n\t\t\treturn false, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := BoolFromStr(valStr, v)\n\tif err != nil {\n\t\treturn false, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc BoolFromStr(valStr string, v *BoolValidation) (bool, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateBoolMissing(v)\n\t}\n\tif len(v.StrToBool) > 0 {\n\t\tcasted, ok := v.StrToBool[strings.ToLower(valStr)]\n\n\t\tif !ok {\n\t\t\tkeys := make([]string, 0, len(v.StrToBool))\n\t\t\tfor key := range v.StrToBool {\n\t\t\t\tkeys = append(keys, key)\n\t\t\t}\n\n\t\t\treturn false, ErrorInvalidStr(valStr, keys[0], keys[1:]...)\n\t\t}\n\t\treturn ValidateBoolProvided(casted, v)\n\t}\n\n\tcasted, castOk := s.ParseBool(valStr)\n\tif !castOk {\n\t\treturn false, ErrorInvalidPrimitiveType(valStr, PrimTypeBool)\n\t}\n\treturn ValidateBoolProvided(casted, v)\n}\n\nfunc BoolFromEnv(envVarName string, v *BoolValidation) (bool, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateBoolMissing(v)\n\t\tif err != nil {\n\t\t\treturn false, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := BoolFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn false, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc BoolFromFile(filePath string, v *BoolValidation) (bool, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateBoolMissing(v)\n\t\tif err != nil {\n\t\t\treturn false, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateBoolMissing(v)\n\t\tif err != nil {\n\t\t\treturn false, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := BoolFromStr(valStr, v)\n\tif err != nil {\n\t\treturn false, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc BoolFromEnvOrFile(envVarName string, filePath string, v *BoolValidation) (bool, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn BoolFromEnv(envVarName, v)\n\t}\n\treturn BoolFromFile(filePath, v)\n}\n\nfunc BoolFromPrompt(promptOpts *prompt.Options, v *BoolValidation) (bool, error) {\n\tpromptOpts.DefaultStr = s.Bool(v.Default)\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateBoolMissing(v)\n\t}\n\treturn BoolFromStr(valStr, v)\n}\n\nfunc ValidateBoolMissing(v *BoolValidation) (bool, error) {\n\tif v.Required {\n\t\treturn false, ErrorMustBeDefined()\n\t}\n\treturn validateBool(v.Default, v)\n}\n\nfunc ValidateBoolProvided(val bool, v *BoolValidation) (bool, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn false, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\treturn validateBool(val, v)\n}\n\nfunc validateBool(val bool, v *BoolValidation) (bool, error) {\n\treturn val, nil\n}\n\n//\n// Musts\n//\n\nfunc MustBoolFromEnv(envVarName string, v *BoolValidation) bool {\n\tval, err := BoolFromEnv(envVarName, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustBoolFromFile(filePath string, v *BoolValidation) bool {\n\tval, err := BoolFromFile(filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustBoolFromEnvOrFile(envVarName string, filePath string, v *BoolValidation) bool {\n\tval, err := BoolFromEnvOrFile(envVarName, filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n"
  },
  {
    "path": "pkg/lib/configreader/bool_list.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype BoolListValidation struct {\n\tRequired              bool\n\tDefault               []bool\n\tAllowExplicitNull     bool\n\tAllowEmpty            bool\n\tCantBeSpecifiedErrStr *string\n\tCastSingleItem        bool\n\tMinLength             int\n\tMaxLength             int\n\tInvalidLengths        []int\n\tValidator             func([]bool) ([]bool, error)\n}\n\nfunc BoolList(inter interface{}, v *BoolListValidation) ([]bool, error) {\n\tcasted, castOk := cast.InterfaceToBoolSlice(inter)\n\tif !castOk {\n\t\tif v.CastSingleItem {\n\t\t\tcastedItem, castOk := inter.(bool)\n\t\t\tif !castOk {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeBool, PrimTypeBoolList)\n\t\t\t}\n\t\t\tcasted = []bool{castedItem}\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeBoolList)\n\t\t}\n\t}\n\treturn ValidateBoolListProvided(casted, v)\n}\n\nfunc BoolListFromInterfaceMap(key string, iMap map[string]interface{}, v *BoolListValidation) ([]bool, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateBoolListMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := BoolList(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateBoolListMissing(v *BoolListValidation) ([]bool, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateBoolList(v.Default, v)\n}\n\nfunc ValidateBoolListProvided(val []bool, v *BoolListValidation) ([]bool, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateBoolList(val, v)\n}\n\nfunc validateBoolList(val []bool, v *BoolListValidation) ([]bool, error) {\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(val) < v.MinLength {\n\t\t\treturn nil, ErrorTooFewElements(v.MinLength)\n\t\t}\n\t}\n\n\tif v.MaxLength != 0 {\n\t\tif len(val) > v.MaxLength {\n\t\t\treturn nil, ErrorTooManyElements(v.MaxLength)\n\t\t}\n\t}\n\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(val) == invalidLength {\n\t\t\treturn nil, ErrorWrongNumberOfElements(v.InvalidLengths)\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/bool_ptr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype BoolPtrValidation struct {\n\tRequired              bool\n\tDefault               *bool\n\tAllowExplicitNull     bool\n\tCantBeSpecifiedErrStr *string\n\tStrToBool             map[string]bool // lowercase\n}\n\nfunc BoolPtr(inter interface{}, v *BoolPtrValidation) (*bool, error) {\n\tif inter == nil {\n\t\treturn ValidateBoolPtrProvided(nil, v)\n\t}\n\tcasted, castOk := inter.(bool)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeBool)\n\t}\n\treturn ValidateBoolPtrProvided(&casted, v)\n}\n\nfunc BoolPtrFromInterfaceMap(key string, iMap map[string]interface{}, v *BoolPtrValidation) (*bool, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateBoolPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := BoolPtr(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc BoolPtrFromStrMap(key string, sMap map[string]string, v *BoolPtrValidation) (*bool, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateBoolPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := BoolPtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc BoolPtrFromStr(valStr string, v *BoolPtrValidation) (*bool, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateBoolPtrMissing(v)\n\t}\n\n\tif len(v.StrToBool) > 0 {\n\t\tcasted, ok := v.StrToBool[strings.ToLower(valStr)]\n\n\t\tif !ok {\n\t\t\tkeys := make([]string, 0, len(v.StrToBool))\n\t\t\tfor key := range v.StrToBool {\n\t\t\t\tkeys = append(keys, key)\n\t\t\t}\n\n\t\t\treturn nil, ErrorInvalidStr(valStr, keys[0], keys[1:]...)\n\t\t}\n\t\treturn ValidateBoolPtrProvided(&casted, v)\n\t}\n\n\tcasted, castOk := s.ParseBool(valStr)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(valStr, PrimTypeBool)\n\t}\n\treturn ValidateBoolPtrProvided(&casted, v)\n}\n\nfunc BoolPtrFromEnv(envVarName string, v *BoolPtrValidation) (*bool, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateBoolPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := BoolPtrFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc BoolPtrFromFile(filePath string, v *BoolPtrValidation) (*bool, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateBoolPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateBoolPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := BoolPtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc BoolPtrFromEnvOrFile(envVarName string, filePath string, v *BoolPtrValidation) (*bool, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn BoolPtrFromEnv(envVarName, v)\n\t}\n\treturn BoolPtrFromFile(filePath, v)\n}\n\nfunc BoolPtrFromPrompt(promptOpts *prompt.Options, v *BoolPtrValidation) (*bool, error) {\n\tif v.Default != nil && promptOpts.DefaultStr == \"\" {\n\t\tpromptOpts.DefaultStr = s.Bool(*v.Default)\n\t}\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateBoolPtrMissing(v)\n\t}\n\treturn BoolPtrFromStr(valStr, v)\n}\n\nfunc ValidateBoolPtrMissing(v *BoolPtrValidation) (*bool, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn v.Default, nil\n}\n\nfunc ValidateBoolPtrProvided(val *bool, v *BoolPtrValidation) (*bool, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrParseConfig                   = \"configreader.parse_config\"\n\tErrUnsupportedFieldValidation    = \"configreader.unsupported_field_validation\"\n\tErrUnsupportedKey                = \"configreader.unsupported_key\"\n\tErrInvalidYAML                   = \"configreader.invalid_yaml\"\n\tErrTooLong                       = \"configreader.too_long\"\n\tErrTooShort                      = \"configreader.too_short\"\n\tErrLeadingWhitespace             = \"configreader.leading_whitespace\"\n\tErrTrailingWhitespace            = \"configreader.trailing_whitespace\"\n\tErrAlphaNumericDashUnderscore    = \"configreader.alpha_numeric_dash_underscore\"\n\tErrAlphaNumericDash              = \"configreader.alpha_numeric_dash\"\n\tErrAlphaNumericDotUnderscore     = \"configreader.alpha_numeric_dot_underscore\"\n\tErrAlphaNumericDashDotUnderscore = \"configreader.alpha_numeric_dash_dot_underscore\"\n\tErrInvalidAWSTag                 = \"configreader.invalid_aws_tag\"\n\tErrInvalidDockerImage            = \"configreader.invalid_docker_image\"\n\tErrMustHavePrefix                = \"configreader.must_have_prefix\"\n\tErrMustHaveSuffix                = \"configreader.must_have_suffix\"\n\tErrCantHavePrefix                = \"configreader.cant_have_prefix\"\n\tErrCantHaveSuffix                = \"configreader.cant_have_suffix\"\n\tErrInvalidInterface              = \"configreader.invalid_interface\"\n\tErrInvalidFloat64                = \"configreader.invalid_float64\"\n\tErrInvalidFloat32                = \"configreader.invalid_float32\"\n\tErrInvalidInt64                  = \"configreader.invalid_int64\"\n\tErrInvalidInt32                  = \"configreader.invalid_int32\"\n\tErrInvalidInt                    = \"configreader.invalid_int\"\n\tErrInvalidStr                    = \"configreader.invalid_str\"\n\tErrDisallowedValue               = \"configreader.disallowed_value\"\n\tErrMustBeLessThanOrEqualTo       = \"configreader.must_be_less_than_or_equal_to\"\n\tErrMustBeLessThan                = \"configreader.must_be_less_than\"\n\tErrMustBeGreaterThanOrEqualTo    = \"configreader.must_be_greater_than_or_equal_to\"\n\tErrMustBeGreaterThan             = \"configreader.must_be_greater_than\"\n\tErrIsNotMultiple                 = \"configreader.is_not_multiple\"\n\tErrNonStringKeyFound             = \"configreader.non_string_key_found\"\n\tErrInvalidPrimitiveType          = \"configreader.invalid_primitive_type\"\n\tErrDuplicatedValue               = \"configreader.duplicated_value\"\n\tErrTooFewElements                = \"configreader.too_few_elements\"\n\tErrTooManyElements               = \"configreader.too_many_elements\"\n\tErrWrongNumberOfElements         = \"configreader.wrong_number_of_elements\"\n\tErrCannotSetStructField          = \"configreader.cannot_set_struct_field\"\n\tErrCannotBeNull                  = \"configreader.cannot_be_null\"\n\tErrCannotBeEmptyOrNull           = \"configreader.cannot_be_empty_or_null\"\n\tErrCannotBeEmpty                 = \"configreader.cannot_be_empty\"\n\tErrMustBeDefined                 = \"configreader.must_be_defined\"\n\tErrMapMustBeDefined              = \"configreader.map_must_be_defined\"\n\tErrMustBeEmpty                   = \"configreader.must_be_empty\"\n\tErrEmailTooLong                  = \"configreader.email_too_long\"\n\tErrEmailInvalid                  = \"configreader.email_invalid\"\n\tErrCortexResourceOnlyAllowed     = \"configreader.cortex_resource_only_allowed\"\n\tErrCortexResourceNotAllowed      = \"configreader.cortex_resource_not_allowed\"\n\tErrImageVersionMismatch          = \"configreader.image_version_mismatch\"\n\tErrFieldCantBeSpecified          = \"configreader.field_cant_be_specified\"\n)\n\nfunc ErrorParseConfig() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrParseConfig,\n\t\tMessage: \"failed to parse config file\",\n\t})\n}\n\nfunc ErrorUnsupportedFieldValidation() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUnsupportedFieldValidation,\n\t\tMessage: \"undefined or unsupported field validation\",\n\t})\n}\n\nfunc ErrorUnsupportedKey(key interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUnsupportedKey,\n\t\tMessage: fmt.Sprintf(\"key %s is not supported\", s.UserStr(key)),\n\t})\n}\n\nfunc ErrorInvalidYAML(err error) error {\n\tstr := strings.TrimPrefix(errors.Message(err), \"yaml: \")\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidYAML,\n\t\tMessage: fmt.Sprintf(\"invalid yaml: %s\", str),\n\t})\n}\n\nfunc ErrorTooLong(provided string, maxLen int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrTooLong,\n\t\tMessage: fmt.Sprintf(\"%s must be no more than %d characters\", s.UserStr(provided), maxLen),\n\t})\n}\n\nfunc ErrorTooShort(provided string, minLen int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrTooShort,\n\t\tMessage: fmt.Sprintf(\"%s must be at least %d characters\", s.UserStr(provided), minLen),\n\t})\n}\n\nfunc ErrorLeadingWhitespace(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrLeadingWhitespace,\n\t\tMessage: fmt.Sprintf(\"%s cannot start with whitespace\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorTrailingWhitespace(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrTrailingWhitespace,\n\t\tMessage: fmt.Sprintf(\"%s cannot end with whitespace\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorAlphaNumericDashUnderscore(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAlphaNumericDashUnderscore,\n\t\tMessage: fmt.Sprintf(\"%s must contain only letters, numbers, underscores, and dashes\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorAlphaNumericDash(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAlphaNumericDash,\n\t\tMessage: fmt.Sprintf(\"%s must contain only letters, numbers, and dashes\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorAlphaNumericDotUnderscore(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAlphaNumericDotUnderscore,\n\t\tMessage: fmt.Sprintf(\"%s must contain only letters, numbers, underscores and periods\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorAlphaNumericDashDotUnderscore(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAlphaNumericDashDotUnderscore,\n\t\tMessage: fmt.Sprintf(\"%s must contain only letters, numbers, underscores, dashes, and periods\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorInvalidAWSTag(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidAWSTag,\n\t\tMessage: fmt.Sprintf(\"%s must contain only letters, numbers, spaces, and the following characters: _ . : / + - @\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorInvalidDockerImage(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidDockerImage,\n\t\tMessage: fmt.Sprintf(\"%s is not a valid docker image path\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorMustHavePrefix(provided string, prefix string, prefixes ...string) error {\n\tallAllowedPrefixes := append([]string{prefix}, prefixes...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMustHavePrefix,\n\t\tMessage: fmt.Sprintf(\"%s must start with %s\", s.UserStr(provided), s.UserStrsOr(allAllowedPrefixes)),\n\t})\n}\n\nfunc ErrorMustHaveSuffix(provided string, suffix string, suffixes ...string) error {\n\tallAllowedSuffixes := append([]string{suffix}, suffixes...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMustHaveSuffix,\n\t\tMessage: fmt.Sprintf(\"%s must end with %s\", s.UserStr(provided), s.UserStrsOr(allAllowedSuffixes)),\n\t})\n}\n\nfunc ErrorCantHavePrefix(provided string, prefix string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCantHavePrefix,\n\t\tMessage: fmt.Sprintf(\"%s cannot start with %s\", s.UserStr(provided), s.UserStr(prefix)),\n\t})\n}\n\nfunc ErrorCantHaveSuffix(provided string, suffix string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCantHaveSuffix,\n\t\tMessage: fmt.Sprintf(\"%s cannot end with %s\", s.UserStr(provided), s.UserStr(suffix)),\n\t})\n}\n\nfunc ErrorInvalidInterface(provided interface{}, allowed interface{}, allowedVals ...interface{}) error {\n\tallAllowedVals := append([]interface{}{allowed}, allowedVals...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidInterface,\n\t\tMessage: fmt.Sprintf(\"invalid value (got %s, must be %s)\", s.UserStr(provided), s.UserStrsOr(allAllowedVals)),\n\t})\n}\n\nfunc ErrorInvalidFloat64(provided float64, allowed float64, allowedVals ...float64) error {\n\tallAllowedVals := append([]float64{allowed}, allowedVals...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidFloat64,\n\t\tMessage: fmt.Sprintf(\"invalid value (got %s, must be %s)\", s.UserStr(provided), s.UserStrsOr(allAllowedVals)),\n\t})\n}\n\nfunc ErrorInvalidFloat32(provided float32, allowed float32, allowedVals ...float32) error {\n\tallAllowedVals := append([]float32{allowed}, allowedVals...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidFloat32,\n\t\tMessage: fmt.Sprintf(\"invalid value (got %s, must be %s)\", s.UserStr(provided), s.UserStrsOr(allAllowedVals)),\n\t})\n}\n\nfunc ErrorInvalidInt64(provided int64, allowed int64, allowedVals ...int64) error {\n\tallAllowedVals := append([]int64{allowed}, allowedVals...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidInt64,\n\t\tMessage: fmt.Sprintf(\"invalid value (got %s, must be %s)\", s.UserStr(provided), s.UserStrsOr(allAllowedVals)),\n\t})\n}\n\nfunc ErrorInvalidInt32(provided int32, allowed int32, allowedVals ...int32) error {\n\tallAllowedVals := append([]int32{allowed}, allowedVals...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidInt32,\n\t\tMessage: fmt.Sprintf(\"invalid value (got %s, must be %s)\", s.UserStr(provided), s.UserStrsOr(allAllowedVals)),\n\t})\n}\n\nfunc ErrorInvalidInt(provided int, allowed int, allowedVals ...int) error {\n\tallAllowedVals := append([]int{allowed}, allowedVals...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidInt,\n\t\tMessage: fmt.Sprintf(\"invalid value (got %s, must be %s)\", s.UserStr(provided), s.UserStrsOr(allAllowedVals)),\n\t})\n}\n\nfunc ErrorInvalidStr(provided string, allowed string, allowedVals ...string) error {\n\tallAllowedVals := append([]string{allowed}, allowedVals...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidStr,\n\t\tMessage: fmt.Sprintf(\"invalid value (got %s, must be %s)\", s.UserStr(provided), s.UserStrsOr(allAllowedVals)),\n\t})\n}\n\nfunc ErrorDisallowedValue(provided interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDisallowedValue,\n\t\tMessage: fmt.Sprintf(\"%s is not allowed, please use a different value\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorMustBeLessThanOrEqualTo(provided interface{}, boundary interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMustBeLessThanOrEqualTo,\n\t\tMessage: fmt.Sprintf(\"must be less than or equal to %s (got %s)\", s.UserStr(boundary), s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorMustBeLessThan(provided interface{}, boundary interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMustBeLessThan,\n\t\tMessage: fmt.Sprintf(\"must be less than %s (got %s)\", s.UserStr(boundary), s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorMustBeGreaterThanOrEqualTo(provided interface{}, boundary interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMustBeGreaterThanOrEqualTo,\n\t\tMessage: fmt.Sprintf(\"must be greater than or equal to %s (got %s)\", s.UserStr(boundary), s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorMustBeGreaterThan(provided interface{}, boundary interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMustBeGreaterThan,\n\t\tMessage: fmt.Sprintf(\"must be greater than %s (got %s)\", s.UserStr(boundary), s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorIsNotMultiple(provided interface{}, multiple interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIsNotMultiple,\n\t\tMessage: fmt.Sprintf(\"%s is not a multiple of %s\", s.UserStr(provided), s.UserStr(multiple)),\n\t})\n}\n\nfunc ErrorNonStringKeyFound(key interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNonStringKeyFound,\n\t\tMessage: fmt.Sprintf(\"non string key found: %s\", s.ObjFlat(key)),\n\t})\n}\n\nfunc ErrorInvalidPrimitiveType(provided interface{}, allowedType PrimitiveType, allowedTypes ...PrimitiveType) error {\n\tallAllowedTypes := append([]PrimitiveType{allowedType}, allowedTypes...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidPrimitiveType,\n\t\tMessage: fmt.Sprintf(\"%s: invalid type (expected %s)\", s.UserStr(provided), s.StrsOr(PrimitiveTypes(allAllowedTypes).StringList())),\n\t})\n}\n\nfunc ErrorDuplicatedValue(val interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDuplicatedValue,\n\t\tMessage: fmt.Sprintf(\"%s is duplicated\", s.UserStr(val)),\n\t})\n}\n\nfunc ErrorTooFewElements(minLength int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrTooFewElements,\n\t\tMessage: fmt.Sprintf(\"must contain at least %d elements\", minLength),\n\t})\n}\n\nfunc ErrorTooManyElements(maxLength int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrTooManyElements,\n\t\tMessage: fmt.Sprintf(\"must contain at most %d elements\", maxLength),\n\t})\n}\n\nfunc ErrorWrongNumberOfElements(invalidLengths []int) error {\n\tinvalidElementsStr := \"elements\"\n\tif len(invalidLengths) == 1 && invalidLengths[0] == 1 {\n\t\tinvalidElementsStr = \"element\"\n\t}\n\n\tinvalidLengthStrs := make([]string, len(invalidLengths))\n\tfor i, length := range invalidLengths {\n\t\tinvalidLengthStrs[i] = s.Int(length)\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrWrongNumberOfElements,\n\t\tMessage: fmt.Sprintf(\"cannot contain %s %s\", s.StrsOr(invalidLengthStrs), invalidElementsStr),\n\t})\n}\n\nfunc ErrorCannotSetStructField() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCannotSetStructField,\n\t\tMessage: \"unable to set struct field\",\n\t})\n}\n\nfunc ErrorCannotBeNull(isRequired bool) error {\n\tmsg := \"cannot be null\"\n\tif !isRequired {\n\t\tmsg = \"cannot be null (specify a value, or remove the key to use the default value)\"\n\t}\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCannotBeNull,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorCannotBeEmptyOrNull(isRequired bool) error {\n\tmsg := \"cannot be empty or null\"\n\tif !isRequired {\n\t\tmsg = \"cannot be empty or null (specify a value, or remove the key to use the default value)\"\n\t}\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCannotBeEmptyOrNull,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorCannotBeEmpty() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCannotBeEmpty,\n\t\tMessage: \"cannot be empty\",\n\t})\n}\n\nfunc ErrorMustBeDefined(validValues ...interface{}) error {\n\tmsg := \"must be defined\"\n\tif len(validValues) > 0 && !reflect.ValueOf(validValues[0]).IsNil() { // reflect is necessary here\n\t\tmsg = fmt.Sprintf(\"must be defined, and set to %s\", s.UserStrsOr(validValues))\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMustBeDefined,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorMapMustBeDefined(keys ...string) error {\n\tmessage := \"must be defined\"\n\tif len(keys) > 0 {\n\t\tmessage = fmt.Sprintf(\"must be defined, and contain the following keys: %s\", s.UserStrsAnd(keys))\n\t}\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMapMustBeDefined,\n\t\tMessage: message,\n\t})\n}\n\nfunc ErrorMustBeEmpty() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMustBeEmpty,\n\t\tMessage: \"must be empty\",\n\t})\n}\n\nfunc ErrorEmailTooLong() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEmailTooLong,\n\t\tMessage: \"email address exceeds maximum length\",\n\t})\n}\n\nfunc ErrorEmailInvalid() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEmailInvalid,\n\t\tMessage: \"invalid email address\",\n\t})\n}\n\nfunc ErrorCortexResourceOnlyAllowed(invalidStr string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCortexResourceOnlyAllowed,\n\t\tMessage: fmt.Sprintf(\"%s: only cortex resource references (which start with @) are allowed in this context\", invalidStr),\n\t})\n}\n\nfunc ErrorCortexResourceNotAllowed(resourceName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCortexResourceNotAllowed,\n\t\tMessage: fmt.Sprintf(\"@%s: cortex resource references (which start with @) are not allowed in this context\", resourceName),\n\t})\n}\n\nfunc ErrorImageVersionMismatch(image, tag, cortexVersion string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrImageVersionMismatch,\n\t\tMessage: fmt.Sprintf(\"the specified image (%s) has a tag (%s) which does not match your Cortex version (%s); please update the image tag, remove the image registry path from your configuration file (to use the default value), or update your CLI (pip install cortex==%s)\", image, tag, cortexVersion, cortexVersion),\n\t})\n}\n\nfunc ErrorFieldCantBeSpecified(errMsg string) error {\n\tmessage := errMsg\n\tif message == \"\" {\n\t\tmessage = \"cannot be specified\"\n\t}\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFieldCantBeSpecified,\n\t\tMessage: message,\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/configreader/float32.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Float32Validation struct {\n\tRequired              bool\n\tDefault               float32\n\tTreatNullAsZero       bool // `<field>: ` and `<field>: null` will be read as `<field>: 0.0`\n\tAllowedValues         []float32\n\tHiddenAllowedValues   []float32 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []float32\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *float32\n\tGreaterThanOrEqualTo  *float32\n\tLessThan              *float32\n\tLessThanOrEqualTo     *float32\n\tValidator             func(float32) (float32, error)\n}\n\nfunc Float32(inter interface{}, v *Float32Validation) (float32, error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsZero {\n\t\t\treturn ValidateFloat32Provided(0, v)\n\t\t}\n\t\treturn 0, ErrorCannotBeNull(v.Required)\n\t}\n\tcasted, castOk := cast.InterfaceToFloat32(inter)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(inter, PrimTypeFloat)\n\t}\n\treturn ValidateFloat32Provided(casted, v)\n}\n\nfunc Float32FromInterfaceMap(key string, iMap map[string]interface{}, v *Float32Validation) (float32, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateFloat32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float32(inter, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Float32FromStrMap(key string, sMap map[string]string, v *Float32Validation) (float32, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateFloat32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float32FromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Float32FromStr(valStr string, v *Float32Validation) (float32, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateFloat32Missing(v)\n\t}\n\tcasted, castOk := s.ParseFloat32(valStr)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(valStr, PrimTypeFloat)\n\t}\n\treturn ValidateFloat32Provided(casted, v)\n}\n\nfunc Float32FromEnv(envVarName string, v *Float32Validation) (float32, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateFloat32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float32FromStr(*valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc Float32FromFile(filePath string, v *Float32Validation) (float32, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateFloat32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateFloat32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := Float32FromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc Float32FromEnvOrFile(envVarName string, filePath string, v *Float32Validation) (float32, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn Float32FromEnv(envVarName, v)\n\t}\n\treturn Float32FromFile(filePath, v)\n}\n\nfunc Float32FromPrompt(promptOpts *prompt.Options, v *Float32Validation) (float32, error) {\n\tpromptOpts.DefaultStr = s.Float32(v.Default)\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateFloat32Missing(v)\n\t}\n\treturn Float32FromStr(valStr, v)\n}\n\nfunc ValidateFloat32Missing(v *Float32Validation) (float32, error) {\n\tif v.Required {\n\t\treturn 0, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateFloat32(v.Default, v)\n}\n\nfunc ValidateFloat32Provided(val float32, v *Float32Validation) (float32, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\treturn validateFloat32(val, v)\n}\n\nfunc validateFloat32(val float32, v *Float32Validation) (float32, error) {\n\terr := ValidateFloat32Val(val, v)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateFloat32Val(val float32, v *Float32Validation) error {\n\tif v.GreaterThan != nil {\n\t\tif val <= *v.GreaterThan {\n\t\t\treturn ErrorMustBeGreaterThan(val, *v.GreaterThan)\n\t\t}\n\t}\n\tif v.GreaterThanOrEqualTo != nil {\n\t\tif val < *v.GreaterThanOrEqualTo {\n\t\t\treturn ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo)\n\t\t}\n\t}\n\tif v.LessThan != nil {\n\t\tif val >= *v.LessThan {\n\t\t\treturn ErrorMustBeLessThan(val, *v.LessThan)\n\t\t}\n\t}\n\tif v.LessThanOrEqualTo != nil {\n\t\tif val > *v.LessThanOrEqualTo {\n\t\t\treturn ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo)\n\t\t}\n\t}\n\n\tif len(v.AllowedValues) > 0 {\n\t\tif !slices.HasFloat32(append(v.AllowedValues, v.HiddenAllowedValues...), val) {\n\t\t\treturn ErrorInvalidFloat32(val, v.AllowedValues[0], v.AllowedValues[1:]...)\n\t\t}\n\t}\n\n\tif len(v.DisallowedValues) > 0 {\n\t\tif slices.HasFloat32(v.DisallowedValues, val) {\n\t\t\treturn ErrorDisallowedValue(val)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// Musts\n//\n\nfunc MustFloat32FromEnv(envVarName string, v *Float32Validation) float32 {\n\tval, err := Float32FromEnv(envVarName, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustFloat32FromFile(filePath string, v *Float32Validation) float32 {\n\tval, err := Float32FromFile(filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustFloat32FromEnvOrFile(envVarName string, filePath string, v *Float32Validation) float32 {\n\tval, err := Float32FromEnvOrFile(envVarName, filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n"
  },
  {
    "path": "pkg/lib/configreader/float32_list.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype Float32ListValidation struct {\n\tRequired              bool\n\tDefault               []float32\n\tAllowExplicitNull     bool\n\tAllowEmpty            bool\n\tCantBeSpecifiedErrStr *string\n\tCastSingleItem        bool\n\tMinLength             int\n\tMaxLength             int\n\tInvalidLengths        []int\n\tValidator             func([]float32) ([]float32, error)\n}\n\nfunc Float32List(inter interface{}, v *Float32ListValidation) ([]float32, error) {\n\tcasted, castOk := cast.InterfaceToFloat32Slice(inter)\n\tif !castOk {\n\t\tif v.CastSingleItem {\n\t\t\tcastedItem, castOk := cast.InterfaceToFloat32(inter)\n\t\t\tif !castOk {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloat, PrimTypeFloatList)\n\t\t\t}\n\t\t\tcasted = []float32{castedItem}\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloatList)\n\t\t}\n\t}\n\treturn ValidateFloat32ListProvided(casted, v)\n}\n\nfunc Float32ListFromInterfaceMap(key string, iMap map[string]interface{}, v *Float32ListValidation) ([]float32, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateFloat32ListMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float32List(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateFloat32ListMissing(v *Float32ListValidation) ([]float32, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateFloat32List(v.Default, v)\n}\n\nfunc ValidateFloat32ListProvided(val []float32, v *Float32ListValidation) ([]float32, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateFloat32List(val, v)\n}\n\nfunc validateFloat32List(val []float32, v *Float32ListValidation) ([]float32, error) {\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(val) < v.MinLength {\n\t\t\treturn nil, ErrorTooFewElements(v.MinLength)\n\t\t}\n\t}\n\n\tif v.MaxLength != 0 {\n\t\tif len(val) > v.MaxLength {\n\t\t\treturn nil, ErrorTooManyElements(v.MaxLength)\n\t\t}\n\t}\n\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(val) == invalidLength {\n\t\t\treturn nil, ErrorWrongNumberOfElements(v.InvalidLengths)\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/float32_ptr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Float32PtrValidation struct {\n\tRequired              bool\n\tDefault               *float32\n\tAllowExplicitNull     bool\n\tAllowedValues         []float32\n\tHiddenAllowedValues   []float32 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []float32\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *float32\n\tGreaterThanOrEqualTo  *float32\n\tLessThan              *float32\n\tLessThanOrEqualTo     *float32\n\tValidator             func(float32) (float32, error)\n}\n\nfunc makeFloat32ValValidation(v *Float32PtrValidation) *Float32Validation {\n\treturn &Float32Validation{\n\t\tAllowedValues:        v.AllowedValues,\n\t\tHiddenAllowedValues:  v.HiddenAllowedValues,\n\t\tDisallowedValues:     v.DisallowedValues,\n\t\tGreaterThan:          v.GreaterThan,\n\t\tGreaterThanOrEqualTo: v.GreaterThanOrEqualTo,\n\t\tLessThan:             v.LessThan,\n\t\tLessThanOrEqualTo:    v.LessThanOrEqualTo,\n\t}\n}\n\nfunc Float32Ptr(inter interface{}, v *Float32PtrValidation) (*float32, error) {\n\tif inter == nil {\n\t\treturn ValidateFloat32PtrProvided(nil, v)\n\t}\n\tcasted, castOk := cast.InterfaceToFloat32(inter)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloat)\n\t}\n\treturn ValidateFloat32PtrProvided(&casted, v)\n}\n\nfunc Float32PtrFromInterfaceMap(key string, iMap map[string]interface{}, v *Float32PtrValidation) (*float32, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateFloat32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float32Ptr(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Float32PtrFromStrMap(key string, sMap map[string]string, v *Float32PtrValidation) (*float32, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateFloat32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float32PtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Float32PtrFromStr(valStr string, v *Float32PtrValidation) (*float32, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateFloat32PtrMissing(v)\n\t}\n\tcasted, castOk := s.ParseFloat32(valStr)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(valStr, PrimTypeFloat)\n\t}\n\treturn ValidateFloat32PtrProvided(&casted, v)\n}\n\nfunc Float32PtrFromEnv(envVarName string, v *Float32PtrValidation) (*float32, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateFloat32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float32PtrFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc Float32PtrFromFile(filePath string, v *Float32PtrValidation) (*float32, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateFloat32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateFloat32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := Float32PtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc Float32PtrFromEnvOrFile(envVarName string, filePath string, v *Float32PtrValidation) (*float32, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn Float32PtrFromEnv(envVarName, v)\n\t}\n\treturn Float32PtrFromFile(filePath, v)\n}\n\nfunc Float32PtrFromPrompt(promptOpts *prompt.Options, v *Float32PtrValidation) (*float32, error) {\n\tif v.Default != nil && promptOpts.DefaultStr == \"\" {\n\t\tpromptOpts.DefaultStr = s.Float32(*v.Default)\n\t}\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateFloat32PtrMissing(v)\n\t}\n\treturn Float32PtrFromStr(valStr, v)\n}\n\nfunc ValidateFloat32PtrMissing(v *Float32PtrValidation) (*float32, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateFloat32Ptr(v.Default, v)\n}\n\nfunc ValidateFloat32PtrProvided(val *float32, v *Float32PtrValidation) (*float32, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateFloat32Ptr(val, v)\n}\n\nfunc validateFloat32Ptr(val *float32, v *Float32PtrValidation) (*float32, error) {\n\tif val != nil {\n\t\terr := ValidateFloat32Val(*val, makeFloat32ValValidation(v))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif val == nil {\n\t\treturn val, nil\n\t}\n\n\tif v.Validator != nil {\n\t\tvalidated, err := v.Validator(*val)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &validated, nil\n\t}\n\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/float64.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Float64Validation struct {\n\tRequired              bool\n\tDefault               float64\n\tTreatNullAsZero       bool // `<field>: ` and `<field>: null` will be read as `<field>: 0.0`\n\tAllowedValues         []float64\n\tHiddenAllowedValues   []float64 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []float64\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *float64\n\tGreaterThanOrEqualTo  *float64\n\tLessThan              *float64\n\tLessThanOrEqualTo     *float64\n\tValidator             func(float64) (float64, error)\n}\n\nfunc Float64(inter interface{}, v *Float64Validation) (float64, error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsZero {\n\t\t\treturn ValidateFloat64Provided(0, v)\n\t\t}\n\t\treturn 0, ErrorCannotBeNull(v.Required)\n\t}\n\tcasted, castOk := cast.InterfaceToFloat64(inter)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(inter, PrimTypeFloat)\n\t}\n\treturn ValidateFloat64Provided(casted, v)\n}\n\nfunc Float64FromInterfaceMap(key string, iMap map[string]interface{}, v *Float64Validation) (float64, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateFloat64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float64(inter, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Float64FromStrMap(key string, sMap map[string]string, v *Float64Validation) (float64, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateFloat64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float64FromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Float64FromStr(valStr string, v *Float64Validation) (float64, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateFloat64Missing(v)\n\t}\n\tcasted, castOk := s.ParseFloat64(valStr)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(valStr, PrimTypeFloat)\n\t}\n\treturn ValidateFloat64Provided(casted, v)\n}\n\nfunc Float64FromEnv(envVarName string, v *Float64Validation) (float64, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateFloat64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float64FromStr(*valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc Float64FromFile(filePath string, v *Float64Validation) (float64, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateFloat64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateFloat64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := Float64FromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc Float64FromEnvOrFile(envVarName string, filePath string, v *Float64Validation) (float64, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn Float64FromEnv(envVarName, v)\n\t}\n\treturn Float64FromFile(filePath, v)\n}\n\nfunc Float64FromPrompt(promptOpts *prompt.Options, v *Float64Validation) (float64, error) {\n\tpromptOpts.DefaultStr = s.Float64(v.Default)\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateFloat64Missing(v)\n\t}\n\treturn Float64FromStr(valStr, v)\n}\n\nfunc ValidateFloat64Missing(v *Float64Validation) (float64, error) {\n\tif v.Required {\n\t\treturn 0, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateFloat64(v.Default, v)\n}\n\nfunc ValidateFloat64Provided(val float64, v *Float64Validation) (float64, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\treturn validateFloat64(val, v)\n}\n\nfunc validateFloat64(val float64, v *Float64Validation) (float64, error) {\n\terr := ValidateFloat64Val(val, v)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateFloat64Val(val float64, v *Float64Validation) error {\n\tif v.GreaterThan != nil {\n\t\tif val <= *v.GreaterThan {\n\t\t\treturn ErrorMustBeGreaterThan(val, *v.GreaterThan)\n\t\t}\n\t}\n\tif v.GreaterThanOrEqualTo != nil {\n\t\tif val < *v.GreaterThanOrEqualTo {\n\t\t\treturn ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo)\n\t\t}\n\t}\n\tif v.LessThan != nil {\n\t\tif val >= *v.LessThan {\n\t\t\treturn ErrorMustBeLessThan(val, *v.LessThan)\n\t\t}\n\t}\n\tif v.LessThanOrEqualTo != nil {\n\t\tif val > *v.LessThanOrEqualTo {\n\t\t\treturn ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo)\n\t\t}\n\t}\n\n\tif len(v.AllowedValues) > 0 {\n\t\tif !slices.HasFloat64(append(v.AllowedValues, v.HiddenAllowedValues...), val) {\n\t\t\treturn ErrorInvalidFloat64(val, v.AllowedValues[0], v.AllowedValues[1:]...)\n\t\t}\n\t}\n\n\tif len(v.DisallowedValues) > 0 {\n\t\tif slices.HasFloat64(v.DisallowedValues, val) {\n\t\t\treturn ErrorDisallowedValue(val)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// Musts\n//\n\nfunc MustFloat64FromEnv(envVarName string, v *Float64Validation) float64 {\n\tval, err := Float64FromEnv(envVarName, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustFloat64FromFile(filePath string, v *Float64Validation) float64 {\n\tval, err := Float64FromFile(filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustFloat64FromEnvOrFile(envVarName string, filePath string, v *Float64Validation) float64 {\n\tval, err := Float64FromEnvOrFile(envVarName, filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n"
  },
  {
    "path": "pkg/lib/configreader/float64_list.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype Float64ListValidation struct {\n\tRequired              bool\n\tDefault               []float64\n\tAllowExplicitNull     bool\n\tAllowEmpty            bool\n\tCantBeSpecifiedErrStr *string\n\tCastSingleItem        bool\n\tMinLength             int\n\tMaxLength             int\n\tInvalidLengths        []int\n\tValidator             func([]float64) ([]float64, error)\n}\n\nfunc Float64List(inter interface{}, v *Float64ListValidation) ([]float64, error) {\n\tcasted, castOk := cast.InterfaceToFloat64Slice(inter)\n\tif !castOk {\n\t\tif v.CastSingleItem {\n\t\t\tcastedItem, castOk := cast.InterfaceToFloat64(inter)\n\t\t\tif !castOk {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloat, PrimTypeFloatList)\n\t\t\t}\n\t\t\tcasted = []float64{castedItem}\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloatList)\n\t\t}\n\t}\n\treturn ValidateFloat64ListProvided(casted, v)\n}\n\nfunc Float64ListFromInterfaceMap(key string, iMap map[string]interface{}, v *Float64ListValidation) ([]float64, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateFloat64ListMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float64List(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateFloat64ListMissing(v *Float64ListValidation) ([]float64, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateFloat64List(v.Default, v)\n}\n\nfunc ValidateFloat64ListProvided(val []float64, v *Float64ListValidation) ([]float64, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateFloat64List(val, v)\n}\n\nfunc validateFloat64List(val []float64, v *Float64ListValidation) ([]float64, error) {\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(val) < v.MinLength {\n\t\t\treturn nil, ErrorTooFewElements(v.MinLength)\n\t\t}\n\t}\n\n\tif v.MaxLength != 0 {\n\t\tif len(val) > v.MaxLength {\n\t\t\treturn nil, ErrorTooManyElements(v.MaxLength)\n\t\t}\n\t}\n\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(val) == invalidLength {\n\t\t\treturn nil, ErrorWrongNumberOfElements(v.InvalidLengths)\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/float64_ptr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Float64PtrValidation struct {\n\tRequired              bool\n\tDefault               *float64\n\tAllowExplicitNull     bool\n\tAllowedValues         []float64\n\tHiddenAllowedValues   []float64 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []float64\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *float64\n\tGreaterThanOrEqualTo  *float64\n\tLessThan              *float64\n\tLessThanOrEqualTo     *float64\n\tValidator             func(float64) (float64, error)\n}\n\nfunc makeFloat64ValValidation(v *Float64PtrValidation) *Float64Validation {\n\treturn &Float64Validation{\n\t\tAllowedValues:        v.AllowedValues,\n\t\tHiddenAllowedValues:  v.HiddenAllowedValues,\n\t\tDisallowedValues:     v.DisallowedValues,\n\t\tGreaterThan:          v.GreaterThan,\n\t\tGreaterThanOrEqualTo: v.GreaterThanOrEqualTo,\n\t\tLessThan:             v.LessThan,\n\t\tLessThanOrEqualTo:    v.LessThanOrEqualTo,\n\t}\n}\n\nfunc Float64Ptr(inter interface{}, v *Float64PtrValidation) (*float64, error) {\n\tif inter == nil {\n\t\treturn ValidateFloat64PtrProvided(nil, v)\n\t}\n\tcasted, castOk := cast.InterfaceToFloat64(inter)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeFloat)\n\t}\n\treturn ValidateFloat64PtrProvided(&casted, v)\n}\n\nfunc Float64PtrFromInterfaceMap(key string, iMap map[string]interface{}, v *Float64PtrValidation) (*float64, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateFloat64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float64Ptr(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Float64PtrFromStrMap(key string, sMap map[string]string, v *Float64PtrValidation) (*float64, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateFloat64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float64PtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Float64PtrFromStr(valStr string, v *Float64PtrValidation) (*float64, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateFloat64PtrMissing(v)\n\t}\n\tcasted, castOk := s.ParseFloat64(valStr)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(valStr, PrimTypeFloat)\n\t}\n\treturn ValidateFloat64PtrProvided(&casted, v)\n}\n\nfunc Float64PtrFromEnv(envVarName string, v *Float64PtrValidation) (*float64, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateFloat64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Float64PtrFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc Float64PtrFromFile(filePath string, v *Float64PtrValidation) (*float64, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateFloat64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateFloat64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := Float64PtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc Float64PtrFromEnvOrFile(envVarName string, filePath string, v *Float64PtrValidation) (*float64, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn Float64PtrFromEnv(envVarName, v)\n\t}\n\treturn Float64PtrFromFile(filePath, v)\n}\n\nfunc Float64PtrFromPrompt(promptOpts *prompt.Options, v *Float64PtrValidation) (*float64, error) {\n\tif v.Default != nil && promptOpts.DefaultStr == \"\" {\n\t\tpromptOpts.DefaultStr = s.Float64(*v.Default)\n\t}\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateFloat64PtrMissing(v)\n\t}\n\treturn Float64PtrFromStr(valStr, v)\n}\n\nfunc ValidateFloat64PtrMissing(v *Float64PtrValidation) (*float64, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateFloat64Ptr(v.Default, v)\n}\n\nfunc ValidateFloat64PtrProvided(val *float64, v *Float64PtrValidation) (*float64, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateFloat64Ptr(val, v)\n}\n\nfunc validateFloat64Ptr(val *float64, v *Float64PtrValidation) (*float64, error) {\n\tif val != nil {\n\t\terr := ValidateFloat64Val(*val, makeFloat64ValValidation(v))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif val == nil {\n\t\treturn val, nil\n\t}\n\n\tif v.Validator != nil {\n\t\tvalidated, err := v.Validator(*val)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &validated, nil\n\t}\n\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype IntValidation struct {\n\tRequired              bool\n\tDefault               int\n\tTreatNullAsZero       bool // `<field>: ` and `<field>: null` will be read as `<field>: 0`\n\tAllowedValues         []int\n\tHiddenAllowedValues   []int // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []int\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *int\n\tGreaterThanOrEqualTo  *int\n\tLessThan              *int\n\tLessThanOrEqualTo     *int\n\tValidator             func(int) (int, error)\n}\n\nfunc Int(inter interface{}, v *IntValidation) (int, error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsZero {\n\t\t\treturn ValidateIntProvided(0, v)\n\t\t}\n\t\treturn 0, ErrorCannotBeNull(v.Required)\n\t}\n\tcasted, castOk := cast.InterfaceToInt(inter)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(inter, PrimTypeInt)\n\t}\n\treturn ValidateIntProvided(casted, v)\n}\n\nfunc IntFromInterfaceMap(key string, iMap map[string]interface{}, v *IntValidation) (int, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateIntMissing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int(inter, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc IntFromStrMap(key string, sMap map[string]string, v *IntValidation) (int, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateIntMissing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := IntFromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc IntFromStr(valStr string, v *IntValidation) (int, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateIntMissing(v)\n\t}\n\tcasted, castOk := s.ParseInt(valStr)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(valStr, PrimTypeInt)\n\t}\n\treturn ValidateIntProvided(casted, v)\n}\n\nfunc IntFromEnv(envVarName string, v *IntValidation) (int, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateIntMissing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := IntFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc IntFromFile(filePath string, v *IntValidation) (int, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateIntMissing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateIntMissing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := IntFromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc IntFromEnvOrFile(envVarName string, filePath string, v *IntValidation) (int, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn IntFromEnv(envVarName, v)\n\t}\n\treturn IntFromFile(filePath, v)\n}\n\nfunc IntFromPrompt(promptOpts *prompt.Options, v *IntValidation) (int, error) {\n\tpromptOpts.DefaultStr = s.Int(v.Default)\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateIntMissing(v)\n\t}\n\treturn IntFromStr(valStr, v)\n}\n\nfunc ValidateIntMissing(v *IntValidation) (int, error) {\n\tif v.Required {\n\t\treturn 0, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateInt(v.Default, v)\n}\n\nfunc ValidateIntProvided(val int, v *IntValidation) (int, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\treturn validateInt(val, v)\n}\n\nfunc validateInt(val int, v *IntValidation) (int, error) {\n\terr := ValidateIntVal(val, v)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateIntVal(val int, v *IntValidation) error {\n\tif v.GreaterThan != nil {\n\t\tif val <= *v.GreaterThan {\n\t\t\treturn ErrorMustBeGreaterThan(val, *v.GreaterThan)\n\t\t}\n\t}\n\tif v.GreaterThanOrEqualTo != nil {\n\t\tif val < *v.GreaterThanOrEqualTo {\n\t\t\treturn ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo)\n\t\t}\n\t}\n\tif v.LessThan != nil {\n\t\tif val >= *v.LessThan {\n\t\t\treturn ErrorMustBeLessThan(val, *v.LessThan)\n\t\t}\n\t}\n\tif v.LessThanOrEqualTo != nil {\n\t\tif val > *v.LessThanOrEqualTo {\n\t\t\treturn ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo)\n\t\t}\n\t}\n\n\tif len(v.AllowedValues) > 0 {\n\t\tif !slices.HasInt(append(v.AllowedValues, v.HiddenAllowedValues...), val) {\n\t\t\treturn ErrorInvalidInt(val, v.AllowedValues[0], v.AllowedValues[1:]...)\n\t\t}\n\t}\n\n\tif len(v.DisallowedValues) > 0 {\n\t\tif slices.HasInt(v.DisallowedValues, val) {\n\t\t\treturn ErrorDisallowedValue(val)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// Musts\n//\n\nfunc MustIntFromEnv(envVarName string, v *IntValidation) int {\n\tval, err := IntFromEnv(envVarName, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustIntFromFile(filePath string, v *IntValidation) int {\n\tval, err := IntFromFile(filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustIntFromEnvOrFile(envVarName string, filePath string, v *IntValidation) int {\n\tval, err := IntFromEnvOrFile(envVarName, filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int32.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Int32Validation struct {\n\tRequired              bool\n\tDefault               int32\n\tTreatNullAsZero       bool // `<field>: ` and `<field>: null` will be read as `<field>: 0`\n\tAllowedValues         []int32\n\tHiddenAllowedValues   []int32 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []int32\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *int32\n\tGreaterThanOrEqualTo  *int32\n\tLessThan              *int32\n\tLessThanOrEqualTo     *int32\n\tValidator             func(int32) (int32, error)\n}\n\nfunc Int32(inter interface{}, v *Int32Validation) (int32, error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsZero {\n\t\t\treturn ValidateInt32Provided(0, v)\n\t\t}\n\t\treturn 0, ErrorCannotBeNull(v.Required)\n\t}\n\tcasted, castOk := cast.InterfaceToInt32(inter)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(inter, PrimTypeInt)\n\t}\n\treturn ValidateInt32Provided(casted, v)\n}\n\nfunc Int32FromInterfaceMap(key string, iMap map[string]interface{}, v *Int32Validation) (int32, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInt32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int32(inter, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Int32FromStrMap(key string, sMap map[string]string, v *Int32Validation) (int32, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateInt32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int32FromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Int32FromStr(valStr string, v *Int32Validation) (int32, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateInt32Missing(v)\n\t}\n\tcasted, castOk := s.ParseInt32(valStr)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(valStr, PrimTypeInt)\n\t}\n\treturn ValidateInt32Provided(casted, v)\n}\n\nfunc Int32FromEnv(envVarName string, v *Int32Validation) (int32, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateInt32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int32FromStr(*valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc Int32FromFile(filePath string, v *Int32Validation) (int32, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateInt32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateInt32Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := Int32FromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc Int32FromEnvOrFile(envVarName string, filePath string, v *Int32Validation) (int32, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn Int32FromEnv(envVarName, v)\n\t}\n\treturn Int32FromFile(filePath, v)\n}\n\nfunc Int32FromPrompt(promptOpts *prompt.Options, v *Int32Validation) (int32, error) {\n\tpromptOpts.DefaultStr = s.Int32(v.Default)\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateInt32Missing(v)\n\t}\n\treturn Int32FromStr(valStr, v)\n}\n\nfunc ValidateInt32Missing(v *Int32Validation) (int32, error) {\n\tif v.Required {\n\t\treturn 0, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateInt32(v.Default, v)\n}\n\nfunc ValidateInt32Provided(val int32, v *Int32Validation) (int32, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\treturn validateInt32(val, v)\n}\n\nfunc validateInt32(val int32, v *Int32Validation) (int32, error) {\n\terr := ValidateInt32Val(val, v)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateInt32Val(val int32, v *Int32Validation) error {\n\tif v.GreaterThan != nil {\n\t\tif val <= *v.GreaterThan {\n\t\t\treturn ErrorMustBeGreaterThan(val, *v.GreaterThan)\n\t\t}\n\t}\n\tif v.GreaterThanOrEqualTo != nil {\n\t\tif val < *v.GreaterThanOrEqualTo {\n\t\t\treturn ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo)\n\t\t}\n\t}\n\tif v.LessThan != nil {\n\t\tif val >= *v.LessThan {\n\t\t\treturn ErrorMustBeLessThan(val, *v.LessThan)\n\t\t}\n\t}\n\tif v.LessThanOrEqualTo != nil {\n\t\tif val > *v.LessThanOrEqualTo {\n\t\t\treturn ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo)\n\t\t}\n\t}\n\n\tif len(v.AllowedValues) > 0 {\n\t\tif !slices.HasInt32(append(v.AllowedValues, v.HiddenAllowedValues...), val) {\n\t\t\treturn ErrorInvalidInt32(val, v.AllowedValues[0], v.AllowedValues[1:]...)\n\t\t}\n\t}\n\n\tif len(v.DisallowedValues) > 0 {\n\t\tif slices.HasInt32(v.DisallowedValues, val) {\n\t\t\treturn ErrorDisallowedValue(val)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// Musts\n//\n\nfunc MustInt32FromEnv(envVarName string, v *Int32Validation) int32 {\n\tval, err := Int32FromEnv(envVarName, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustInt32FromFile(filePath string, v *Int32Validation) int32 {\n\tval, err := Int32FromFile(filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustInt32FromEnvOrFile(envVarName string, filePath string, v *Int32Validation) int32 {\n\tval, err := Int32FromEnvOrFile(envVarName, filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int32_list.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype Int32ListValidation struct {\n\tRequired              bool\n\tDefault               []int32\n\tAllowExplicitNull     bool\n\tAllowEmpty            bool\n\tCantBeSpecifiedErrStr *string\n\tCastSingleItem        bool\n\tMinLength             int\n\tMaxLength             int\n\tInvalidLengths        []int\n\tValidator             func([]int32) ([]int32, error)\n}\n\nfunc Int32List(inter interface{}, v *Int32ListValidation) ([]int32, error) {\n\tcasted, castOk := cast.InterfaceToInt32Slice(inter)\n\tif !castOk {\n\t\tif v.CastSingleItem {\n\t\t\tcastedItem, castOk := cast.InterfaceToInt32(inter)\n\t\t\tif !castOk {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt, PrimTypeIntList)\n\t\t\t}\n\t\t\tcasted = []int32{castedItem}\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeIntList)\n\t\t}\n\t}\n\treturn ValidateInt32ListProvided(casted, v)\n}\n\nfunc Int32ListFromInterfaceMap(key string, iMap map[string]interface{}, v *Int32ListValidation) ([]int32, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInt32ListMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int32List(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateInt32ListMissing(v *Int32ListValidation) ([]int32, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateInt32List(v.Default, v)\n}\n\nfunc ValidateInt32ListProvided(val []int32, v *Int32ListValidation) ([]int32, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateInt32List(val, v)\n}\n\nfunc validateInt32List(val []int32, v *Int32ListValidation) ([]int32, error) {\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(val) < v.MinLength {\n\t\t\treturn nil, ErrorTooFewElements(v.MinLength)\n\t\t}\n\t}\n\n\tif v.MaxLength != 0 {\n\t\tif len(val) > v.MaxLength {\n\t\t\treturn nil, ErrorTooManyElements(v.MaxLength)\n\t\t}\n\t}\n\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(val) == invalidLength {\n\t\t\treturn nil, ErrorWrongNumberOfElements(v.InvalidLengths)\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int32_ptr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Int32PtrValidation struct {\n\tRequired              bool\n\tDefault               *int32\n\tAllowExplicitNull     bool\n\tAllowedValues         []int32\n\tHiddenAllowedValues   []int32 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []int32\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *int32\n\tGreaterThanOrEqualTo  *int32\n\tLessThan              *int32\n\tLessThanOrEqualTo     *int32\n\tValidator             func(int32) (int32, error)\n}\n\nfunc makeInt32ValValidation(v *Int32PtrValidation) *Int32Validation {\n\treturn &Int32Validation{\n\t\tAllowedValues:        v.AllowedValues,\n\t\tHiddenAllowedValues:  v.HiddenAllowedValues,\n\t\tDisallowedValues:     v.DisallowedValues,\n\t\tGreaterThan:          v.GreaterThan,\n\t\tGreaterThanOrEqualTo: v.GreaterThanOrEqualTo,\n\t\tLessThan:             v.LessThan,\n\t\tLessThanOrEqualTo:    v.LessThanOrEqualTo,\n\t}\n}\n\nfunc Int32Ptr(inter interface{}, v *Int32PtrValidation) (*int32, error) {\n\tif inter == nil {\n\t\treturn ValidateInt32PtrProvided(nil, v)\n\t}\n\tcasted, castOk := cast.InterfaceToInt32(inter)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt)\n\t}\n\treturn ValidateInt32PtrProvided(&casted, v)\n}\n\nfunc Int32PtrFromInterfaceMap(key string, iMap map[string]interface{}, v *Int32PtrValidation) (*int32, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInt32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int32Ptr(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Int32PtrFromStrMap(key string, sMap map[string]string, v *Int32PtrValidation) (*int32, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateInt32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int32PtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Int32PtrFromStr(valStr string, v *Int32PtrValidation) (*int32, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateInt32PtrMissing(v)\n\t}\n\tcasted, castOk := s.ParseInt32(valStr)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(valStr, PrimTypeInt)\n\t}\n\treturn ValidateInt32PtrProvided(&casted, v)\n}\n\nfunc Int32PtrFromEnv(envVarName string, v *Int32PtrValidation) (*int32, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateInt32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int32PtrFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc Int32PtrFromFile(filePath string, v *Int32PtrValidation) (*int32, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateInt32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateInt32PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := Int32PtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc Int32PtrFromEnvOrFile(envVarName string, filePath string, v *Int32PtrValidation) (*int32, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn Int32PtrFromEnv(envVarName, v)\n\t}\n\treturn Int32PtrFromFile(filePath, v)\n}\n\nfunc Int32PtrFromPrompt(promptOpts *prompt.Options, v *Int32PtrValidation) (*int32, error) {\n\tif v.Default != nil && promptOpts.DefaultStr == \"\" {\n\t\tpromptOpts.DefaultStr = s.Int32(*v.Default)\n\t}\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateInt32PtrMissing(v)\n\t}\n\treturn Int32PtrFromStr(valStr, v)\n}\n\nfunc ValidateInt32PtrMissing(v *Int32PtrValidation) (*int32, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateInt32Ptr(v.Default, v)\n}\n\nfunc ValidateInt32PtrProvided(val *int32, v *Int32PtrValidation) (*int32, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateInt32Ptr(val, v)\n}\n\nfunc validateInt32Ptr(val *int32, v *Int32PtrValidation) (*int32, error) {\n\tif val != nil {\n\t\terr := ValidateInt32Val(*val, makeInt32ValValidation(v))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif val == nil {\n\t\treturn val, nil\n\t}\n\n\tif v.Validator != nil {\n\t\tvalidated, err := v.Validator(*val)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &validated, nil\n\t}\n\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int64.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Int64Validation struct {\n\tRequired              bool\n\tDefault               int64\n\tTreatNullAsZero       bool // `<field>: ` and `<field>: null` will be read as `<field>: 0`\n\tAllowedValues         []int64\n\tHiddenAllowedValues   []int64 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []int64\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *int64\n\tGreaterThanOrEqualTo  *int64\n\tLessThan              *int64\n\tLessThanOrEqualTo     *int64\n\tValidator             func(int64) (int64, error)\n}\n\nfunc Int64(inter interface{}, v *Int64Validation) (int64, error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsZero {\n\t\t\treturn ValidateInt64Provided(0, v)\n\t\t}\n\t\treturn 0, ErrorCannotBeNull(v.Required)\n\t}\n\tcasted, castOk := cast.InterfaceToInt64(inter)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(inter, PrimTypeInt)\n\t}\n\treturn ValidateInt64Provided(casted, v)\n}\n\nfunc Int64FromInterfaceMap(key string, iMap map[string]interface{}, v *Int64Validation) (int64, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInt64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int64(inter, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Int64FromStrMap(key string, sMap map[string]string, v *Int64Validation) (int64, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateInt64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int64FromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Int64FromStr(valStr string, v *Int64Validation) (int64, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateInt64Missing(v)\n\t}\n\tcasted, castOk := s.ParseInt64(valStr)\n\tif !castOk {\n\t\treturn 0, ErrorInvalidPrimitiveType(valStr, PrimTypeInt)\n\t}\n\treturn ValidateInt64Provided(casted, v)\n}\n\nfunc Int64FromEnv(envVarName string, v *Int64Validation) (int64, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateInt64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int64FromStr(*valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc Int64FromFile(filePath string, v *Int64Validation) (int64, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateInt64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateInt64Missing(v)\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := Int64FromStr(valStr, v)\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc Int64FromEnvOrFile(envVarName string, filePath string, v *Int64Validation) (int64, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn Int64FromEnv(envVarName, v)\n\t}\n\treturn Int64FromFile(filePath, v)\n}\n\nfunc Int64FromPrompt(promptOpts *prompt.Options, v *Int64Validation) (int64, error) {\n\tpromptOpts.DefaultStr = s.Int64(v.Default)\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateInt64Missing(v)\n\t}\n\treturn Int64FromStr(valStr, v)\n}\n\nfunc ValidateInt64Missing(v *Int64Validation) (int64, error) {\n\tif v.Required {\n\t\treturn 0, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateInt64(v.Default, v)\n}\n\nfunc ValidateInt64Provided(val int64, v *Int64Validation) (int64, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn 0, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\treturn validateInt64(val, v)\n}\n\nfunc validateInt64(val int64, v *Int64Validation) (int64, error) {\n\terr := ValidateInt64Val(val, v)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateInt64Val(val int64, v *Int64Validation) error {\n\tif v.GreaterThan != nil {\n\t\tif val <= *v.GreaterThan {\n\t\t\treturn ErrorMustBeGreaterThan(val, *v.GreaterThan)\n\t\t}\n\t}\n\tif v.GreaterThanOrEqualTo != nil {\n\t\tif val < *v.GreaterThanOrEqualTo {\n\t\t\treturn ErrorMustBeGreaterThanOrEqualTo(val, *v.GreaterThanOrEqualTo)\n\t\t}\n\t}\n\tif v.LessThan != nil {\n\t\tif val >= *v.LessThan {\n\t\t\treturn ErrorMustBeLessThan(val, *v.LessThan)\n\t\t}\n\t}\n\tif v.LessThanOrEqualTo != nil {\n\t\tif val > *v.LessThanOrEqualTo {\n\t\t\treturn ErrorMustBeLessThanOrEqualTo(val, *v.LessThanOrEqualTo)\n\t\t}\n\t}\n\n\tif len(v.AllowedValues) > 0 {\n\t\tif !slices.HasInt64(append(v.AllowedValues, v.HiddenAllowedValues...), val) {\n\t\t\treturn ErrorInvalidInt64(val, v.AllowedValues[0], v.AllowedValues[1:]...)\n\t\t}\n\t}\n\n\tif len(v.DisallowedValues) > 0 {\n\t\tif slices.HasInt64(v.DisallowedValues, val) {\n\t\t\treturn ErrorDisallowedValue(val)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// Musts\n//\n\nfunc MustInt64FromEnv(envVarName string, v *Int64Validation) int64 {\n\tval, err := Int64FromEnv(envVarName, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustInt64FromFile(filePath string, v *Int64Validation) int64 {\n\tval, err := Int64FromFile(filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustInt64FromEnvOrFile(envVarName string, filePath string, v *Int64Validation) int64 {\n\tval, err := Int64FromEnvOrFile(envVarName, filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int64_list.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype Int64ListValidation struct {\n\tRequired              bool\n\tDefault               []int64\n\tAllowExplicitNull     bool\n\tAllowEmpty            bool\n\tCantBeSpecifiedErrStr *string\n\tCastSingleItem        bool\n\tMinLength             int\n\tMaxLength             int\n\tInvalidLengths        []int\n\tValidator             func([]int64) ([]int64, error)\n}\n\nfunc Int64List(inter interface{}, v *Int64ListValidation) ([]int64, error) {\n\tcasted, castOk := cast.InterfaceToInt64Slice(inter)\n\tif !castOk {\n\t\tif v.CastSingleItem {\n\t\t\tcastedItem, castOk := cast.InterfaceToInt64(inter)\n\t\t\tif !castOk {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt, PrimTypeIntList)\n\t\t\t}\n\t\t\tcasted = []int64{castedItem}\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeIntList)\n\t\t}\n\t}\n\treturn ValidateInt64ListProvided(casted, v)\n}\n\nfunc Int64ListFromInterfaceMap(key string, iMap map[string]interface{}, v *Int64ListValidation) ([]int64, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInt64ListMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int64List(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateInt64ListMissing(v *Int64ListValidation) ([]int64, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateInt64List(v.Default, v)\n}\n\nfunc ValidateInt64ListProvided(val []int64, v *Int64ListValidation) ([]int64, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateInt64List(val, v)\n}\n\nfunc validateInt64List(val []int64, v *Int64ListValidation) ([]int64, error) {\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(val) < v.MinLength {\n\t\t\treturn nil, ErrorTooFewElements(v.MinLength)\n\t\t}\n\t}\n\n\tif v.MaxLength != 0 {\n\t\tif len(val) > v.MaxLength {\n\t\t\treturn nil, ErrorTooManyElements(v.MaxLength)\n\t\t}\n\t}\n\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(val) == invalidLength {\n\t\t\treturn nil, ErrorWrongNumberOfElements(v.InvalidLengths)\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int64_ptr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Int64PtrValidation struct {\n\tRequired              bool\n\tDefault               *int64\n\tAllowExplicitNull     bool\n\tAllowedValues         []int64\n\tHiddenAllowedValues   []int64 // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []int64\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *int64\n\tGreaterThanOrEqualTo  *int64\n\tLessThan              *int64\n\tLessThanOrEqualTo     *int64\n\tValidator             func(int64) (int64, error)\n}\n\nfunc makeInt64ValValidation(v *Int64PtrValidation) *Int64Validation {\n\treturn &Int64Validation{\n\t\tAllowedValues:        v.AllowedValues,\n\t\tHiddenAllowedValues:  v.HiddenAllowedValues,\n\t\tDisallowedValues:     v.DisallowedValues,\n\t\tGreaterThan:          v.GreaterThan,\n\t\tGreaterThanOrEqualTo: v.GreaterThanOrEqualTo,\n\t\tLessThan:             v.LessThan,\n\t\tLessThanOrEqualTo:    v.LessThanOrEqualTo,\n\t}\n}\n\nfunc Int64Ptr(inter interface{}, v *Int64PtrValidation) (*int64, error) {\n\tif inter == nil {\n\t\treturn ValidateInt64PtrProvided(nil, v)\n\t}\n\tcasted, castOk := cast.InterfaceToInt64(inter)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt)\n\t}\n\treturn ValidateInt64PtrProvided(&casted, v)\n}\n\nfunc Int64PtrFromInterfaceMap(key string, iMap map[string]interface{}, v *Int64PtrValidation) (*int64, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInt64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int64Ptr(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Int64PtrFromStrMap(key string, sMap map[string]string, v *Int64PtrValidation) (*int64, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateInt64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int64PtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc Int64PtrFromStr(valStr string, v *Int64PtrValidation) (*int64, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateInt64PtrMissing(v)\n\t}\n\tcasted, castOk := s.ParseInt64(valStr)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(valStr, PrimTypeInt)\n\t}\n\treturn ValidateInt64PtrProvided(&casted, v)\n}\n\nfunc Int64PtrFromEnv(envVarName string, v *Int64PtrValidation) (*int64, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateInt64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := Int64PtrFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc Int64PtrFromFile(filePath string, v *Int64PtrValidation) (*int64, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateInt64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateInt64PtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := Int64PtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc Int64PtrFromEnvOrFile(envVarName string, filePath string, v *Int64PtrValidation) (*int64, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn Int64PtrFromEnv(envVarName, v)\n\t}\n\treturn Int64PtrFromFile(filePath, v)\n}\n\nfunc Int64PtrFromPrompt(promptOpts *prompt.Options, v *Int64PtrValidation) (*int64, error) {\n\tif v.Default != nil && promptOpts.DefaultStr == \"\" {\n\t\tpromptOpts.DefaultStr = s.Int64(*v.Default)\n\t}\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateInt64PtrMissing(v)\n\t}\n\treturn Int64PtrFromStr(valStr, v)\n}\n\nfunc ValidateInt64PtrMissing(v *Int64PtrValidation) (*int64, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateInt64Ptr(v.Default, v)\n}\n\nfunc ValidateInt64PtrProvided(val *int64, v *Int64PtrValidation) (*int64, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateInt64Ptr(val, v)\n}\n\nfunc validateInt64Ptr(val *int64, v *Int64PtrValidation) (*int64, error) {\n\tif val != nil {\n\t\terr := ValidateInt64Val(*val, makeInt64ValValidation(v))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif val == nil {\n\t\treturn val, nil\n\t}\n\n\tif v.Validator != nil {\n\t\tvalidated, err := v.Validator(*val)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &validated, nil\n\t}\n\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int_list.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype IntListValidation struct {\n\tRequired              bool\n\tDefault               []int\n\tAllowExplicitNull     bool\n\tAllowEmpty            bool\n\tCantBeSpecifiedErrStr *string\n\tCastSingleItem        bool\n\tMinLength             int\n\tMaxLength             int\n\tInvalidLengths        []int\n\tValidator             func([]int) ([]int, error)\n}\n\nfunc IntList(inter interface{}, v *IntListValidation) ([]int, error) {\n\tcasted, castOk := cast.InterfaceToIntSlice(inter)\n\tif !castOk {\n\t\tif v.CastSingleItem {\n\t\t\tcastedItem, castOk := cast.InterfaceToInt(inter)\n\t\t\tif !castOk {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt, PrimTypeIntList)\n\t\t\t}\n\t\t\tcasted = []int{castedItem}\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeIntList)\n\t\t}\n\t}\n\treturn ValidateIntListProvided(casted, v)\n}\n\nfunc IntListFromInterfaceMap(key string, iMap map[string]interface{}, v *IntListValidation) ([]int, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateIntListMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := IntList(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateIntListMissing(v *IntListValidation) ([]int, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateIntList(v.Default, v)\n}\n\nfunc ValidateIntListProvided(val []int, v *IntListValidation) ([]int, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateIntList(val, v)\n}\n\nfunc validateIntList(val []int, v *IntListValidation) ([]int, error) {\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(val) < v.MinLength {\n\t\t\treturn nil, ErrorTooFewElements(v.MinLength)\n\t\t}\n\t}\n\n\tif v.MaxLength != 0 {\n\t\tif len(val) > v.MaxLength {\n\t\t\treturn nil, ErrorTooManyElements(v.MaxLength)\n\t\t}\n\t}\n\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(val) == invalidLength {\n\t\t\treturn nil, ErrorWrongNumberOfElements(v.InvalidLengths)\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/int_ptr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype IntPtrValidation struct {\n\tRequired              bool\n\tDefault               *int\n\tAllowExplicitNull     bool\n\tAllowedValues         []int\n\tHiddenAllowedValues   []int // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues      []int\n\tCantBeSpecifiedErrStr *string\n\tGreaterThan           *int\n\tGreaterThanOrEqualTo  *int\n\tLessThan              *int\n\tLessThanOrEqualTo     *int\n\tValidator             func(int) (int, error)\n}\n\nfunc makeIntValValidation(v *IntPtrValidation) *IntValidation {\n\treturn &IntValidation{\n\t\tAllowedValues:        v.AllowedValues,\n\t\tHiddenAllowedValues:  v.HiddenAllowedValues,\n\t\tDisallowedValues:     v.DisallowedValues,\n\t\tGreaterThan:          v.GreaterThan,\n\t\tGreaterThanOrEqualTo: v.GreaterThanOrEqualTo,\n\t\tLessThan:             v.LessThan,\n\t\tLessThanOrEqualTo:    v.LessThanOrEqualTo,\n\t}\n}\n\nfunc IntPtr(inter interface{}, v *IntPtrValidation) (*int, error) {\n\tif inter == nil {\n\t\treturn ValidateIntPtrProvided(nil, v)\n\t}\n\tcasted, castOk := cast.InterfaceToInt(inter)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeInt)\n\t}\n\treturn ValidateIntPtrProvided(&casted, v)\n}\n\nfunc IntPtrFromInterfaceMap(key string, iMap map[string]interface{}, v *IntPtrValidation) (*int, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateIntPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := IntPtr(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc IntPtrFromStrMap(key string, sMap map[string]string, v *IntPtrValidation) (*int, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok || valStr == \"\" {\n\t\tval, err := ValidateIntPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := IntPtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc IntPtrFromStr(valStr string, v *IntPtrValidation) (*int, error) {\n\tif valStr == \"\" {\n\t\treturn ValidateIntPtrMissing(v)\n\t}\n\tcasted, castOk := s.ParseInt(valStr)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(valStr, PrimTypeInt)\n\t}\n\treturn ValidateIntPtrProvided(&casted, v)\n}\n\nfunc IntPtrFromEnv(envVarName string, v *IntPtrValidation) (*int, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil || *valStr == \"\" {\n\t\tval, err := ValidateIntPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := IntPtrFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc IntPtrFromFile(filePath string, v *IntPtrValidation) (*int, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateIntPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateIntPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := IntPtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc IntPtrFromEnvOrFile(envVarName string, filePath string, v *IntPtrValidation) (*int, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil && *valStr != \"\" {\n\t\treturn IntPtrFromEnv(envVarName, v)\n\t}\n\treturn IntPtrFromFile(filePath, v)\n}\n\nfunc IntPtrFromPrompt(promptOpts *prompt.Options, v *IntPtrValidation) (*int, error) {\n\tif v.Default != nil && promptOpts.DefaultStr == \"\" {\n\t\tpromptOpts.DefaultStr = s.Int(*v.Default)\n\t}\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" {\n\t\treturn ValidateIntPtrMissing(v)\n\t}\n\treturn IntPtrFromStr(valStr, v)\n}\n\nfunc ValidateIntPtrMissing(v *IntPtrValidation) (*int, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateIntPtr(v.Default, v)\n}\n\nfunc ValidateIntPtrProvided(val *int, v *IntPtrValidation) (*int, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateIntPtr(val, v)\n}\n\nfunc validateIntPtr(val *int, v *IntPtrValidation) (*int, error) {\n\tif val != nil {\n\t\terr := ValidateIntVal(*val, makeIntValValidation(v))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif val == nil {\n\t\treturn val, nil\n\t}\n\n\tif v.Validator != nil {\n\t\tvalidated, err := v.Validator(*val)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &validated, nil\n\t}\n\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/interface.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/maps\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/yaml\"\n)\n\ntype InterfaceValidation struct {\n\tRequired               bool\n\tDefault                interface{}\n\tAllowExplicitNull      bool\n\tCantBeSpecifiedErrStr  *string\n\tAllowCortexResources   bool\n\tRequireCortexResources bool\n\tValidator              func(interface{}) (interface{}, error)\n}\n\nfunc Interface(inter interface{}, v *InterfaceValidation) (interface{}, error) {\n\treturn ValidateInterfaceProvided(inter, v)\n}\n\nfunc InterfaceFromInterfaceMap(key string, iMap map[string]interface{}, v *InterfaceValidation) (interface{}, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInterfaceMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := ValidateInterfaceProvided(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateInterfaceMissing(v *InterfaceValidation) (interface{}, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateInterface(v.Default, v)\n}\n\nfunc ValidateInterfaceProvided(val interface{}, v *InterfaceValidation) (interface{}, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateInterface(val, v)\n}\n\nfunc validateInterface(val interface{}, v *InterfaceValidation) (interface{}, error) {\n\tif v.RequireCortexResources {\n\t\tif err := checkOnlyCortexResources(val); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if !v.AllowCortexResources {\n\t\tif err := checkNoCortexResources(val); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n\nfunc checkNoCortexResources(obj interface{}) error {\n\tif objStr, ok := obj.(string); ok {\n\t\tif resourceName, ok := yaml.ExtractAtSymbolText(objStr); ok {\n\t\t\treturn ErrorCortexResourceNotAllowed(resourceName)\n\t\t}\n\t}\n\n\tif objSlice, ok := cast.InterfaceToInterfaceSlice(obj); ok {\n\t\tfor i, objItem := range objSlice {\n\t\t\tif err := checkNoCortexResources(objItem); err != nil {\n\t\t\t\treturn errors.Wrap(err, s.Index(i))\n\t\t\t}\n\t\t}\n\t}\n\n\tif objMap, ok := cast.InterfaceToInterfaceInterfaceMap(obj); ok {\n\t\tfor k, v := range objMap {\n\t\t\tif err := checkNoCortexResources(k); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err := checkNoCortexResources(v); err != nil {\n\t\t\t\treturn errors.Wrap(err, s.UserStrStripped(k))\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc checkOnlyCortexResources(obj interface{}) error {\n\tif objStr, ok := obj.(string); ok {\n\t\tif _, ok := yaml.ExtractAtSymbolText(objStr); !ok {\n\t\t\treturn ErrorCortexResourceOnlyAllowed(objStr)\n\t\t}\n\t}\n\n\tif objSlice, ok := cast.InterfaceToInterfaceSlice(obj); ok {\n\t\tfor i, objItem := range objSlice {\n\t\t\tif err := checkOnlyCortexResources(objItem); err != nil {\n\t\t\t\treturn errors.Wrap(err, s.Index(i))\n\t\t\t}\n\t\t}\n\t}\n\n\tif objMap, ok := cast.InterfaceToInterfaceInterfaceMap(obj); ok {\n\t\tfor k, v := range objMap {\n\t\t\tif err := checkOnlyCortexResources(k); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err := checkOnlyCortexResources(v); err != nil {\n\t\t\t\treturn errors.Wrap(err, s.UserStrStripped(k))\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// FlattenAllStrValues assumes that the order for maps is deterministic\nfunc FlattenAllStrValues(obj interface{}) ([]string, error) {\n\tobj = pointer.IndirectSafe(obj)\n\tflattened := []string{}\n\n\tif objStr, ok := obj.(string); ok {\n\t\treturn append(flattened, objStr), nil\n\t}\n\n\tif objSlice, ok := cast.InterfaceToInterfaceSlice(obj); ok {\n\t\tfor i, elem := range objSlice {\n\t\t\tsubFlattened, err := FlattenAllStrValues(elem)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrap(err, s.Index(i))\n\t\t\t}\n\t\t\tflattened = append(flattened, subFlattened...)\n\t\t}\n\t\treturn flattened, nil\n\t}\n\n\tif objMap, ok := cast.InterfaceToStrInterfaceMap(obj); ok {\n\t\tfor _, key := range maps.InterfaceMapSortedKeys(objMap) {\n\t\t\tsubFlattened, err := FlattenAllStrValues(objMap[key])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrap(err, s.UserStrStripped(key))\n\t\t\t}\n\t\t\tflattened = append(flattened, subFlattened...)\n\t\t}\n\t\treturn flattened, nil\n\t}\n\n\treturn nil, ErrorInvalidPrimitiveType(obj, PrimTypeString, PrimTypeList, PrimTypeMap)\n}\n\nfunc FlattenAllStrValuesAsSet(obj interface{}) (strset.Set, error) {\n\tstrs, err := FlattenAllStrValues(obj)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tset := strset.New()\n\tfor _, str := range strs {\n\t\tset.Add(str)\n\t}\n\treturn set, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/interface_map.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n)\n\ntype InterfaceMapValidation struct {\n\tRequired               bool\n\tDefault                map[string]interface{}\n\tAllowExplicitNull      bool\n\tAllowEmpty             bool\n\tCantBeSpecifiedErrStr  *string\n\tConvertNullToEmpty     bool\n\tScalarsOnly            bool\n\tStringLeavesOnly       bool\n\tStringKeysOnly         bool // Useful for ensuring this field is JSON parsable; validates that all maps and nested maps only use string keys\n\tAllowedLeafValues      []string\n\tAllowCortexResources   bool\n\tRequireCortexResources bool\n\tValidator              func(map[string]interface{}) (map[string]interface{}, error)\n}\n\nfunc InterfaceMap(inter interface{}, v *InterfaceMapValidation) (map[string]interface{}, error) {\n\tcasted, castOk := cast.InterfaceToStrInterfaceMap(inter)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeMap)\n\t}\n\treturn ValidateInterfaceMapProvided(casted, v)\n}\n\nfunc InterfaceMapFromInterfaceMap(key string, iMap map[string]interface{}, v *InterfaceMapValidation) (map[string]interface{}, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInterfaceMapMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := InterfaceMap(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateInterfaceMapMissing(v *InterfaceMapValidation) (map[string]interface{}, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateInterfaceMap(v.Default, v)\n}\n\nfunc ValidateInterfaceMapProvided(val map[string]interface{}, v *InterfaceMapValidation) (map[string]interface{}, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateInterfaceMap(val, v)\n}\n\nfunc validateInterfaceMap(val map[string]interface{}, v *InterfaceMapValidation) (map[string]interface{}, error) {\n\tif v.RequireCortexResources {\n\t\tif err := checkOnlyCortexResources(val); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if !v.AllowCortexResources {\n\t\tif err := checkNoCortexResources(val); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.ScalarsOnly {\n\t\tfor k, v := range val {\n\t\t\tif !cast.IsScalarType(v) {\n\t\t\t\treturn nil, errors.Wrap(ErrorInvalidPrimitiveType(v, PrimTypeString, PrimTypeInt, PrimTypeFloat, PrimTypeBool), k)\n\t\t\t}\n\t\t}\n\t}\n\n\tif v.StringLeavesOnly {\n\t\t_, err := FlattenAllStrValues(val)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif v.AllowedLeafValues != nil {\n\t\tleafVals, err := FlattenAllStrValues(val)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfor _, leafVal := range leafVals {\n\t\t\tif !slices.HasString(v.AllowedLeafValues, leafVal) {\n\t\t\t\treturn nil, ErrorInvalidStr(leafVal, v.AllowedLeafValues[0], v.AllowedLeafValues[1:]...)\n\t\t\t}\n\t\t}\n\t}\n\n\tif v.StringKeysOnly {\n\t\tfor key, value := range val {\n\t\t\tm, ok := cast.InterfaceToInterfaceInterfaceMap(value)\n\t\t\tif !ok {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tstringToIntMap := map[string]interface{}{}\n\t\t\tfor kInterface, vInterface := range m {\n\t\t\t\tkString, ok := kInterface.(string)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn nil, errors.Wrap(ErrorNonStringKeyFound(kInterface), key)\n\t\t\t\t}\n\t\t\t\tstringToIntMap[kString] = vInterface\n\t\t\t}\n\n\t\t\t_, err := validateInterfaceMap(stringToIntMap, v)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrap(err, key)\n\t\t\t}\n\n\t\t\tval[key] = stringToIntMap\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\n\tif val == nil && v.ConvertNullToEmpty {\n\t\tval = make(map[string]interface{})\n\t}\n\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/interface_map_list.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype InterfaceMapListValidation struct {\n\tRequired               bool\n\tDefault                []map[string]interface{}\n\tAllowExplicitNull      bool\n\tAllowEmpty             bool\n\tCantBeSpecifiedErrStr  *string\n\tCastSingleItem         bool\n\tMinLength              int\n\tMaxLength              int\n\tInvalidLengths         []int\n\tAllowCortexResources   bool\n\tRequireCortexResources bool\n\tValidator              func([]map[string]interface{}) ([]map[string]interface{}, error)\n}\n\nfunc InterfaceMapList(inter interface{}, v *InterfaceMapListValidation) ([]map[string]interface{}, error) {\n\tcasted, castOk := cast.InterfaceToStrInterfaceMapSlice(inter)\n\tif !castOk {\n\t\tif v.CastSingleItem {\n\t\t\tcastedItem, castOk := cast.InterfaceToStrInterfaceMap(inter)\n\t\t\tif !castOk {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeMap, PrimTypeMapList)\n\t\t\t}\n\t\t\tcasted = []map[string]interface{}{castedItem}\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeMapList)\n\t\t}\n\t}\n\treturn ValidateInterfaceMapListProvided(casted, v)\n}\n\nfunc InterfaceMapListFromInterfaceMap(key string, iMap map[string]interface{}, v *InterfaceMapListValidation) ([]map[string]interface{}, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateInterfaceMapListMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := InterfaceMapList(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateInterfaceMapListMissing(v *InterfaceMapListValidation) ([]map[string]interface{}, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateInterfaceMapList(v.Default, v)\n}\n\nfunc ValidateInterfaceMapListProvided(val []map[string]interface{}, v *InterfaceMapListValidation) ([]map[string]interface{}, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateInterfaceMapList(val, v)\n}\n\nfunc validateInterfaceMapList(val []map[string]interface{}, v *InterfaceMapListValidation) ([]map[string]interface{}, error) {\n\tif v.RequireCortexResources {\n\t\tif err := checkOnlyCortexResources(val); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if !v.AllowCortexResources {\n\t\tif err := checkNoCortexResources(val); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(val) < v.MinLength {\n\t\t\treturn nil, ErrorTooFewElements(v.MinLength)\n\t\t}\n\t}\n\n\tif v.MaxLength != 0 {\n\t\tif len(val) > v.MaxLength {\n\t\t\treturn nil, ErrorTooManyElements(v.MaxLength)\n\t\t}\n\t}\n\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(val) == invalidLength {\n\t\t\treturn nil, ErrorWrongNumberOfElements(v.InvalidLengths)\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/interface_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestFlattenAllStrValues(t *testing.T) {\n\tvar input interface{}\n\tvar expected []string\n\n\tinput = \"test\"\n\texpected = []string{\"test\"}\n\tCheckFlattenAllStrValues(input, expected, t)\n\n\tinput = []interface{}{\"test1\", \"test2\", \"test3\"}\n\texpected = []string{\"test1\", \"test2\", \"test3\"}\n\tCheckFlattenAllStrValues(input, expected, t)\n\n\tinput = map[interface{}]interface{}{\n\t\t\"k1\": \"test1\",\n\t\t\"k2\": \"test2\",\n\t\t\"k3\": \"test3\",\n\t}\n\texpected = []string{\"test1\", \"test2\", \"test3\"}\n\tCheckFlattenAllStrValues(input, expected, t)\n\n\tinput = map[interface{}]interface{}{\n\t\t\"k1\": []interface{}{\"test1\"},\n\t\t\"k2\": \"test2\",\n\t\t\"k3\": []interface{}{\"test3\", \"test4\"},\n\t}\n\texpected = []string{\"test1\", \"test2\", \"test3\", \"test4\"}\n\tCheckFlattenAllStrValues(input, expected, t)\n\n\tinput = map[string]interface{}{\n\t\t\"k1\": []interface{}{\"test1\"},\n\t\t\"k2\": \"test2\",\n\t\t\"k3\": []interface{}{\"test3\", \"test4\"},\n\t\t\"k4\": map[string]interface{}{\n\t\t\t\"k2\": \"test6\",\n\t\t\t\"k1\": []interface{}{\"test5\"},\n\t\t\t\"k3\": []interface{}{\"test7\", \"test8\"},\n\t\t},\n\t}\n\texpected = []string{\"test1\", \"test2\", \"test3\", \"test4\", \"test5\", \"test6\", \"test7\", \"test8\"}\n\tCheckFlattenAllStrValues(input, expected, t)\n\n\tinput = MustReadYAMLStr(\n\t\t`\n    test:\n      key1: [test5]\n      key2: test6\n      key3: [test7, test8]\n      a2:\n        key1: [test1]\n        key2: test2\n        key3: [test3, test4]\n      z:\n        - key1: [test9]\n          key2: test10\n          key3: [test11, test12]\n        - key1: [test13]\n          key2: test14\n          key3: [test15, test16]\n  `)\n\texpected = []string{\"test1\", \"test2\", \"test3\", \"test4\", \"test5\", \"test6\", \"test7\", \"test8\", \"test9\", \"test10\", \"test11\", \"test12\", \"test13\", \"test14\", \"test15\", \"test16\"}\n\tCheckFlattenAllStrValues(input, expected, t)\n}\n\nfunc CheckFlattenAllStrValues(obj interface{}, expected []string, t *testing.T) {\n\tflattened, err := FlattenAllStrValues(obj)\n\trequire.NoError(t, err)\n\trequire.Equal(t, expected, flattened)\n}\n"
  },
  {
    "path": "pkg/lib/configreader/reader.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/debug\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/maps\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/yaml\"\n)\n\ntype StructFieldValidation struct {\n\tStructField                string                          // Required (can be omitted to skip writing of value to struct)\n\tKey                        string                          // Required, or defaults to json key or \"StructField\"\n\tDefaultField               string                          // Optional. Will set the default to the runtime value of this field\n\tDefaultDependentFields     []string                        // Optional. Will be passed in to DefaultDependentFieldsFunc. Dependent fields must be listed first in the `[]*cr.StructFieldValidation`.\n\tDefaultDependentFieldsFunc func([]interface{}) interface{} // Optional. Will be called with DefaultDependentFields\n\n\t// Provide one of the following:\n\tStringValidation              *StringValidation\n\tStringPtrValidation           *StringPtrValidation\n\tStringListValidation          *StringListValidation\n\tBoolValidation                *BoolValidation\n\tBoolPtrValidation             *BoolPtrValidation\n\tBoolListValidation            *BoolListValidation\n\tIntValidation                 *IntValidation\n\tIntPtrValidation              *IntPtrValidation\n\tIntListValidation             *IntListValidation\n\tInt32Validation               *Int32Validation\n\tInt32PtrValidation            *Int32PtrValidation\n\tInt32ListValidation           *Int32ListValidation\n\tInt64Validation               *Int64Validation\n\tInt64PtrValidation            *Int64PtrValidation\n\tInt64ListValidation           *Int64ListValidation\n\tFloat32Validation             *Float32Validation\n\tFloat32PtrValidation          *Float32PtrValidation\n\tFloat32ListValidation         *Float32ListValidation\n\tFloat64Validation             *Float64Validation\n\tFloat64PtrValidation          *Float64PtrValidation\n\tFloat64ListValidation         *Float64ListValidation\n\tStringMapValidation           *StringMapValidation\n\tInterfaceMapValidation        *InterfaceMapValidation\n\tInterfaceMapListValidation    *InterfaceMapListValidation\n\tInterfaceValidation           *InterfaceValidation\n\tStructValidation              *StructValidation\n\tStructListValidation          *StructListValidation\n\tInterfaceStructValidation     *InterfaceStructValidation\n\tInterfaceStructListValidation *InterfaceStructListValidation\n\tNil                           bool\n\n\t// Additional parsing step for StringValidation or StringPtrValidation\n\tParser func(string) (interface{}, error)\n}\n\ntype StructValidation struct {\n\tStructFieldValidations []*StructFieldValidation\n\tRequired               bool\n\tAllowExplicitNull      bool\n\tTreatNullAsEmpty       bool // If explicit null or if it's top level and the file is empty, treat as empty map\n\tDefaultNil             bool // If this struct is nested and its key is not defined, set it to nil instead of defaults or erroring (e.g. if any subfields are required)\n\tCantBeSpecifiedErrStr  *string\n\tShortCircuit           bool\n\tAllowExtraFields       bool\n}\n\ntype StructListValidation struct {\n\tStructValidation      *StructValidation\n\tRequired              bool\n\tAllowExplicitNull     bool\n\tTreatNullAsEmpty      bool // If explicit null or if it's top level and the file is empty, treat as empty list\n\tMinLength             int\n\tMaxLength             int\n\tInvalidLengths        []int\n\tCantBeSpecifiedErrStr *string\n\tShortCircuit          bool\n}\n\ntype InterfaceStructValidation struct {\n\tTypeKey                    string                               // required\n\tTypeStructField            string                               // optional (will set this field if present)\n\tInterfaceStructTypes       map[string]*InterfaceStructType      // specify this or ParsedInterfaceStructTypes\n\tParsedInterfaceStructTypes map[interface{}]*InterfaceStructType // must specify Parser if using this\n\tParser                     func(string) (interface{}, error)\n\tRequired                   bool\n\tAllowExplicitNull          bool\n\tTreatNullAsEmpty           bool // If explicit null or if it's top level and the file is empty, treat as empty map\n\tCantBeSpecifiedErrStr      *string\n\tShortCircuit               bool\n\tAllowExtraFields           bool\n}\n\ntype InterfaceStructType struct {\n\tType                   interface{} // e.g. (*MyType)(nil)\n\tStructFieldValidations []*StructFieldValidation\n}\n\ntype InterfaceStructListValidation struct {\n\tInterfaceStructValidation *InterfaceStructValidation\n\tRequired                  bool\n\tAllowExplicitNull         bool\n\tTreatNullAsEmpty          bool // If explicit null or if it's top level and the file is empty, treat as empty map\n\tCantBeSpecifiedErrStr     *string\n\tShortCircuit              bool\n}\n\nfunc Struct(dest interface{}, inter interface{}, v *StructValidation) []error {\n\tallowedFields := []string{}\n\tallErrs := []error{}\n\tvar ok bool\n\n\tif inter == nil {\n\t\tif v.TreatNullAsEmpty {\n\t\t\tinter = make(map[interface{}]interface{}, 0)\n\t\t} else {\n\t\t\tif !v.AllowExplicitNull {\n\t\t\t\treturn []error{ErrorCannotBeEmptyOrNull(v.Required)}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tinterMap, ok := cast.InterfaceToStrInterfaceMap(inter)\n\tif !ok {\n\t\treturn []error{ErrorInvalidPrimitiveType(inter, PrimTypeMap)}\n\t}\n\n\tfor _, structFieldValidation := range v.StructFieldValidations {\n\t\tkey := inferKey(reflect.TypeOf(dest), structFieldValidation.StructField, structFieldValidation.Key)\n\t\tallowedFields = append(allowedFields, key)\n\n\t\tif structFieldValidation.Nil == true {\n\t\t\tcontinue\n\t\t}\n\n\t\tvar err error\n\t\tvar errs []error\n\t\tvar val interface{}\n\n\t\tif structFieldValidation.StringValidation != nil {\n\t\t\tvalidation := *structFieldValidation.StringValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = StringFromInterfaceMap(key, interMap, &validation)\n\t\t\tif err == nil && structFieldValidation.Parser != nil {\n\t\t\t\tval, err = structFieldValidation.Parser(val.(string))\n\t\t\t\terr = errors.Wrap(err, key)\n\t\t\t}\n\t\t} else if structFieldValidation.StringPtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.StringPtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = StringPtrFromInterfaceMap(key, interMap, &validation)\n\t\t\tif err == nil && structFieldValidation.Parser != nil {\n\t\t\t\tif val.(*string) == nil {\n\t\t\t\t\tval = nil\n\t\t\t\t} else {\n\t\t\t\t\tval, err = structFieldValidation.Parser(*val.(*string))\n\t\t\t\t\tif err == nil && val != nil {\n\t\t\t\t\t\tvalValue := reflect.ValueOf(val)\n\t\t\t\t\t\tvalPtrValue := reflect.New(valValue.Type())\n\t\t\t\t\t\tvalPtrValue.Elem().Set(valValue)\n\t\t\t\t\t\tval = valPtrValue.Interface()\n\t\t\t\t\t} else {\n\t\t\t\t\t\tval = nil\n\t\t\t\t\t\terr = errors.Wrap(err, key)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else if structFieldValidation.StringListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.StringListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = StringListFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.BoolValidation != nil {\n\t\t\tvalidation := *structFieldValidation.BoolValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = BoolFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.BoolPtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.BoolPtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = BoolPtrFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.BoolListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.BoolListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = BoolListFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.IntValidation != nil {\n\t\t\tvalidation := *structFieldValidation.IntValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = IntFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.IntPtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.IntPtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = IntPtrFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.IntListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.IntListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = IntListFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Int32Validation != nil {\n\t\t\tvalidation := *structFieldValidation.Int32Validation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Int32FromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Int32PtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Int32PtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Int32PtrFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Int32ListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Int32ListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Int32ListFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Int64Validation != nil {\n\t\t\tvalidation := *structFieldValidation.Int64Validation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Int64FromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Int64PtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Int64PtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Int64PtrFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Int64ListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Int64ListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Int64ListFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Float32Validation != nil {\n\t\t\tvalidation := *structFieldValidation.Float32Validation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Float32FromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Float32PtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Float32PtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Float32PtrFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Float64Validation != nil {\n\t\t\tvalidation := *structFieldValidation.Float64Validation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Float64FromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Float64PtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Float64PtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Float64PtrFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Float32ListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Float32ListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Float32ListFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.Float64ListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Float64ListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = Float64ListFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.StringMapValidation != nil {\n\t\t\tvalidation := *structFieldValidation.StringMapValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = StringMapFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.InterfaceMapValidation != nil {\n\t\t\tvalidation := *structFieldValidation.InterfaceMapValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = InterfaceMapFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.InterfaceMapListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.InterfaceMapListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = InterfaceMapListFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.InterfaceValidation != nil {\n\t\t\tvalidation := *structFieldValidation.InterfaceValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tval, err = InterfaceFromInterfaceMap(key, interMap, &validation)\n\t\t} else if structFieldValidation.StructValidation != nil {\n\t\t\tvalidation := *structFieldValidation.StructValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tnestedType := reflect.ValueOf(dest).Elem().FieldByName(structFieldValidation.StructField).Type()\n\t\t\tinterMapVal, ok := ReadInterfaceMapValue(key, interMap)\n\t\t\tif ok && validation.CantBeSpecifiedErrStr != nil {\n\t\t\t\terr = errors.Wrap(ErrorFieldCantBeSpecified(*validation.CantBeSpecifiedErrStr), key)\n\t\t\t} else if !ok && validation.Required {\n\t\t\t\terr = errors.Wrap(ErrorMustBeDefined(), key)\n\t\t\t} else if !ok && validation.DefaultNil {\n\t\t\t\tval = nil\n\t\t\t} else {\n\t\t\t\tif !ok {\n\t\t\t\t\tinterMapVal = make(map[string]interface{}) // Here validation.DefaultNil == false, so create an empty map to hold the nested default values\n\t\t\t\t}\n\t\t\t\tval = reflect.New(nestedType.Elem()).Interface()\n\t\t\t\terrs = Struct(val, interMapVal, &validation)\n\t\t\t\tif interMapVal == nil {\n\t\t\t\t\tval = nil // If the object was nil, set val to nil rather than a pointer to the initialized zero value\n\t\t\t\t}\n\t\t\t\terrs = errors.WrapAll(errs, key)\n\t\t\t}\n\n\t\t} else if structFieldValidation.StructListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.StructListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tnestedType := reflect.ValueOf(dest).Elem().FieldByName(structFieldValidation.StructField).Type()\n\t\t\tinterMapVal, ok := ReadInterfaceMapValue(key, interMap)\n\t\t\tif ok && validation.CantBeSpecifiedErrStr != nil {\n\t\t\t\terr = errors.Wrap(ErrorFieldCantBeSpecified(*validation.CantBeSpecifiedErrStr), key)\n\t\t\t} else if !ok && validation.Required {\n\t\t\t\terr = errors.Wrap(ErrorMustBeDefined(), key)\n\t\t\t} else {\n\t\t\t\tval = reflect.Indirect(reflect.New(nestedType)).Interface()\n\t\t\t\tval, errs = StructList(val, interMapVal, &validation)\n\t\t\t\terrs = errors.WrapAll(errs, key)\n\t\t\t}\n\n\t\t} else if structFieldValidation.InterfaceStructValidation != nil {\n\t\t\tvalidation := *structFieldValidation.InterfaceStructValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tinterMapVal, ok := ReadInterfaceMapValue(key, interMap)\n\t\t\tif ok && validation.CantBeSpecifiedErrStr != nil {\n\t\t\t\terr = errors.Wrap(ErrorFieldCantBeSpecified(*validation.CantBeSpecifiedErrStr), key)\n\t\t\t} else if !ok && validation.Required {\n\t\t\t\terr = errors.Wrap(ErrorMustBeDefined(), key)\n\t\t\t} else {\n\t\t\t\tval, errs = InterfaceStruct(interMapVal, &validation)\n\t\t\t\terrs = errors.WrapAll(errs, key)\n\t\t\t}\n\n\t\t} else if structFieldValidation.InterfaceStructListValidation != nil {\n\t\t\tvalidation := *structFieldValidation.InterfaceStructListValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tnestedType := reflect.ValueOf(dest).Elem().FieldByName(structFieldValidation.StructField).Type()\n\t\t\tinterMapVal, ok := ReadInterfaceMapValue(key, interMap)\n\t\t\tif ok && validation.CantBeSpecifiedErrStr != nil {\n\t\t\t\terr = errors.Wrap(ErrorFieldCantBeSpecified(*validation.CantBeSpecifiedErrStr), key)\n\t\t\t} else if !ok && validation.Required {\n\t\t\t\terr = errors.Wrap(ErrorMustBeDefined(), key)\n\t\t\t} else {\n\t\t\t\tval = reflect.Indirect(reflect.New(nestedType)).Interface()\n\t\t\t\tval, errs = InterfaceStructList(val, interMapVal, &validation)\n\t\t\t\terrs = errors.WrapAll(errs, key)\n\t\t\t}\n\n\t\t} else {\n\t\t\texit.Panic(ErrorUnsupportedFieldValidation())\n\t\t}\n\n\t\tallErrs, _ = errors.AddError(allErrs, err)\n\t\tallErrs, _ = errors.AddErrors(allErrs, errs)\n\t\tif errors.HasError(allErrs) {\n\t\t\tif v.ShortCircuit {\n\t\t\t\treturn allErrs\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tif structFieldValidation.StructField != \"\" {\n\t\t\tif val == nil {\n\t\t\t\terr = setFieldNil(dest, structFieldValidation.StructField)\n\t\t\t} else {\n\t\t\t\terr = setField(val, dest, structFieldValidation.StructField)\n\t\t\t}\n\t\t\tif allErrs, ok = errors.AddError(allErrs, err, key); ok {\n\t\t\t\tif v.ShortCircuit {\n\t\t\t\t\treturn allErrs\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif !v.AllowExtraFields {\n\t\textraFields := slices.SubtractStrSlice(maps.InterfaceMapKeys(interMap), allowedFields)\n\t\tfor _, extraField := range extraFields {\n\t\t\tallErrs = append(allErrs, ErrorUnsupportedKey(extraField))\n\t\t}\n\t}\n\tif errors.HasError(allErrs) {\n\t\treturn allErrs\n\t}\n\treturn nil\n}\n\nfunc StructList(dest interface{}, inter interface{}, v *StructListValidation) (interface{}, []error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsEmpty {\n\t\t\tinter = make([]interface{}, 0)\n\t\t} else {\n\t\t\tif !v.AllowExplicitNull {\n\t\t\t\treturn nil, []error{ErrorCannotBeEmptyOrNull(v.Required)}\n\t\t\t}\n\t\t\treturn nil, nil\n\t\t}\n\t}\n\n\tinterSlice, ok := cast.InterfaceToInterfaceSlice(inter)\n\tif !ok {\n\t\treturn nil, []error{ErrorInvalidPrimitiveType(inter, PrimTypeList)}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(interSlice) < v.MinLength {\n\t\t\treturn nil, []error{ErrorTooFewElements(v.MinLength)}\n\t\t}\n\t}\n\tif v.MaxLength != 0 {\n\t\tif len(interSlice) > v.MaxLength {\n\t\t\treturn nil, []error{ErrorTooManyElements(v.MaxLength)}\n\t\t}\n\t}\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(interSlice) == invalidLength {\n\t\t\treturn nil, []error{ErrorWrongNumberOfElements(v.InvalidLengths)}\n\t\t}\n\t}\n\n\terrs := []error{}\n\tfor i, interItem := range interSlice {\n\t\tval := reflect.New(reflect.ValueOf(dest).Type().Elem().Elem()).Interface()\n\t\tsubErrs := Struct(val, interItem, v.StructValidation)\n\t\tvar ok bool\n\t\tif errs, ok = errors.AddErrors(errs, subErrs, s.Index(i)); ok {\n\t\t\tif v.ShortCircuit {\n\t\t\t\treturn nil, errs\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif interItem == nil {\n\t\t\tval = nil // If the object was nil, set val to nil rather than a pointer to the initialized zero value\n\t\t}\n\t\tdest = appendVal(dest, val)\n\t}\n\n\treturn dest, errs\n}\n\nfunc InterfaceStruct(inter interface{}, v *InterfaceStructValidation) (interface{}, []error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsEmpty {\n\t\t\tinter = make(map[interface{}]interface{}, 0)\n\t\t} else {\n\t\t\tif !v.AllowExplicitNull {\n\t\t\t\treturn nil, []error{ErrorCannotBeEmptyOrNull(v.Required)}\n\t\t\t}\n\t\t\treturn nil, nil\n\t\t}\n\t}\n\n\tinterMap, ok := cast.InterfaceToStrInterfaceMap(inter)\n\tif !ok {\n\t\treturn nil, []error{ErrorInvalidPrimitiveType(inter, PrimTypeMap)}\n\t}\n\n\tvar validTypeStrs []string\n\tif v.InterfaceStructTypes != nil {\n\t\tfor typeStr := range v.InterfaceStructTypes {\n\t\t\tvalidTypeStrs = append(validTypeStrs, typeStr)\n\t\t}\n\t}\n\n\ttypeStrValidation := &StringValidation{\n\t\tRequired:      true,\n\t\tAllowedValues: validTypeStrs,\n\t}\n\n\ttypeStr, err := StringFromInterfaceMap(v.TypeKey, interMap, typeStrValidation)\n\tif err != nil {\n\t\treturn nil, []error{err}\n\t}\n\tvar typeObj interface{}\n\tif v.Parser != nil {\n\t\ttypeObj, err = v.Parser(typeStr)\n\t\tif err != nil {\n\t\t\treturn nil, []error{errors.Wrap(err, v.TypeKey)}\n\t\t}\n\t}\n\n\tvar typeFieldValidation *StructFieldValidation\n\tif v.TypeStructField == \"\" {\n\t\ttypeFieldValidation = &StructFieldValidation{\n\t\t\tKey: v.TypeKey,\n\t\t\tNil: true,\n\t\t}\n\t} else {\n\t\ttypeFieldValidation = &StructFieldValidation{\n\t\t\tKey:              v.TypeKey,\n\t\t\tStructField:      v.TypeStructField,\n\t\t\tStringValidation: typeStrValidation,\n\t\t\tParser:           v.Parser,\n\t\t}\n\t}\n\n\tvar structType *InterfaceStructType\n\tif v.InterfaceStructTypes != nil {\n\t\tstructType = v.InterfaceStructTypes[typeStr]\n\t} else {\n\t\tstructType = v.ParsedInterfaceStructTypes[typeObj]\n\t\tif structType == nil {\n\t\t\t// This error case may or may not be handled by v.Parser()\n\t\t\tvar validTypeObjs []interface{}\n\t\t\tfor typeObj := range v.ParsedInterfaceStructTypes {\n\t\t\t\tvalidTypeObjs = append(validTypeObjs, typeObj)\n\t\t\t}\n\t\t\treturn nil, []error{errors.Wrap(ErrorInvalidInterface(typeStr, validTypeObjs[0], validTypeObjs[1:]...), v.TypeKey)}\n\t\t}\n\t}\n\n\tval := reflect.New(reflect.TypeOf(structType.Type).Elem()).Interface()\n\tstructValidation := &StructValidation{\n\t\tStructFieldValidations: append(structType.StructFieldValidations, typeFieldValidation),\n\t\tRequired:               v.Required,\n\t\tAllowExplicitNull:      v.AllowExplicitNull,\n\t\tShortCircuit:           v.ShortCircuit,\n\t\tAllowExtraFields:       v.AllowExtraFields,\n\t}\n\terrs := Struct(val, inter, structValidation)\n\treturn val, errs\n}\n\nfunc InterfaceStructList(dest interface{}, inter interface{}, v *InterfaceStructListValidation) (interface{}, []error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsEmpty {\n\t\t\tinter = make([]interface{}, 0)\n\t\t} else {\n\t\t\tif !v.AllowExplicitNull {\n\t\t\t\treturn nil, []error{ErrorCannotBeEmptyOrNull(v.Required)}\n\t\t\t}\n\t\t\treturn nil, nil\n\t\t}\n\t}\n\n\tinterSlice, ok := cast.InterfaceToInterfaceSlice(inter)\n\tif !ok {\n\t\treturn nil, []error{ErrorInvalidPrimitiveType(inter, PrimTypeList)}\n\t}\n\n\terrs := []error{}\n\tfor i, interItem := range interSlice {\n\t\tval, subErrs := InterfaceStruct(interItem, v.InterfaceStructValidation)\n\t\tvar ok bool\n\t\tif errs, ok = errors.AddErrors(errs, subErrs, s.Index(i)); ok {\n\t\t\tif v.ShortCircuit {\n\t\t\t\treturn nil, errs\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tdest = appendVal(dest, val)\n\t}\n\n\treturn dest, errs\n}\n\nfunc updateValidation(validation interface{}, dest interface{}, structFieldValidation *StructFieldValidation) {\n\tif structFieldValidation.DefaultField != \"\" {\n\t\truntimeVal := reflect.ValueOf(dest).Elem().FieldByName(structFieldValidation.DefaultField).Interface()\n\t\tsetField(runtimeVal, validation, \"Default\")\n\t} else if structFieldValidation.DefaultDependentFieldsFunc != nil {\n\t\truntimeVals := make([]interface{}, len(structFieldValidation.DefaultDependentFields))\n\t\tfor i, fieldName := range structFieldValidation.DefaultDependentFields {\n\t\t\truntimeVals[i] = reflect.ValueOf(dest).Elem().FieldByName(fieldName).Interface()\n\t\t}\n\t\tval := structFieldValidation.DefaultDependentFieldsFunc(runtimeVals)\n\t\tsetField(val, validation, \"Default\")\n\t}\n}\n\nfunc ReadInterfaceMapValue(name string, interMap map[string]interface{}) (interface{}, bool) {\n\tif interMap == nil {\n\t\treturn nil, false\n\t}\n\n\tval, ok := interMap[name]\n\tif !ok {\n\t\treturn nil, false\n\t}\n\treturn val, true\n}\n\n//\n// Prompt\n//\n\ntype PromptItemValidation struct {\n\tStructField string          // Required\n\tPromptOpts  *prompt.Options // Required\n\n\t// Provide one of the following:\n\tStringValidation     *StringValidation\n\tStringPtrValidation  *StringPtrValidation\n\tBoolValidation       *BoolValidation\n\tBoolPtrValidation    *BoolPtrValidation\n\tIntValidation        *IntValidation\n\tIntPtrValidation     *IntPtrValidation\n\tInt32Validation      *Int32Validation\n\tInt32PtrValidation   *Int32PtrValidation\n\tInt64Validation      *Int64Validation\n\tInt64PtrValidation   *Int64PtrValidation\n\tFloat32Validation    *Float32Validation\n\tFloat32PtrValidation *Float32PtrValidation\n\tFloat64Validation    *Float64Validation\n\tFloat64PtrValidation *Float64PtrValidation\n\n\t// Additional parsing step for StringValidation or StringPtrValidation\n\tParser func(string) (interface{}, error)\n}\n\ntype PromptValidation struct {\n\tPromptItemValidations  []*PromptItemValidation\n\tSkipNonEmptyFields     bool // skips fields that are not zero-valued\n\tSkipNonNilFields       bool // skips pointer fields that are not nil\n\tPrintNewLineIfPrompted bool // prints an extra new line at the end if any questions were asked\n}\n\nfunc ReadPrompt(dest interface{}, promptValidation *PromptValidation) error {\n\tvar val interface{}\n\tvar err error\n\tshouldPrintTrailingNewLine := false\n\n\t// Validate any skipped fields first, so that any errors are returned before prompting\n\tif promptValidation.SkipNonEmptyFields {\n\t\tfor _, promptItemValidation := range promptValidation.PromptItemValidations {\n\t\t\tv := reflect.ValueOf(dest).Elem().FieldByName(promptItemValidation.StructField)\n\t\t\tif !v.IsZero() {\n\t\t\t\tif promptItemValidation.StringValidation != nil && promptItemValidation.Parser == nil {\n\t\t\t\t\tif _, err := ValidateStringProvided(v.Interface().(string), promptItemValidation.StringValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.StringPtrValidation != nil && promptItemValidation.Parser == nil {\n\t\t\t\t\tif _, err := ValidateStringPtrProvided(v.Interface().(*string), promptItemValidation.StringPtrValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.BoolValidation != nil {\n\t\t\t\t\tif _, err := ValidateBoolProvided(v.Interface().(bool), promptItemValidation.BoolValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.BoolPtrValidation != nil {\n\t\t\t\t\tif _, err := ValidateBoolPtrProvided(v.Interface().(*bool), promptItemValidation.BoolPtrValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.IntValidation != nil {\n\t\t\t\t\tif _, err := ValidateIntProvided(v.Interface().(int), promptItemValidation.IntValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.IntPtrValidation != nil {\n\t\t\t\t\tif _, err := ValidateIntPtrProvided(v.Interface().(*int), promptItemValidation.IntPtrValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.Int32Validation != nil {\n\t\t\t\t\tif _, err := ValidateInt32Provided(v.Interface().(int32), promptItemValidation.Int32Validation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.Int32PtrValidation != nil {\n\t\t\t\t\tif _, err := ValidateInt32PtrProvided(v.Interface().(*int32), promptItemValidation.Int32PtrValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.Int64Validation != nil {\n\t\t\t\t\tif _, err := ValidateInt64Provided(v.Interface().(int64), promptItemValidation.Int64Validation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.Int64PtrValidation != nil {\n\t\t\t\t\tif _, err := ValidateInt64PtrProvided(v.Interface().(*int64), promptItemValidation.Int64PtrValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.Float32Validation != nil {\n\t\t\t\t\tif _, err := ValidateFloat32Provided(v.Interface().(float32), promptItemValidation.Float32Validation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.Float32PtrValidation != nil {\n\t\t\t\t\tif _, err := ValidateFloat32PtrProvided(v.Interface().(*float32), promptItemValidation.Float32PtrValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.Float64Validation != nil {\n\t\t\t\t\tif _, err := ValidateFloat64Provided(v.Interface().(float64), promptItemValidation.Float64Validation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t} else if promptItemValidation.Float64PtrValidation != nil {\n\t\t\t\t\tif _, err := ValidateFloat64PtrProvided(v.Interface().(*float64), promptItemValidation.Float64PtrValidation); err != nil {\n\t\t\t\t\t\treturn errors.Wrap(err, inferPromptFieldName(reflect.TypeOf(dest), promptItemValidation.StructField))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, promptItemValidation := range promptValidation.PromptItemValidations {\n\t\tif promptValidation.SkipNonEmptyFields {\n\t\t\tv := reflect.ValueOf(dest).Elem().FieldByName(promptItemValidation.StructField)\n\t\t\tif !v.IsZero() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t} else if promptValidation.SkipNonNilFields {\n\t\t\tv := reflect.ValueOf(dest).Elem().FieldByName(promptItemValidation.StructField)\n\t\t\tif v.Kind() == reflect.Ptr && !v.IsNil() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif promptValidation.PrintNewLineIfPrompted {\n\t\t\tshouldPrintTrailingNewLine = true\n\t\t}\n\n\t\tfor {\n\t\t\tif promptItemValidation.StringValidation != nil {\n\t\t\t\tval, err = StringFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.StringValidation)\n\t\t\t\tif err == nil && promptItemValidation.Parser != nil {\n\t\t\t\t\tval, err = promptItemValidation.Parser(val.(string))\n\t\t\t\t}\n\t\t\t} else if promptItemValidation.StringPtrValidation != nil {\n\t\t\t\tval, err = StringPtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.StringPtrValidation)\n\t\t\t\tif err == nil && promptItemValidation.Parser != nil {\n\t\t\t\t\tif val.(*string) == nil {\n\t\t\t\t\t\tval = nil\n\t\t\t\t\t} else {\n\t\t\t\t\t\tval, err = promptItemValidation.Parser(*val.(*string))\n\t\t\t\t\t\tif err == nil && val != nil {\n\t\t\t\t\t\t\tvalValue := reflect.ValueOf(val)\n\t\t\t\t\t\t\tvalPtrValue := reflect.New(valValue.Type())\n\t\t\t\t\t\t\tvalPtrValue.Elem().Set(valValue)\n\t\t\t\t\t\t\tval = valPtrValue.Interface()\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tval = nil\n\t\t\t\t\t\t\t// err is already set\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if promptItemValidation.BoolValidation != nil {\n\t\t\t\tval, err = BoolFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.BoolValidation)\n\t\t\t} else if promptItemValidation.BoolPtrValidation != nil {\n\t\t\t\tval, err = BoolPtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.BoolPtrValidation)\n\t\t\t} else if promptItemValidation.IntValidation != nil {\n\t\t\t\tval, err = IntFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.IntValidation)\n\t\t\t} else if promptItemValidation.IntPtrValidation != nil {\n\t\t\t\tval, err = IntPtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.IntPtrValidation)\n\t\t\t} else if promptItemValidation.Int32Validation != nil {\n\t\t\t\tval, err = Int32FromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Int32Validation)\n\t\t\t} else if promptItemValidation.Int32PtrValidation != nil {\n\t\t\t\tval, err = Int32PtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Int32PtrValidation)\n\t\t\t} else if promptItemValidation.Int64Validation != nil {\n\t\t\t\tval, err = Int64FromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Int64Validation)\n\t\t\t} else if promptItemValidation.Int64PtrValidation != nil {\n\t\t\t\tval, err = Int64PtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Int64PtrValidation)\n\t\t\t} else if promptItemValidation.Float32Validation != nil {\n\t\t\t\tval, err = Float32FromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Float32Validation)\n\t\t\t} else if promptItemValidation.Float32PtrValidation != nil {\n\t\t\t\tval, err = Float32PtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Float32PtrValidation)\n\t\t\t} else if promptItemValidation.Float64Validation != nil {\n\t\t\t\tval, err = Float64FromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Float64Validation)\n\t\t\t} else if promptItemValidation.Float64PtrValidation != nil {\n\t\t\t\tval, err = Float64PtrFromPrompt(promptItemValidation.PromptOpts, promptItemValidation.Float64PtrValidation)\n\t\t\t} else {\n\t\t\t\texit.Panic(ErrorUnsupportedFieldValidation())\n\t\t\t}\n\n\t\t\tif err == nil {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tif promptItemValidation.PromptOpts.SkipTrailingNewline {\n\t\t\t\tfmt.Printf(\"error: %s\\n\", errors.Message(err))\n\t\t\t} else {\n\t\t\t\tfmt.Printf(\"error: %s\\n\\n\", errors.Message(err))\n\t\t\t}\n\t\t}\n\n\t\tif val == nil {\n\t\t\terr = setFieldNil(dest, promptItemValidation.StructField)\n\t\t} else {\n\t\t\terr = setField(val, dest, promptItemValidation.StructField)\n\t\t}\n\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif shouldPrintTrailingNewLine {\n\t\tfmt.Println()\n\t}\n\n\treturn nil\n}\n\n// Reads a string map into a struct\nfunc StructFromStringMap(dest interface{}, strMap map[string]string, v *StructValidation) []error {\n\tallowedFields := []string{}\n\tallErrs := []error{}\n\tvar ok bool\n\n\tif strMap == nil {\n\t\tif v.TreatNullAsEmpty {\n\t\t\tstrMap = make(map[string]string, 0)\n\t\t} else {\n\t\t\tif !v.AllowExplicitNull {\n\t\t\t\treturn []error{ErrorCannotBeEmptyOrNull(v.Required)}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tfor _, structFieldValidation := range v.StructFieldValidations {\n\t\tkey := inferKey(reflect.TypeOf(dest), structFieldValidation.StructField, structFieldValidation.Key)\n\t\tallowedFields = append(allowedFields, key)\n\n\t\tif structFieldValidation.Nil == true {\n\t\t\tcontinue\n\t\t}\n\n\t\tstrMapVal, keyExists := strMap[key]\n\n\t\tvar err error\n\t\tvar errs []error\n\t\tvar val interface{}\n\n\t\tif structFieldValidation.StringValidation != nil {\n\t\t\tvalidation := *structFieldValidation.StringValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = StringFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateStringMissing(&validation)\n\t\t\t}\n\t\t\tif err == nil && structFieldValidation.Parser != nil {\n\t\t\t\tval, err = structFieldValidation.Parser(val.(string))\n\t\t\t}\n\t\t} else if structFieldValidation.StringPtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.StringPtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = StringPtrFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateStringPtrMissing(&validation)\n\t\t\t}\n\t\t\tif err == nil && structFieldValidation.Parser != nil {\n\t\t\t\tif val.(*string) == nil {\n\t\t\t\t\tval = nil\n\t\t\t\t} else {\n\t\t\t\t\tval, err = structFieldValidation.Parser(*val.(*string))\n\t\t\t\t\tif err == nil && val != nil {\n\t\t\t\t\t\tvalValue := reflect.ValueOf(val)\n\t\t\t\t\t\tvalPtrValue := reflect.New(valValue.Type())\n\t\t\t\t\t\tvalPtrValue.Elem().Set(valValue)\n\t\t\t\t\t\tval = valPtrValue.Interface()\n\t\t\t\t\t} else {\n\t\t\t\t\t\tval = nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} else if structFieldValidation.BoolValidation != nil {\n\t\t\tvalidation := *structFieldValidation.BoolValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = BoolFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateBoolMissing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.BoolPtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.BoolPtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = BoolPtrFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateBoolPtrMissing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.IntValidation != nil {\n\t\t\tvalidation := *structFieldValidation.IntValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = IntFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateIntMissing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.IntPtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.IntPtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = IntPtrFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateIntPtrMissing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.Int32Validation != nil {\n\t\t\tvalidation := *structFieldValidation.Int32Validation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = Int32FromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateInt32Missing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.Int32PtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Int32PtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = Int32PtrFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateInt32PtrMissing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.Int64Validation != nil {\n\t\t\tvalidation := *structFieldValidation.Int64Validation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = Int64FromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateInt64Missing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.Int64PtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Int64PtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = Int64PtrFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateInt64PtrMissing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.Float32Validation != nil {\n\t\t\tvalidation := *structFieldValidation.Float32Validation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = Float32FromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateFloat32Missing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.Float32PtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Float32PtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = Float32PtrFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateFloat32PtrMissing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.Float64Validation != nil {\n\t\t\tvalidation := *structFieldValidation.Float64Validation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = Float64FromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateFloat64Missing(&validation)\n\t\t\t}\n\t\t} else if structFieldValidation.Float64PtrValidation != nil {\n\t\t\tvalidation := *structFieldValidation.Float64PtrValidation\n\t\t\tupdateValidation(&validation, dest, structFieldValidation)\n\t\t\tif keyExists {\n\t\t\t\tval, err = Float64PtrFromStr(strMapVal, &validation)\n\t\t\t} else {\n\t\t\t\tval, err = ValidateFloat64PtrMissing(&validation)\n\t\t\t}\n\t\t} else {\n\t\t\texit.Panic(ErrorUnsupportedFieldValidation())\n\t\t}\n\n\t\terr = errors.Wrap(err, key)\n\t\terrs = errors.WrapAll(errs, key)\n\n\t\tallErrs, _ = errors.AddError(allErrs, err)\n\t\tallErrs, _ = errors.AddErrors(allErrs, errs)\n\t\tif errors.HasError(allErrs) {\n\t\t\tif v.ShortCircuit {\n\t\t\t\treturn allErrs\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tif val == nil {\n\t\t\terr = setFieldNil(dest, structFieldValidation.StructField)\n\t\t} else {\n\t\t\terr = setField(val, dest, structFieldValidation.StructField)\n\t\t}\n\t\tif allErrs, ok = errors.AddError(allErrs, err, key); ok {\n\t\t\tif v.ShortCircuit {\n\t\t\t\treturn allErrs\n\t\t\t}\n\t\t}\n\t}\n\n\tif !v.AllowExtraFields {\n\t\textraFields := slices.SubtractStrSlice(maps.StrMapKeysString(strMap), allowedFields)\n\t\tfor _, extraField := range extraFields {\n\t\t\tallErrs = append(allErrs, ErrorUnsupportedKey(extraField))\n\t\t}\n\t}\n\tif errors.HasError(allErrs) {\n\t\treturn allErrs\n\t}\n\treturn nil\n}\n\n// Reads a directory of files into a struct, where each file name is the key and the contents is the value\nfunc StructFromFiles(dest interface{}, dirPath string, v *StructValidation) []error {\n\tstrMap := map[string]string{}\n\n\tfileNames, err := files.ListDir(dirPath, true)\n\tif err != nil {\n\t\treturn []error{err}\n\t}\n\n\tfor _, fileName := range fileNames {\n\t\tfileBytes, err := files.ReadFileBytes(filepath.Join(dirPath, fileName))\n\t\tif err != nil {\n\t\t\treturn []error{err}\n\t\t}\n\t\tstrMap[fileName] = strings.TrimSpace(string(fileBytes))\n\t}\n\n\treturn StructFromStringMap(dest, strMap, v)\n}\n\n//\n// Environment variable\n//\n\nfunc ReadEnvVar(envVarName string) *string {\n\tenvVar, envVarIsSet := os.LookupEnv(envVarName)\n\tif envVarIsSet {\n\t\treturn &envVar\n\t}\n\treturn nil\n}\n\n//\n// JSON and YAML Config\n//\n\nfunc ParseYAMLFile(dest interface{}, validation *StructValidation, filePath string) []error {\n\tfileInterface, err := ReadYAMLFile(filePath)\n\tif err != nil {\n\t\treturn []error{err}\n\t}\n\n\terrs := Struct(dest, fileInterface, validation)\n\tif errors.HasError(errs) {\n\t\treturn errors.WrapAll(errs, filePath)\n\t}\n\n\treturn nil\n}\n\nfunc ParseYAMLBytes(dest interface{}, validation *StructValidation, data []byte) error {\n\tfileInterface, err := ReadYAMLBytes(data)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terrs := Struct(dest, fileInterface, validation)\n\tif errors.HasError(errs) {\n\t\treturn errors.FirstError(errs...)\n\t}\n\n\treturn nil\n}\n\nfunc ReadYAMLFile(filePath string) (interface{}, error) {\n\tfileBytes, err := files.ReadFileBytes(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfileInterface, err := ReadYAMLBytes(fileBytes)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, filePath)\n\t}\n\n\treturn fileInterface, nil\n}\n\nfunc ReadYAMLFileStrMap(filePath string) (map[string]interface{}, error) {\n\tparsed, err := ReadYAMLFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcasted, ok := cast.InterfaceToStrInterfaceMap(parsed)\n\tif !ok {\n\t\treturn nil, ErrorInvalidPrimitiveType(parsed, PrimTypeMap)\n\t}\n\treturn casted, nil\n}\n\nfunc ReadYAMLBytes(yamlBytes []byte) (interface{}, error) {\n\tif len(yamlBytes) == 0 {\n\t\treturn nil, nil\n\t}\n\tvar parsed interface{}\n\terr := yaml.Unmarshal(yamlBytes, &parsed)\n\tif err != nil {\n\t\treturn nil, ErrorInvalidYAML(err)\n\t}\n\treturn parsed, nil\n}\n\nfunc ReadJSONBytes(jsonBytes []byte) (interface{}, error) {\n\tif len(jsonBytes) == 0 {\n\t\treturn nil, nil\n\t}\n\tvar parsed interface{}\n\terr := json.DecodeWithNumber(jsonBytes, &parsed)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn parsed, nil\n}\n\nfunc MustReadYAMLStr(yamlStr string) interface{} {\n\tparsed, err := ReadYAMLBytes([]byte(yamlStr))\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn parsed\n}\n\nfunc MustReadYAMLStrMap(yamlStr string) map[string]interface{} {\n\tparsed, err := ReadYAMLBytes([]byte(yamlStr))\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\tcasted, ok := cast.InterfaceToStrInterfaceMap(parsed)\n\tif !ok {\n\t\texit.Panic(ErrorInvalidPrimitiveType(parsed, PrimTypeMap))\n\t}\n\treturn casted\n}\n\nfunc MustReadJSONStr(jsonStr string) interface{} {\n\tparsed, err := ReadJSONBytes([]byte(jsonStr))\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn parsed\n}\n\n//\n// Helpers\n//\n\nfunc appendVal(slice interface{}, val interface{}) interface{} {\n\treturn reflect.Append(reflect.ValueOf(slice), reflect.ValueOf(val)).Interface()\n}\n\n// destStruct must be a pointer to a struct\nfunc setField(val interface{}, destStruct interface{}, fieldName string) error {\n\tv := reflect.ValueOf(destStruct).Elem().FieldByName(fieldName)\n\tif !v.IsValid() || !v.CanSet() {\n\t\tdebug.Ppg(val)\n\t\tdebug.Ppg(destStruct)\n\t\treturn errors.Wrap(ErrorCannotSetStructField(), fieldName)\n\t}\n\n\tif val == nil {\n\t\t// Check for nil-able types\n\t\tif v.Kind() == reflect.Chan || v.Kind() == reflect.Func || v.Kind() == reflect.Interface || v.Kind() == reflect.Map || v.Kind() == reflect.Ptr || v.Kind() == reflect.Slice {\n\t\t\tv.Set(reflect.Zero(v.Type()))\n\t\t\treturn nil\n\t\t}\n\t\tdebug.Ppg(val)\n\t\tdebug.Ppg(destStruct)\n\t\treturn errors.Wrap(ErrorCannotSetStructField(), fieldName)\n\t}\n\n\tif !reflect.ValueOf(val).Type().AssignableTo(v.Type()) {\n\t\tdebug.Ppg(val)\n\t\tdebug.Ppg(destStruct)\n\t\treturn errors.Wrap(ErrorCannotSetStructField(), fieldName)\n\t}\n\n\tv.Set(reflect.ValueOf(val))\n\treturn nil\n}\n\n// destStruct must be a pointer to a struct\nfunc setFirstField(val interface{}, destStruct interface{}) error {\n\tv := reflect.ValueOf(destStruct).Elem().FieldByIndex([]int{0})\n\tif !v.IsValid() || !v.CanSet() {\n\t\tdebug.Ppg(val)\n\t\tdebug.Ppg(destStruct)\n\t\treturn errors.Wrap(ErrorCannotSetStructField(), \"first field\")\n\t}\n\tv.Set(reflect.ValueOf(val))\n\treturn nil\n}\n\n// destStruct must be a pointer to a struct\nfunc setFieldNil(destStruct interface{}, fieldName string) error {\n\tv := reflect.ValueOf(destStruct).Elem().FieldByName(fieldName)\n\tif !v.IsValid() || !v.CanSet() {\n\t\tdebug.Ppg(destStruct)\n\t\treturn errors.Wrap(ErrorCannotSetStructField(), fieldName)\n\t}\n\tv.Set(reflect.Zero(v.Type()))\n\treturn nil\n}\n\n// destStruct must be a pointer to a struct\nfunc setFieldIfExists(val interface{}, destStruct interface{}, fieldName string) bool {\n\tif structHasKey(destStruct, fieldName) {\n\t\terr := setField(val, destStruct, fieldName)\n\t\treturn err == nil\n\t}\n\treturn false\n}\n\n// structVal must be a pointer to a struct\nfunc structHasKey(val interface{}, fieldName string) bool {\n\tv := reflect.ValueOf(val).Elem().FieldByName(fieldName)\n\tif v.IsValid() && v.CanSet() {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc inferKey(structType reflect.Type, typeStructField string, typeKey string) string {\n\tif typeKey != \"\" {\n\t\treturn typeKey\n\t}\n\tfield, _ := structType.Elem().FieldByName(typeStructField)\n\ttag, ok := getTagFieldName(field)\n\tif ok {\n\t\treturn tag\n\t}\n\treturn typeStructField\n}\n\nfunc inferPromptFieldName(structType reflect.Type, typeStructField string) string {\n\tfield, _ := structType.Elem().FieldByName(typeStructField)\n\ttag, ok := getTagFieldName(field)\n\tif ok {\n\t\treturn tag\n\t}\n\treturn typeStructField\n}\n\nfunc getTagFieldName(field reflect.StructField) (string, bool) {\n\ttag, ok := field.Tag.Lookup(\"json\")\n\tif ok {\n\t\treturn strings.Split(tag, \",\")[0], true\n\t}\n\ttag, ok = field.Tag.Lookup(\"yaml\")\n\tif ok {\n\t\treturn strings.Split(tag, \",\")[0], true\n\t}\n\treturn \"\", false\n}\n"
  },
  {
    "path": "pkg/lib/configreader/reader_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/stretchr/testify/require\"\n)\n\ntype SimpleConfig struct {\n\tKey1 bool `json:\"key1,omitempty\"`\n\tKey2 bool `json:\"key2\"`\n}\n\nfunc TestSimple(t *testing.T) {\n\tstructValidation := &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\t// Key:         \"key1\",\n\t\t\t\tStructField: \"Key1\",\n\t\t\t\tBoolValidation: &BoolValidation{\n\t\t\t\t\tRequired: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\t// Key:         \"key2\",\n\t\t\t\tStructField: \"Key2\",\n\t\t\t\tBoolValidation: &BoolValidation{\n\t\t\t\t\tDefault: true,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tRequired:     true,\n\t\tShortCircuit: true,\n\t}\n\n\tconfigData := MustReadYAMLStr(\n\t\t`\n    key1: true\n    `)\n\n\texpected := &SimpleConfig{\n\t\tKey1: true,\n\t\tKey2: true,\n\t}\n\n\ttestConfig(structValidation, configData, expected, t)\n}\n\ntype NestedConfig struct {\n\tKey0 float64  `json:\"key0\"`\n\tKey1 *Nested1 `json:\"key1\"`\n\tKey2 *Nested2 `json:\"key2\"`\n}\ntype Nested1 struct {\n\tKey11 int32 `json:\"key11\"`\n}\ntype Nested2 struct {\n\tKey21 string   `json:\"key21\"`\n\tKey22 *Nested3 `json:\"key22\"`\n}\ntype Nested3 struct {\n\tKey31 int `json:\"key31\"`\n}\n\nfunc TestNested(t *testing.T) {\n\tstructValidation := &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\t// Key:               \"key0\",\n\t\t\t\tStructField:       \"Key0\",\n\t\t\t\tFloat64Validation: &Float64Validation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\t// Key:         \"key1\",\n\t\t\t\tStructField: \"Key1\",\n\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t// Key:             \"key11\",\n\t\t\t\t\t\t\tStructField:     \"Key11\",\n\t\t\t\t\t\t\tInt32Validation: &Int32Validation{},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tRequired:     true,\n\t\t\t\t\tShortCircuit: true,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\t// Key:         \"key2\",\n\t\t\t\tStructField: \"Key2\",\n\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t// Key:              \"key21\",\n\t\t\t\t\t\t\tStructField:      \"Key21\",\n\t\t\t\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t// Key:         \"key22\",\n\t\t\t\t\t\t\tStructField: \"Key22\",\n\t\t\t\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\t\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t// Key:           \"key31\",\n\t\t\t\t\t\t\t\t\t\tStructField:   \"Key31\",\n\t\t\t\t\t\t\t\t\t\tIntValidation: &IntValidation{},\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\tRequired:     true,\n\t\t\t\t\t\t\t\tShortCircuit: true,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tRequired:     true,\n\t\t\t\t\tShortCircuit: true,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tRequired:     true,\n\t\tShortCircuit: true,\n\t}\n\n\tconfigData := MustReadYAMLStr(\n\t\t`\n    key0: 1.1\n    key1:\n      key11: 2\n    key2:\n      key21: test\n      key22:\n        key31: 0\n    `)\n\n\texpected := &NestedConfig{\n\t\tKey0: 1.1,\n\t\tKey1: &Nested1{\n\t\t\tKey11: 2,\n\t\t},\n\t\tKey2: &Nested2{\n\t\t\tKey21: \"test\",\n\t\t\tKey22: &Nested3{\n\t\t\t\tKey31: 0,\n\t\t\t},\n\t\t},\n\t}\n\n\ttestConfig(structValidation, configData, expected, t)\n}\n\ntype NestedListConfig struct {\n\tKey0 float64      `json:\"key0\"`\n\tKey1 *NestedList1 `json:\"key1\"`\n}\ntype NestedList1 struct {\n\tKey11 []*NestedList2 `json:\"key11\"`\n}\ntype NestedList2 struct {\n\tKeyA string       `json:\"keyA\"`\n\tKeyB int          `json:\"keyB\"`\n\tKeyC []float64    `json:\"keyC\"`\n\tKeyD *NestedList3 `json:\"keyD\"`\n}\ntype NestedList3 struct {\n\tKeyX string `json:\"keyX\"`\n\tKeyY string `json:\"keyY\"`\n}\n\nfunc TestNestedList(t *testing.T) {\n\tstructValidation := &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\t// Key:               \"key0\",\n\t\t\t\tStructField:       \"Key0\",\n\t\t\t\tFloat64Validation: &Float64Validation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\t// Key:         \"key1\",\n\t\t\t\tStructField: \"Key1\",\n\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t// Key:         \"key11\",\n\t\t\t\t\t\t\tStructField: \"Key11\",\n\t\t\t\t\t\t\tStructListValidation: &StructListValidation{\n\t\t\t\t\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\t\t\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t// Key:              \"keyA\",\n\t\t\t\t\t\t\t\t\t\t\tStructField:      \"KeyA\",\n\t\t\t\t\t\t\t\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t// Key:           \"keyB\",\n\t\t\t\t\t\t\t\t\t\t\tStructField:   \"KeyB\",\n\t\t\t\t\t\t\t\t\t\t\tIntValidation: &IntValidation{},\n\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t// Key:                   \"keyC\",\n\t\t\t\t\t\t\t\t\t\t\tStructField:           \"KeyC\",\n\t\t\t\t\t\t\t\t\t\t\tFloat64ListValidation: &Float64ListValidation{},\n\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t// Key:         \"keyD\",\n\t\t\t\t\t\t\t\t\t\t\tStructField: \"KeyD\",\n\t\t\t\t\t\t\t\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\t\t\t\t\t\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t// Key:              \"keyX\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\tStructField:      \"KeyX\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t// Key:              \"keyY\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\tStructField:      \"KeyY\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tRequired:     true,\n\t\tShortCircuit: true,\n\t}\n\n\tconfigData := MustReadYAMLStr(\n\t\t`\n    key0: 1.1\n    key1:\n      key11:\n        - keyA: A\n          keyB: 0\n          keyC:\n            - 0.1\n            - 0.2\n          keyD:\n            keyX: test1\n            keyY: test2\n        - keyA: X\n          keyB: 1\n          keyC:\n            - 1.1\n            - 1.2\n            - 1.3\n          keyD:\n            keyX: test3\n            keyY: test4\n    `)\n\n\texpected := &NestedListConfig{\n\t\tKey0: 1.1,\n\t\tKey1: &NestedList1{\n\t\t\tKey11: []*NestedList2{\n\t\t\t\t{\n\t\t\t\t\tKeyA: \"A\",\n\t\t\t\t\tKeyB: 0,\n\t\t\t\t\tKeyC: []float64{float64(0.1), float64(0.2)},\n\t\t\t\t\tKeyD: &NestedList3{\n\t\t\t\t\t\tKeyX: \"test1\",\n\t\t\t\t\t\tKeyY: \"test2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tKeyA: \"X\",\n\t\t\t\t\tKeyB: 1,\n\t\t\t\t\tKeyC: []float64{float64(1.1), float64(1.2), float64(1.3)},\n\t\t\t\t\tKeyD: &NestedList3{\n\t\t\t\t\t\tKeyX: \"test3\",\n\t\t\t\t\t\tKeyY: \"test4\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\ttestConfig(structValidation, configData, expected, t)\n}\n\ntype Typed interface {\n\tGetType() string\n}\n\ntype Typed1 struct {\n\tKey0 string `json:\"key0\"`\n\tKey1 string `json:\"key1\"`\n}\n\ntype Typed1WithType struct {\n\tType string `json:\"type\"`\n\tKey0 string `json:\"key0\"`\n\tKey1 string `json:\"key1\"`\n}\n\nfunc (t *Typed1) GetType() string {\n\treturn \"type1\"\n}\n\nfunc (t *Typed1WithType) GetType() string {\n\treturn \"type1\"\n}\n\ntype Typed2 struct {\n\tKeyA int `json:\"keyA\"`\n\tKeyB int `json:\"keyB\"`\n}\n\ntype Typed2WithType struct {\n\tType string `json:\"type\"`\n\tKeyA int    `json:\"keyA\"`\n\tKeyB int    `json:\"keyB\"`\n}\n\nfunc (t *Typed2) GetType() string {\n\treturn \"type2\"\n}\n\nfunc (t *Typed2WithType) GetType() string {\n\treturn \"type2\"\n}\n\ntype TypedConfig struct {\n\tTyped `json:\"typed\"`\n}\n\nvar _interfaceStructValidation = &InterfaceStructValidation{\n\tTypeKey: \"type\",\n\tInterfaceStructTypes: map[string]*InterfaceStructType{\n\t\t\"type1\": {\n\t\t\tType: (*Typed1)(nil),\n\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\t// Key:              \"key0\",\n\t\t\t\t\tStructField:      \"Key0\",\n\t\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t// Key:              \"key1\",\n\t\t\t\t\tStructField:      \"Key1\",\n\t\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"type2\": {\n\t\t\tType: (*Typed2)(nil),\n\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\t// Key:           \"keyA\",\n\t\t\t\t\tStructField:   \"KeyA\",\n\t\t\t\t\tIntValidation: &IntValidation{},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t// Key:           \"keyB\",\n\t\t\t\t\tStructField:   \"KeyB\",\n\t\t\t\t\tIntValidation: &IntValidation{},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n}\n\nvar _interfaceStructValidationWithTypeKeyConfig = &InterfaceStructValidation{\n\tTypeKey:         \"type\",\n\tTypeStructField: \"Type\",\n\tInterfaceStructTypes: map[string]*InterfaceStructType{\n\t\t\"type1\": {\n\t\t\tType: (*Typed1WithType)(nil),\n\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\t// Key:              \"key0\",\n\t\t\t\t\tStructField:      \"Key0\",\n\t\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t// Key:              \"key1\",\n\t\t\t\t\tStructField:      \"Key1\",\n\t\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"type2\": {\n\t\t\tType: (*Typed2WithType)(nil),\n\t\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\t// Key:           \"keyA\",\n\t\t\t\t\tStructField:   \"KeyA\",\n\t\t\t\t\tIntValidation: &IntValidation{},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t// Key:           \"keyB\",\n\t\t\t\t\tStructField:   \"KeyB\",\n\t\t\t\t\tIntValidation: &IntValidation{},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n}\n\nfunc TestInterface(t *testing.T) {\n\tstructValidation := &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\t// Key:         \"typed\",\n\t\t\t\tStructField:               \"Typed\",\n\t\t\t\tInterfaceStructValidation: _interfaceStructValidation,\n\t\t\t},\n\t\t},\n\t}\n\n\tconfigDataType1 := MustReadYAMLStr(\n\t\t`\n    typed:\n      type: type1\n      key0: testA\n      key1: testB\n    `)\n\n\tconfigDataType2 := MustReadYAMLStr(\n\t\t`\n    typed:\n      type: type2\n      keyA: 0\n      keyB: 1\n    `)\n\n\texpectedType1 := &TypedConfig{\n\t\tTyped: &Typed1{\n\t\t\tKey0: \"testA\",\n\t\t\tKey1: \"testB\",\n\t\t},\n\t}\n\n\texpectedType2 := &TypedConfig{\n\t\tTyped: &Typed2{\n\t\t\tKeyA: 0,\n\t\t\tKeyB: 1,\n\t\t},\n\t}\n\n\ttestConfig(structValidation, configDataType1, expectedType1, t)\n\ttestConfig(structValidation, configDataType2, expectedType2, t)\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\t// Key:         \"typed\",\n\t\t\t\tStructField:               \"Typed\",\n\t\t\t\tInterfaceStructValidation: _interfaceStructValidationWithTypeKeyConfig,\n\t\t\t},\n\t\t},\n\t}\n\n\texpectedTypeWithTypeKey1 := &TypedConfig{\n\t\tTyped: &Typed1WithType{\n\t\t\tType: \"type1\",\n\t\t\tKey0: \"testA\",\n\t\t\tKey1: \"testB\",\n\t\t},\n\t}\n\n\texpectedTypeWithTypeKey2 := &TypedConfig{\n\t\tTyped: &Typed2WithType{\n\t\t\tType: \"type2\",\n\t\t\tKeyA: 0,\n\t\t\tKeyB: 1,\n\t\t},\n\t}\n\n\ttestConfig(structValidation, configDataType1, expectedTypeWithTypeKey1, t)\n\ttestConfig(structValidation, configDataType2, expectedTypeWithTypeKey2, t)\n}\n\ntype TypedListConfig struct {\n\tTypeds []Typed `json:\"typeds\"`\n}\n\nfunc TestInterfaceList(t *testing.T) {\n\tstructValidation := &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\t// Key:         \"typeds\",\n\t\t\t\tStructField: \"Typeds\",\n\t\t\t\tInterfaceStructListValidation: &InterfaceStructListValidation{\n\t\t\t\t\tInterfaceStructValidation: _interfaceStructValidation,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tconfigData := MustReadYAMLStr(\n\t\t`\n    typeds:\n      - type: type1\n        key0: testA\n        key1: testB\n\n      - type: type2\n        keyA: 0\n        keyB: 1\n\n      - type: type1\n        key0: test1\n        key1: test2\n\n      - type: type2\n        keyA: 0\n        keyB: -1\n    `)\n\n\texpected := &TypedListConfig{\n\t\tTypeds: []Typed{\n\t\t\t&Typed1{\n\t\t\t\tKey0: \"testA\",\n\t\t\t\tKey1: \"testB\",\n\t\t\t},\n\t\t\t&Typed2{\n\t\t\t\tKeyA: 0,\n\t\t\t\tKeyB: 1,\n\t\t\t},\n\t\t\t&Typed1{\n\t\t\t\tKey0: \"test1\",\n\t\t\t\tKey1: \"test2\",\n\t\t\t},\n\t\t\t&Typed2{\n\t\t\t\tKeyA: 0,\n\t\t\t\tKeyB: -1,\n\t\t\t},\n\t\t},\n\t}\n\n\ttestConfig(structValidation, configData, expected, t)\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\t// Key:         \"typeds\",\n\t\t\t\tStructField: \"Typeds\",\n\t\t\t\tInterfaceStructListValidation: &InterfaceStructListValidation{\n\t\t\t\t\tInterfaceStructValidation: _interfaceStructValidationWithTypeKeyConfig,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\texpected = &TypedListConfig{\n\t\tTypeds: []Typed{\n\t\t\t&Typed1WithType{\n\t\t\t\tType: \"type1\",\n\t\t\t\tKey0: \"testA\",\n\t\t\t\tKey1: \"testB\",\n\t\t\t},\n\t\t\t&Typed2WithType{\n\t\t\t\tType: \"type2\",\n\t\t\t\tKeyA: 0,\n\t\t\t\tKeyB: 1,\n\t\t\t},\n\t\t\t&Typed1WithType{\n\t\t\t\tType: \"type1\",\n\t\t\t\tKey0: \"test1\",\n\t\t\t\tKey1: \"test2\",\n\t\t\t},\n\t\t\t&Typed2WithType{\n\t\t\t\tType: \"type2\",\n\t\t\t\tKeyA: 0,\n\t\t\t\tKeyB: -1,\n\t\t\t},\n\t\t},\n\t}\n\n\ttestConfig(structValidation, configData, expected, t)\n}\n\ntype NullableConfig struct {\n\tKey1 *string     `json:\"key1\"`\n\tKey2 []string    `json:\"key2\"`\n\tKey3 interface{} `json:\"key3\"`\n}\n\ntype NullableParentConfig struct {\n\tKeyA *NullableConfig `json:\"key_a\"`\n}\n\nfunc TestDefaultNull(t *testing.T) {\n\tconfigDataEmpty := MustReadYAMLStr(``)\n\tconfigDataNull := MustReadYAMLStr(`Null`)\n\tconfigDataEmptyMap := MustReadYAMLStr(`{}`)\n\tconfigDataNullValues := MustReadYAMLStr(\n\t\t`\n     key1: null\n     key2: null\n     key3: null\n     `)\n\tconfigDataParentNullValues := MustReadYAMLStr(\n\t\t`\n     key_a:\n       key1: null\n       key2: null\n       key3: null\n     `)\n\tconfigDataParentNull := MustReadYAMLStr(\n\t\t`\n     key_a: null\n     `)\n\n\tstructFieldValidations := []*StructFieldValidation{\n\t\t{\n\t\t\tStructField: \"Key1\",\n\t\t\tStringPtrValidation: &StringPtrValidation{\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Key2\",\n\t\t\tStringListValidation: &StringListValidation{\n\t\t\t\tDefault:           []string{\"key2\"},\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Key3\",\n\t\t\tInterfaceValidation: &InterfaceValidation{\n\t\t\t\tDefault:           \"key3\",\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t},\n\t\t},\n\t}\n\n\t// AllowExplicitNull = true\n\n\tstructValidation := &StructValidation{\n\t\tStructFieldValidations: structFieldValidations,\n\t\tAllowExplicitNull:      true,\n\t}\n\n\tvar expected interface{}\n\n\texpected = &NullableConfig{}\n\ttestConfig(structValidation, configDataEmpty, expected, t)\n\ttestConfig(structValidation, configDataNull, expected, t)\n\n\texpected = &NullableConfig{\n\t\tKey1: nil,\n\t\tKey2: []string{\"key2\"},\n\t\tKey3: \"key3\",\n\t}\n\ttestConfig(structValidation, configDataEmptyMap, expected, t)\n\n\texpected = &NullableConfig{\n\t\tKey1: nil,\n\t\tKey2: nil,\n\t\tKey3: nil,\n\t}\n\ttestConfig(structValidation, configDataNullValues, expected, t)\n\n\t// AllowExplicitNull = false\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: structFieldValidations,\n\t\tAllowExplicitNull:      false,\n\t}\n\n\texpected = &NullableConfig{}\n\ttestConfigError(structValidation, configDataEmpty, expected, t)\n\ttestConfigError(structValidation, configDataNull, expected, t)\n\n\texpected = &NullableConfig{\n\t\tKey1: nil,\n\t\tKey2: []string{\"key2\"},\n\t\tKey3: \"key3\",\n\t}\n\ttestConfig(structValidation, configDataEmptyMap, expected, t)\n\n\texpected = &NullableConfig{\n\t\tKey1: nil,\n\t\tKey2: nil,\n\t\tKey3: nil,\n\t}\n\ttestConfig(structValidation, configDataNullValues, expected, t)\n\n\t// parent, AllowExplicitNull = true on both\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\tStructField: \"KeyA\",\n\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\tStructFieldValidations: structFieldValidations,\n\t\t\t\t\tAllowExplicitNull:      true,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tAllowExplicitNull: true,\n\t}\n\n\texpected = &NullableParentConfig{}\n\ttestConfig(structValidation, configDataEmpty, expected, t)\n\ttestConfig(structValidation, configDataNull, expected, t)\n\n\texpected = &NullableParentConfig{\n\t\tKeyA: &NullableConfig{\n\t\t\tKey1: nil,\n\t\t\tKey2: []string{\"key2\"},\n\t\t\tKey3: \"key3\",\n\t\t},\n\t}\n\ttestConfig(structValidation, configDataEmptyMap, expected, t)\n\n\texpected = &NullableParentConfig{}\n\ttestConfig(structValidation, configDataParentNull, expected, t)\n\n\texpected = &NullableParentConfig{\n\t\tKeyA: &NullableConfig{\n\t\t\tKey1: nil,\n\t\t\tKey2: nil,\n\t\t\tKey3: nil,\n\t\t},\n\t}\n\ttestConfig(structValidation, configDataParentNullValues, expected, t)\n\n\t// parent, AllowExplicitNull = false on both\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\tStructField: \"KeyA\",\n\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\tStructFieldValidations: structFieldValidations,\n\t\t\t\t\tAllowExplicitNull:      false,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tAllowExplicitNull: false,\n\t}\n\n\texpected = &NullableParentConfig{}\n\ttestConfigError(structValidation, configDataEmpty, expected, t)\n\ttestConfigError(structValidation, configDataNull, expected, t)\n\n\texpected = &NullableParentConfig{\n\t\tKeyA: &NullableConfig{\n\t\t\tKey1: nil,\n\t\t\tKey2: []string{\"key2\"},\n\t\t\tKey3: \"key3\",\n\t\t},\n\t}\n\ttestConfig(structValidation, configDataEmptyMap, expected, t)\n\n\ttestConfigError(structValidation, configDataParentNull, expected, t)\n\n\texpected = &NullableParentConfig{\n\t\tKeyA: &NullableConfig{\n\t\t\tKey1: nil,\n\t\t\tKey2: nil,\n\t\t\tKey3: nil,\n\t\t},\n\t}\n\ttestConfig(structValidation, configDataParentNullValues, expected, t)\n\n\t// parent, AllowExplicitNull = true on child, DefaultNil = true\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\tStructField: \"KeyA\",\n\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\tStructFieldValidations: structFieldValidations,\n\t\t\t\t\tAllowExplicitNull:      true,\n\t\t\t\t\tDefaultNil:             true,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tAllowExplicitNull: false,\n\t}\n\n\texpected = &NullableParentConfig{}\n\ttestConfigError(structValidation, configDataEmpty, expected, t)\n\ttestConfigError(structValidation, configDataNull, expected, t)\n\n\texpected = &NullableParentConfig{}\n\ttestConfig(structValidation, configDataEmptyMap, expected, t)\n\n\texpected = &NullableParentConfig{}\n\ttestConfig(structValidation, configDataParentNull, expected, t)\n\n\texpected = &NullableParentConfig{\n\t\tKeyA: &NullableConfig{\n\t\t\tKey1: nil,\n\t\t\tKey2: nil,\n\t\t\tKey3: nil,\n\t\t},\n\t}\n\ttestConfig(structValidation, configDataParentNullValues, expected, t)\n\n\t// parent, AllowExplicitNull = false on both, DefaultNil = true\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\tStructField: \"KeyA\",\n\t\t\t\tStructValidation: &StructValidation{\n\t\t\t\t\tStructFieldValidations: structFieldValidations,\n\t\t\t\t\tAllowExplicitNull:      false,\n\t\t\t\t\tDefaultNil:             true,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tAllowExplicitNull: false,\n\t}\n\n\texpected = &NullableParentConfig{}\n\ttestConfigError(structValidation, configDataEmpty, expected, t)\n\ttestConfigError(structValidation, configDataNull, expected, t)\n\n\texpected = &NullableParentConfig{}\n\ttestConfig(structValidation, configDataEmptyMap, expected, t)\n\n\texpected = &NullableParentConfig{}\n\ttestConfigError(structValidation, configDataParentNull, expected, t)\n\n\texpected = &NullableParentConfig{\n\t\tKeyA: &NullableConfig{\n\t\t\tKey1: nil,\n\t\t\tKey2: nil,\n\t\t\tKey3: nil,\n\t\t},\n\t}\n\ttestConfig(structValidation, configDataParentNullValues, expected, t)\n}\n\ntype DefaultConfig struct {\n\tKey1 bool   `json:\"key1\"`\n\tKey2 string `json:\"key2\"`\n\tKey3 string `json:\"key3\"`\n}\n\nfunc TestDefaultField(t *testing.T) {\n\tstructValidation := &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\tStructField:    \"Key1\",\n\t\t\t\tBoolValidation: &BoolValidation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\tStructField:      \"Key2\",\n\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\tStructField:      \"Key3\",\n\t\t\t\tDefaultField:     \"Key2\",\n\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t},\n\t\t},\n\t}\n\n\tconfigData := MustReadYAMLStr(\n\t\t`\n    key1: true\n    key2: \"key2\"\n    key3: \"key3\"\n    `)\n\texpected := &DefaultConfig{\n\t\tKey1: true,\n\t\tKey2: \"key2\",\n\t\tKey3: \"key3\",\n\t}\n\ttestConfig(structValidation, configData, expected, t)\n\n\tconfigData = MustReadYAMLStr(\n\t\t`\n    key1: true\n    key2: \"key2\"\n    `)\n\texpected = &DefaultConfig{\n\t\tKey1: true,\n\t\tKey2: \"key2\",\n\t\tKey3: \"key2\",\n\t}\n\ttestConfig(structValidation, configData, expected, t)\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\tStructField:    \"Key1\",\n\t\t\t\tBoolValidation: &BoolValidation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\tStructField:      \"Key2\",\n\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\tStructField:            \"Key3\",\n\t\t\t\tDefaultDependentFields: []string{\"Key2\"},\n\t\t\t\tDefaultDependentFieldsFunc: func(vals []interface{}) interface{} {\n\t\t\t\t\treturn vals[0].(string) + \".py\"\n\t\t\t\t},\n\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t},\n\t\t},\n\t}\n\n\tconfigData = MustReadYAMLStr(\n\t\t`\n    key1: true\n    key2: \"key2\"\n    `)\n\texpected = &DefaultConfig{\n\t\tKey1: true,\n\t\tKey2: \"key2\",\n\t\tKey3: \"key2.py\",\n\t}\n\ttestConfig(structValidation, configData, expected, t)\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\tStructField:    \"Key1\",\n\t\t\t\tBoolValidation: &BoolValidation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\tStructField:      \"Key2\",\n\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\tStructField:            \"Key3\",\n\t\t\t\tDefaultDependentFields: []string{\"Key1\"},\n\t\t\t\tDefaultDependentFieldsFunc: func(vals []interface{}) interface{} {\n\t\t\t\t\tif vals[0].(bool) {\n\t\t\t\t\t\treturn \"It was true\"\n\t\t\t\t\t}\n\t\t\t\t\treturn \"It was false\"\n\t\t\t\t},\n\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t},\n\t\t},\n\t}\n\n\tconfigData = MustReadYAMLStr(\n\t\t`\n    key1: true\n    key2: \"key2\"\n    `)\n\texpected = &DefaultConfig{\n\t\tKey1: true,\n\t\tKey2: \"key2\",\n\t\tKey3: \"It was true\",\n\t}\n\ttestConfig(structValidation, configData, expected, t)\n\n\tstructValidation = &StructValidation{\n\t\tStructFieldValidations: []*StructFieldValidation{\n\t\t\t{\n\t\t\t\tStructField:      \"Key2\",\n\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\tStructField:      \"Key3\",\n\t\t\t\tStringValidation: &StringValidation{},\n\t\t\t},\n\t\t\t{\n\t\t\t\tStructField:            \"Key1\",\n\t\t\t\tDefaultDependentFields: []string{\"Key2\"},\n\t\t\t\tDefaultDependentFieldsFunc: func(vals []interface{}) interface{} {\n\t\t\t\t\tif vals[0].(string) == \"key2\" {\n\t\t\t\t\t\treturn true\n\t\t\t\t\t}\n\t\t\t\t\treturn false\n\t\t\t\t},\n\t\t\t\tBoolValidation: &BoolValidation{},\n\t\t\t},\n\t\t},\n\t}\n\n\tconfigData = MustReadYAMLStr(\n\t\t`\n    key2: \"key2\"\n    key3: \"key3\"\n    `)\n\texpected = &DefaultConfig{\n\t\tKey1: true,\n\t\tKey2: \"key2\",\n\t\tKey3: \"key3\",\n\t}\n\ttestConfig(structValidation, configData, expected, t)\n\n\tconfigData = MustReadYAMLStr(\n\t\t`\n    key2: \"test\"\n    key3: \"key3\"\n    `)\n\texpected = &DefaultConfig{\n\t\tKey1: false,\n\t\tKey2: \"test\",\n\t\tKey3: \"key3\",\n\t}\n\ttestConfig(structValidation, configData, expected, t)\n}\n\nfunc testConfig(structValidation *StructValidation, configData interface{}, expected interface{}, t *testing.T) {\n\tconfig := reflect.New(reflect.TypeOf(expected).Elem()).Interface()\n\n\terrs := Struct(config, configData, structValidation)\n\n\tif errs != nil {\n\t\tfor _, err := range errs {\n\t\t\tfmt.Println(\"ERROR: \" + errors.Message(err))\n\t\t}\n\t}\n\trequire.Empty(t, errs)\n\n\trequire.Equal(t, expected, config)\n}\n\nfunc testConfigError(structValidation *StructValidation, configData interface{}, expectedTypeInstance interface{}, t *testing.T) {\n\tconfig := reflect.New(reflect.TypeOf(expectedTypeInstance).Elem()).Interface()\n\terrs := Struct(config, configData, structValidation)\n\trequire.NotEmpty(t, errs)\n}\n"
  },
  {
    "path": "pkg/lib/configreader/string.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/regex\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n)\n\ntype StringValidation struct {\n\tRequired                             bool\n\tDefault                              string\n\tAllowEmpty                           bool // Allow `<field>: \"\"`\n\tTreatNullAsEmpty                     bool // `<field>: ` and `<field>: null` will be read as `<field>: \"\"`\n\tAllowedValues                        []string\n\tHiddenAllowedValues                  []string // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues                     []string\n\tCantBeSpecifiedErrStr                *string\n\tPrefix                               string\n\tSuffix                               string\n\tInvalidPrefixes                      []string\n\tInvalidSuffixes                      []string\n\tAllowedPrefixes                      []string\n\tAllowedSuffixes                      []string\n\tMaxLength                            int\n\tMinLength                            int\n\tDisallowLeadingWhitespace            bool\n\tDisallowTrailingWhitespace           bool\n\tAlphaNumericDashDotUnderscoreOrEmpty bool\n\tAlphaNumericDashDotUnderscore        bool\n\tAlphaNumericDashUnderscoreOrEmpty    bool\n\tAlphaNumericDashUnderscore           bool\n\tAlphaNumericDotUnderscore            bool\n\tAlphaNumericDash                     bool\n\tAWSTag                               bool\n\tDNS1035                              bool\n\tDNS1123                              bool\n\tCastInt                              bool\n\tCastNumeric                          bool\n\tCastScalar                           bool\n\tAllowCortexResources                 bool\n\tRequireCortexResources               bool\n\tDockerImage                          bool\n\tDockerImageOrEmpty                   bool\n\tValidator                            func(string) (string, error)\n}\n\nfunc EnvVar(envVarName string) string {\n\treturn fmt.Sprintf(\"environment variable \\\"%s\\\"\", envVarName)\n}\n\nfunc String(inter interface{}, v *StringValidation) (string, error) {\n\tif inter == nil {\n\t\tif v.TreatNullAsEmpty {\n\t\t\treturn ValidateStringProvided(\"\", v)\n\t\t}\n\t\treturn \"\", ErrorCannotBeNull(v.Required)\n\t}\n\tcasted, castOk := inter.(string)\n\tif !castOk {\n\t\tif v.CastScalar {\n\t\t\tif !cast.IsScalarType(inter) {\n\t\t\t\treturn \"\", ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt, PrimTypeFloat, PrimTypeBool)\n\t\t\t}\n\t\t\tcasted = s.ObjFlatNoQuotes(inter)\n\t\t} else if v.CastNumeric {\n\t\t\tif !cast.IsNumericType(inter) {\n\t\t\t\treturn \"\", ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt, PrimTypeFloat)\n\t\t\t}\n\t\t\tcasted = s.ObjFlatNoQuotes(inter)\n\t\t} else if v.CastInt {\n\t\t\tif !cast.IsIntType(inter) {\n\t\t\t\treturn \"\", ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt)\n\t\t\t}\n\t\t\tcasted = s.ObjFlatNoQuotes(inter)\n\t\t} else {\n\t\t\treturn \"\", ErrorInvalidPrimitiveType(inter, PrimTypeString)\n\t\t}\n\t}\n\treturn ValidateStringProvided(casted, v)\n}\n\nfunc StringFromInterfaceMap(key string, iMap map[string]interface{}, v *StringValidation) (string, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateStringMissing(v)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := String(inter, v)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc StringFromStrMap(key string, sMap map[string]string, v *StringValidation) (string, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok {\n\t\tval, err := ValidateStringMissing(v)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := StringFromStr(valStr, v)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc StringFromStr(valStr string, v *StringValidation) (string, error) {\n\treturn ValidateStringProvided(valStr, v)\n}\n\nfunc StringFromEnv(envVarName string, v *StringValidation) (string, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil {\n\t\tval, err := ValidateStringMissing(v)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := StringFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc StringFromFile(filePath string, v *StringValidation) (string, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateStringMissing(v)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateStringMissing(v)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := StringFromStr(valStr, v)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc StringFromEnvOrFile(envVarName string, filePath string, v *StringValidation) (string, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil {\n\t\treturn StringFromEnv(envVarName, v)\n\t}\n\treturn StringFromFile(filePath, v)\n}\n\nfunc StringFromPrompt(promptOpts *prompt.Options, v *StringValidation) (string, error) {\n\tpromptOpts.DefaultStr = v.Default\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" { // Treat empty prompt value as missing\n\t\treturn ValidateStringMissing(v)\n\t}\n\treturn StringFromStr(valStr, v)\n}\n\nfunc ValidateStringMissing(v *StringValidation) (string, error) {\n\tif v.Required {\n\t\treturn \"\", ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateString(v.Default, v)\n}\n\nfunc ValidateStringProvided(val string, v *StringValidation) (string, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn \"\", ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\treturn validateString(val, v)\n}\n\nfunc validateString(val string, v *StringValidation) (string, error) {\n\terr := ValidateStringVal(val, v)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateStringVal(val string, v *StringValidation) error {\n\tif v.RequireCortexResources {\n\t\tif err := checkOnlyCortexResources(val); err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if !v.AllowCortexResources {\n\t\tif err := checkNoCortexResources(val); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif !v.AllowEmpty {\n\t\tif len(val) == 0 {\n\t\t\treturn ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif len(v.AllowedValues) > 0 {\n\t\tif !slices.HasString(append(v.AllowedValues, v.HiddenAllowedValues...), val) {\n\t\t\treturn ErrorInvalidStr(val, v.AllowedValues[0], v.AllowedValues[1:]...)\n\t\t}\n\t}\n\n\tif len(v.DisallowedValues) > 0 {\n\t\tif slices.HasString(v.DisallowedValues, val) {\n\t\t\treturn ErrorDisallowedValue(val)\n\t\t}\n\t}\n\n\tif v.MaxLength > 0 && len(val) > v.MaxLength {\n\t\treturn ErrorTooLong(val, v.MaxLength)\n\t}\n\n\tif v.MinLength > 0 && len(val) < v.MinLength {\n\t\treturn ErrorTooShort(val, v.MinLength)\n\t}\n\n\tif v.Prefix != \"\" {\n\t\tif !strings.HasPrefix(val, v.Prefix) {\n\t\t\treturn ErrorMustHavePrefix(val, v.Prefix)\n\t\t}\n\t}\n\n\tif v.Suffix != \"\" {\n\t\tif !strings.HasSuffix(val, v.Suffix) {\n\t\t\treturn ErrorMustHaveSuffix(val, v.Suffix)\n\t\t}\n\t}\n\n\tfor _, invalidPrefix := range v.InvalidPrefixes {\n\t\tif strings.HasPrefix(val, invalidPrefix) {\n\t\t\treturn ErrorCantHavePrefix(val, invalidPrefix)\n\t\t}\n\t}\n\n\tfor _, invalidSuffix := range v.InvalidSuffixes {\n\t\tif strings.HasSuffix(val, invalidSuffix) {\n\t\t\treturn ErrorCantHaveSuffix(val, invalidSuffix)\n\t\t}\n\t}\n\n\tif len(v.AllowedPrefixes) > 0 {\n\t\tmatchedPrefixes := 0\n\t\tfor _, allowedPrefix := range v.AllowedPrefixes {\n\t\t\tif strings.HasPrefix(val, allowedPrefix) {\n\t\t\t\tmatchedPrefixes++\n\t\t\t}\n\t\t}\n\t\tif matchedPrefixes == 0 {\n\t\t\treturn ErrorMustHavePrefix(val, v.AllowedPrefixes[0], v.AllowedPrefixes[1:]...)\n\t\t}\n\t}\n\n\tif len(v.AllowedSuffixes) > 0 {\n\t\tmatchedSuffixes := 0\n\t\tfor _, allowedSuffix := range v.AllowedSuffixes {\n\t\t\tif strings.HasSuffix(val, allowedSuffix) {\n\t\t\t\tmatchedSuffixes++\n\t\t\t}\n\t\t}\n\t\tif matchedSuffixes == 0 {\n\t\t\treturn ErrorMustHaveSuffix(val, v.AllowedSuffixes[0], v.AllowedSuffixes[1:]...)\n\t\t}\n\t}\n\n\tif v.DisallowLeadingWhitespace {\n\t\tif regex.HasLeadingWhitespace(val) {\n\t\t\treturn ErrorLeadingWhitespace(val)\n\t\t}\n\t}\n\n\tif v.DisallowTrailingWhitespace {\n\t\tif regex.HasTrailingWhitespace(val) {\n\t\t\treturn ErrorTrailingWhitespace(val)\n\t\t}\n\t}\n\n\tif v.AlphaNumericDashDotUnderscore {\n\t\tif !regex.IsAlphaNumericDashDotUnderscore(val) {\n\t\t\treturn ErrorAlphaNumericDashDotUnderscore(val)\n\t\t}\n\t}\n\n\tif v.AlphaNumericDashUnderscore {\n\t\tif !regex.IsAlphaNumericDashUnderscore(val) {\n\t\t\treturn ErrorAlphaNumericDashUnderscore(val)\n\t\t}\n\t}\n\n\tif v.AlphaNumericDotUnderscore {\n\t\tif !regex.IsAlphaNumericDotUnderscore(val) {\n\t\t\treturn ErrorAlphaNumericDotUnderscore(val)\n\t\t}\n\t}\n\n\tif v.AlphaNumericDash {\n\t\tif !regex.IsAlphaNumericDash(val) {\n\t\t\treturn ErrorAlphaNumericDash(val)\n\t\t}\n\t}\n\n\tif v.AlphaNumericDashUnderscoreOrEmpty {\n\t\tif !regex.IsAlphaNumericDashUnderscore(val) && val != \"\" {\n\t\t\treturn ErrorAlphaNumericDashUnderscore(val)\n\t\t}\n\t}\n\n\tif v.AlphaNumericDashDotUnderscoreOrEmpty {\n\t\tif !regex.IsAlphaNumericDashDotUnderscore(val) && val != \"\" {\n\t\t\treturn ErrorAlphaNumericDashDotUnderscore(val)\n\t\t}\n\t}\n\n\tif v.AWSTag {\n\t\tif !regex.IsValidAWSTag(val) && val != \"\" {\n\t\t\treturn ErrorInvalidAWSTag(val)\n\t\t}\n\t}\n\n\tif v.DockerImage {\n\t\tif !regex.IsValidDockerImage(val) {\n\t\t\treturn ErrorInvalidDockerImage(val)\n\t\t}\n\t}\n\n\tif v.DockerImageOrEmpty {\n\t\tif !regex.IsValidDockerImage(val) && val != \"\" {\n\t\t\treturn ErrorInvalidDockerImage(val)\n\t\t}\n\t}\n\n\tif v.DNS1035 {\n\t\tif err := urls.CheckDNS1035(val); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif v.DNS1123 {\n\t\tif err := urls.CheckDNS1123(val); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// Musts\n//\n\nfunc MustStringFromEnv(envVarName string, v *StringValidation) string {\n\tval, err := StringFromEnv(envVarName, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustStringFromFile(filePath string, v *StringValidation) string {\n\tval, err := StringFromFile(filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n\nfunc MustStringFromEnvOrFile(envVarName string, filePath string, v *StringValidation) string {\n\tval, err := StringFromEnvOrFile(envVarName, filePath, v)\n\tif err != nil {\n\t\texit.Panic(err)\n\t}\n\treturn val\n}\n"
  },
  {
    "path": "pkg/lib/configreader/string_list.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype StringListValidation struct {\n\tRequired                bool\n\tDefault                 []string\n\tAllowExplicitNull       bool\n\tAllowEmpty              bool\n\tCantBeSpecifiedErrStr   *string\n\tCastSingleItem          bool\n\tDisallowDups            bool\n\tMinLength               int\n\tMaxLength               int\n\tInvalidLengths          []int\n\tAllowCortexResources    bool\n\tRequireCortexResources  bool\n\tElementStringValidation *StringValidation // Required, Default, AllowEmpty, TreatNullAsEmpty & Validator fields not applicable here\n\tValidator               func([]string) ([]string, error)\n}\n\nfunc StringList(inter interface{}, v *StringListValidation) ([]string, error) {\n\tcasted, castOk := cast.InterfaceToStrSlice(inter)\n\tif !castOk {\n\t\tif v.CastSingleItem {\n\t\t\tcastedItem, castOk := inter.(string)\n\t\t\tif !castOk {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeStringList)\n\t\t\t}\n\t\t\tcasted = []string{castedItem}\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeStringList)\n\t\t}\n\t}\n\treturn ValidateStringListProvided(casted, v)\n}\n\nfunc StringListFromInterfaceMap(key string, iMap map[string]interface{}, v *StringListValidation) ([]string, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateStringListMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := StringList(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateStringListMissing(v *StringListValidation) ([]string, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateStringList(v.Default, v)\n}\n\nfunc ValidateStringListProvided(val []string, v *StringListValidation) ([]string, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateStringList(val, v)\n}\n\nfunc validateStringList(val []string, v *StringListValidation) ([]string, error) {\n\tif v.RequireCortexResources {\n\t\tif err := checkOnlyCortexResources(val); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if !v.AllowCortexResources {\n\t\tif err := checkNoCortexResources(val); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.DisallowDups {\n\t\tif dups := slices.FindDuplicateStrs(val); len(dups) > 0 {\n\t\t\treturn nil, ErrorDuplicatedValue(dups[0])\n\t\t}\n\t}\n\n\tif v.MinLength != 0 {\n\t\tif len(val) < v.MinLength {\n\t\t\treturn nil, ErrorTooFewElements(v.MinLength)\n\t\t}\n\t}\n\n\tif v.MaxLength != 0 {\n\t\tif len(val) > v.MaxLength {\n\t\t\treturn nil, ErrorTooManyElements(v.MaxLength)\n\t\t}\n\t}\n\n\tfor _, invalidLength := range v.InvalidLengths {\n\t\tif len(val) == invalidLength {\n\t\t\treturn nil, ErrorWrongNumberOfElements(v.InvalidLengths)\n\t\t}\n\t}\n\n\tif v.ElementStringValidation != nil {\n\t\tfor i, element := range val {\n\t\t\terr := ValidateStringVal(element, v.ElementStringValidation)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrap(err, s.Index(i))\n\t\t\t}\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/string_map.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype StringMapValidation struct {\n\tRequired              bool\n\tDefault               map[string]string\n\tAllowExplicitNull     bool\n\tAllowEmpty            bool\n\tConvertNullToEmpty    bool\n\tCantBeSpecifiedErrStr *string\n\tKeyStringValidator    *StringValidation\n\tValueStringValidator  *StringValidation\n\tValidator             func(map[string]string) (map[string]string, error)\n}\n\nfunc StringMap(inter interface{}, v *StringMapValidation) (map[string]string, error) {\n\tcasted, castOk := cast.InterfaceToStrStrMap(inter)\n\tif !castOk {\n\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeStringToStringMap)\n\t}\n\treturn ValidateStringMapProvided(casted, v)\n}\n\nfunc StringMapFromInterfaceMap(key string, iMap map[string]interface{}, v *StringMapValidation) (map[string]string, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateStringMapMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := StringMap(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc ValidateStringMapMissing(v *StringMapValidation) (map[string]string, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined()\n\t}\n\treturn validateStringMap(v.Default, v)\n}\n\nfunc ValidateStringMapProvided(val map[string]string, v *StringMapValidation) (map[string]string, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateStringMap(val, v)\n}\n\nfunc validateStringMap(val map[string]string, v *StringMapValidation) (map[string]string, error) {\n\tif !v.AllowEmpty {\n\t\tif val != nil && len(val) == 0 {\n\t\t\treturn nil, ErrorCannotBeEmpty()\n\t\t}\n\t}\n\n\tif v.KeyStringValidator != nil {\n\t\tfor mapKey := range val {\n\t\t\terr := ValidateStringVal(mapKey, v.KeyStringValidator)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\tif v.ValueStringValidator != nil {\n\t\tfor mapKey, mapVal := range val {\n\t\t\terr := ValidateStringVal(mapVal, v.ValueStringValidator)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.Wrap(err, mapKey)\n\t\t\t}\n\t\t}\n\t}\n\n\tif v.Validator != nil {\n\t\treturn v.Validator(val)\n\t}\n\n\tif val == nil && v.ConvertNullToEmpty {\n\t\tval = make(map[string]string)\n\t}\n\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/string_ptr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype StringPtrValidation struct {\n\tRequired                             bool\n\tDefault                              *string\n\tAllowExplicitNull                    bool\n\tAllowEmpty                           bool\n\tAllowedValues                        []string\n\tHiddenAllowedValues                  []string // allowed, but will not be listed as valid values (must be used in conjunction with AllowedValues)\n\tDisallowedValues                     []string\n\tCantBeSpecifiedErrStr                *string\n\tPrefix                               string\n\tSuffix                               string\n\tInvalidPrefixes                      []string\n\tInvalidSuffixes                      []string\n\tAllowedPrefixes                      []string\n\tAllowedSuffixes                      []string\n\tMaxLength                            int\n\tMinLength                            int\n\tDisallowLeadingWhitespace            bool\n\tDisallowTrailingWhitespace           bool\n\tAlphaNumericDashDotUnderscoreOrEmpty bool\n\tAlphaNumericDashDotUnderscore        bool\n\tAlphaNumericDashUnderscore           bool\n\tAlphaNumericDotUnderscore            bool\n\tAWSTag                               bool\n\tDNS1035                              bool\n\tDNS1123                              bool\n\tCastInt                              bool\n\tCastNumeric                          bool\n\tCastScalar                           bool\n\tAllowCortexResources                 bool\n\tRequireCortexResources               bool\n\tDockerImage                          bool\n\tDockerImageOrEmpty                   bool\n\tValidator                            func(string) (string, error)\n}\n\nfunc makeStringValValidation(v *StringPtrValidation) *StringValidation {\n\treturn &StringValidation{\n\t\tAllowEmpty:                           v.AllowEmpty,\n\t\tAllowedValues:                        v.AllowedValues,\n\t\tHiddenAllowedValues:                  v.HiddenAllowedValues,\n\t\tDisallowedValues:                     v.DisallowedValues,\n\t\tPrefix:                               v.Prefix,\n\t\tSuffix:                               v.Suffix,\n\t\tInvalidPrefixes:                      v.InvalidPrefixes,\n\t\tInvalidSuffixes:                      v.InvalidSuffixes,\n\t\tAllowedPrefixes:                      v.AllowedPrefixes,\n\t\tAllowedSuffixes:                      v.AllowedSuffixes,\n\t\tMaxLength:                            v.MaxLength,\n\t\tMinLength:                            v.MinLength,\n\t\tDisallowLeadingWhitespace:            v.DisallowLeadingWhitespace,\n\t\tDisallowTrailingWhitespace:           v.DisallowTrailingWhitespace,\n\t\tAlphaNumericDashDotUnderscoreOrEmpty: v.AlphaNumericDashDotUnderscoreOrEmpty,\n\t\tAlphaNumericDashDotUnderscore:        v.AlphaNumericDashDotUnderscore,\n\t\tAlphaNumericDashUnderscore:           v.AlphaNumericDashUnderscore,\n\t\tAlphaNumericDotUnderscore:            v.AlphaNumericDotUnderscore,\n\t\tAWSTag:                               v.AWSTag,\n\t\tDNS1035:                              v.DNS1035,\n\t\tDNS1123:                              v.DNS1123,\n\t\tCastInt:                              v.CastInt,\n\t\tCastNumeric:                          v.CastNumeric,\n\t\tCastScalar:                           v.CastScalar,\n\t\tAllowCortexResources:                 v.AllowCortexResources,\n\t\tRequireCortexResources:               v.RequireCortexResources,\n\t\tDockerImage:                          v.DockerImage,\n\t\tDockerImageOrEmpty:                   v.DockerImageOrEmpty,\n\t}\n}\n\nfunc StringPtr(inter interface{}, v *StringPtrValidation) (*string, error) {\n\tif inter == nil {\n\t\treturn ValidateStringPtrProvided(nil, v)\n\t}\n\tcasted, castOk := inter.(string)\n\tif !castOk {\n\t\tif v.CastScalar {\n\t\t\tif !cast.IsScalarType(inter) {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt, PrimTypeFloat, PrimTypeBool)\n\t\t\t}\n\t\t\tcasted = s.ObjFlatNoQuotes(inter)\n\t\t} else if v.CastNumeric {\n\t\t\tif !cast.IsNumericType(inter) {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt, PrimTypeFloat)\n\t\t\t}\n\t\t\tcasted = s.ObjFlatNoQuotes(inter)\n\t\t} else if v.CastInt {\n\t\t\tif !cast.IsIntType(inter) {\n\t\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeString, PrimTypeInt)\n\t\t\t}\n\t\t\tcasted = s.ObjFlatNoQuotes(inter)\n\t\t} else {\n\t\t\treturn nil, ErrorInvalidPrimitiveType(inter, PrimTypeString)\n\t\t}\n\t}\n\n\treturn ValidateStringPtrProvided(&casted, v)\n}\n\nfunc StringPtrFromInterfaceMap(key string, iMap map[string]interface{}, v *StringPtrValidation) (*string, error) {\n\tinter, ok := ReadInterfaceMapValue(key, iMap)\n\tif !ok {\n\t\tval, err := ValidateStringPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := StringPtr(inter, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc StringPtrFromStrMap(key string, sMap map[string]string, v *StringPtrValidation) (*string, error) {\n\tvalStr, ok := sMap[key]\n\tif !ok {\n\t\tval, err := ValidateStringPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, key)\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := StringPtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, key)\n\t}\n\treturn val, nil\n}\n\nfunc StringPtrFromStr(str string, v *StringPtrValidation) (*string, error) {\n\treturn ValidateStringPtrProvided(&str, v)\n}\n\nfunc StringPtrFromEnv(envVarName string, v *StringPtrValidation) (*string, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr == nil {\n\t\tval, err := ValidateStringPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t\t}\n\t\treturn val, nil\n\t}\n\tval, err := StringPtrFromStr(*valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, EnvVar(envVarName))\n\t}\n\treturn val, nil\n}\n\nfunc StringPtrFromFile(filePath string, v *StringPtrValidation) (*string, error) {\n\tif !files.IsFile(filePath) {\n\t\tval, err := ValidateStringPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tvalStr, err := files.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(valStr) == 0 {\n\t\tval, err := ValidateStringPtrMissing(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, filePath)\n\t\t}\n\t\treturn val, nil\n\t}\n\n\tval, err := StringPtrFromStr(valStr, v)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, filePath)\n\t}\n\treturn val, nil\n}\n\nfunc StringPtrFromEnvOrFile(envVarName string, filePath string, v *StringPtrValidation) (*string, error) {\n\tvalStr := ReadEnvVar(envVarName)\n\tif valStr != nil {\n\t\treturn StringPtrFromEnv(envVarName, v)\n\t}\n\treturn StringPtrFromFile(filePath, v)\n}\n\nfunc StringPtrFromPrompt(promptOpts *prompt.Options, v *StringPtrValidation) (*string, error) {\n\tif v.Default != nil && promptOpts.DefaultStr == \"\" {\n\t\tpromptOpts.DefaultStr = *v.Default\n\t}\n\tvalStr := prompt.Prompt(promptOpts)\n\tif valStr == \"\" { // Treat empty prompt value as missing\n\t\treturn ValidateStringPtrMissing(v)\n\t}\n\treturn StringPtrFromStr(valStr, v)\n}\n\nfunc ValidateStringPtrMissing(v *StringPtrValidation) (*string, error) {\n\tif v.Required {\n\t\treturn nil, ErrorMustBeDefined(v.AllowedValues)\n\t}\n\treturn validateStringPtr(v.Default, v)\n}\n\nfunc ValidateStringPtrProvided(val *string, v *StringPtrValidation) (*string, error) {\n\tif v.CantBeSpecifiedErrStr != nil {\n\t\treturn nil, ErrorFieldCantBeSpecified(*v.CantBeSpecifiedErrStr)\n\t}\n\n\tif !v.AllowExplicitNull && val == nil {\n\t\treturn nil, ErrorCannotBeNull(v.Required)\n\t}\n\treturn validateStringPtr(val, v)\n}\n\nfunc validateStringPtr(val *string, v *StringPtrValidation) (*string, error) {\n\tif val != nil {\n\t\terr := ValidateStringVal(*val, makeStringValValidation(v))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif val == nil {\n\t\treturn val, nil\n\t}\n\n\tif v.Validator != nil {\n\t\tvalidated, err := v.Validator(*val)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &validated, nil\n\t}\n\n\treturn val, nil\n}\n"
  },
  {
    "path": "pkg/lib/configreader/types.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\ntype TypePlaceholder struct {\n\tType string `json:\"type\"`\n}\n\ntype PrimitiveType string\ntype PrimitiveTypes []PrimitiveType\n\nvar (\n\tPrimTypeInt        PrimitiveType = \"integer\"\n\tPrimTypeIntList    PrimitiveType = \"integer list\"\n\tPrimTypeFloat      PrimitiveType = \"float\"\n\tPrimTypeFloatList  PrimitiveType = \"float list\"\n\tPrimTypeString     PrimitiveType = \"string\"\n\tPrimTypeStringList PrimitiveType = \"string list\"\n\tPrimTypeBool       PrimitiveType = \"boolean\"\n\tPrimTypeBoolList   PrimitiveType = \"boolean list\"\n\n\tPrimTypeMap     PrimitiveType = \"map\"\n\tPrimTypeMapList PrimitiveType = \"list of maps\"\n\tPrimTypeList    PrimitiveType = \"list\"\n\n\tPrimTypeStringToStringMap PrimitiveType = \"map of strings to strings\"\n)\n\nvar PrimTypeScalars = []PrimitiveType{PrimTypeInt, PrimTypeFloat, PrimTypeString, PrimTypeBool}\n\nfunc (ts PrimitiveTypes) StringList() []string {\n\tstrs := make([]string, len(ts))\n\tfor i, t := range ts {\n\t\tstrs[i] = string(t)\n\t}\n\treturn strs\n}\n"
  },
  {
    "path": "pkg/lib/configreader/validators.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage configreader\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/docker\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n)\n\nvar _emailRegex *regexp.Regexp\n\nfunc init() {\n\t_emailRegex = regexp.MustCompile(\"^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$\")\n}\n\nfunc GetFilePathValidator(baseDir string) func(string) (string, error) {\n\treturn func(val string) (string, error) {\n\t\tif err := files.CheckFile(val); err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\treturn val, nil\n\t}\n}\n\nfunc S3aPathValidator(val string) (string, error) {\n\tif !aws.IsValidS3aPath(val) {\n\t\treturn \"\", aws.ErrorInvalidS3aPath(val)\n\t}\n\treturn val, nil\n}\n\nfunc S3PathValidator(val string) (string, error) {\n\tif !aws.IsValidS3Path(val) {\n\t\treturn \"\", aws.ErrorInvalidS3Path(val)\n\t}\n\treturn val, nil\n}\n\nfunc EmailValidator(val string) (string, error) {\n\tif len(val) > 320 {\n\t\treturn \"\", ErrorEmailTooLong()\n\t}\n\n\tif !_emailRegex.MatchString(val) {\n\t\treturn \"\", ErrorEmailInvalid()\n\t}\n\n\treturn val, nil\n}\n\ntype DurationValidation struct {\n\tGreaterThan          *time.Duration\n\tGreaterThanOrEqualTo *time.Duration\n\tLessThan             *time.Duration\n\tLessThanOrEqualTo    *time.Duration\n\tMultipleOf           *time.Duration\n}\n\nfunc DurationParser(v *DurationValidation) func(string) (interface{}, error) {\n\treturn func(str string) (interface{}, error) {\n\t\td, err := time.ParseDuration(str)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif v == nil {\n\t\t\treturn d, nil\n\t\t}\n\n\t\tif v.GreaterThan != nil {\n\t\t\tif d <= *v.GreaterThan {\n\t\t\t\treturn nil, ErrorMustBeGreaterThan(str, *v.GreaterThan)\n\t\t\t}\n\t\t}\n\n\t\tif v.GreaterThanOrEqualTo != nil {\n\t\t\tif d < *v.GreaterThanOrEqualTo {\n\t\t\t\treturn nil, ErrorMustBeGreaterThanOrEqualTo(str, *v.GreaterThanOrEqualTo)\n\t\t\t}\n\t\t}\n\n\t\tif v.LessThan != nil {\n\t\t\tif d >= *v.LessThan {\n\t\t\t\treturn nil, ErrorMustBeLessThan(str, *v.LessThan)\n\t\t\t}\n\t\t}\n\n\t\tif v.LessThanOrEqualTo != nil {\n\t\t\tif d > *v.LessThanOrEqualTo {\n\t\t\t\treturn nil, ErrorMustBeLessThanOrEqualTo(str, *v.LessThanOrEqualTo)\n\t\t\t}\n\t\t}\n\n\t\tif v.MultipleOf != nil {\n\t\t\tif d.Nanoseconds()%(*v.MultipleOf).Nanoseconds() != 0 {\n\t\t\t\treturn nil, ErrorIsNotMultiple(d, *v.MultipleOf)\n\t\t\t}\n\t\t}\n\n\t\treturn d, nil\n\t}\n}\n\nfunc ValidateImageVersion(image, cortexVersion string) (string, error) {\n\tif !strings.HasPrefix(image, \"quay.io/cortexlabs/\") && !strings.HasPrefix(image, \"quay.io/cortexlabsdev/\") && !strings.HasPrefix(image, \"docker.io/cortexlabs/\") && !strings.HasPrefix(image, \"docker.io/cortexlabsdev/\") && !strings.HasPrefix(image, \"cortexlabs/\") && !strings.HasPrefix(image, \"cortexlabsdev/\") {\n\t\treturn image, nil\n\t}\n\n\ttag := docker.ExtractImageTag(image)\n\t// in docker, missing tag implies \"latest\"\n\tif tag == \"\" {\n\t\ttag = \"latest\"\n\t}\n\n\tif !strings.HasPrefix(tag, cortexVersion) {\n\t\treturn \"\", ErrorImageVersionMismatch(image, tag, cortexVersion)\n\t}\n\n\treturn image, nil\n}\n"
  },
  {
    "path": "pkg/lib/console/format.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage console\n\nimport (\n\t\"github.com/fatih/color\"\n)\n\nvar _bold = color.New(color.Bold).SprintFunc()\n\n// Bold returns a string formatted in bold\nfunc Bold(a ...interface{}) string {\n\treturn _bold(a...)\n}\n\n// BoolColor converts a boolean into a colored string (green: true, red: false)\nfunc BoolColor(b bool) string {\n\tif b {\n\t\treturn color.GreenString(\"%t\", b)\n\t}\n\treturn color.RedString(\"%t\", b)\n}\n"
  },
  {
    "path": "pkg/lib/cron/cron.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage cron\n\nimport (\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\ntype Cron struct {\n\tcronRun    chan struct{}\n\tcronCancel chan struct{}\n}\n\nfunc Run(f func() error, errHandler func(error), delay time.Duration) Cron {\n\tcronRun := make(chan struct{}, 1)\n\tcronCancel := make(chan struct{}, 1)\n\n\trunCron := func() {\n\t\tdefer Recoverer(errHandler)\n\t\terr := f()\n\t\tif err != nil && errHandler != nil {\n\t\t\terrHandler(err)\n\t\t}\n\t}\n\n\tgo func() {\n\t\ttimer := time.NewTimer(0)\n\t\tdefer timer.Stop()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-cronCancel:\n\t\t\t\treturn\n\t\t\tcase <-cronRun:\n\t\t\t\trunCron()\n\t\t\tcase <-timer.C:\n\t\t\t\trunCron()\n\t\t\t}\n\t\t\ttimer.Reset(delay)\n\t\t}\n\t}()\n\n\treturn Cron{\n\t\tcronRun:    cronRun,\n\t\tcronCancel: cronCancel,\n\t}\n}\n\nfunc (c *Cron) RunNow() {\n\tc.cronRun <- struct{}{}\n}\n\nfunc (c *Cron) Cancel() {\n\tc.cronCancel <- struct{}{}\n}\n\nfunc Recoverer(errHandler func(error)) {\n\tif errInterface := recover(); errInterface != nil {\n\t\terr := errors.CastRecoverError(errInterface)\n\t\terrors.PrintStacktrace(err)\n\t\tif errHandler != nil {\n\t\t\terrHandler(err)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/debug/debug.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage debug\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/yaml\"\n\t\"github.com/davecgh/go-spew/spew\"\n)\n\nfunc Pp(obj interface{}) {\n\tfmt.Println(s.Obj(obj))\n}\n\nfunc Ppg(obj interface{}) {\n\tfmt.Print(Sppg(obj))\n}\n\nfunc Sppg(obj interface{}) string {\n\tspew.Config.SortKeys = true\n\tspew.Config.SpewKeys = true\n\tspew.Config.Indent = \"  \"\n\tspew.Config.ContinueOnMethod = true\n\tspew.Config.DisablePointerAddresses = true\n\tspew.Config.DisableCapacities = true\n\treturn spew.Sdump(obj)\n}\n\nfunc Ppj(obj interface{}) {\n\tb, err := json.MarshalIndent(obj, \"\", \" \")\n\tif err != nil {\n\t\terrors.PrintError(err)\n\t}\n\tfmt.Println(string(b))\n}\n\nfunc Ppy(obj interface{}) {\n\tb, err := yaml.Marshal(obj)\n\tif err != nil {\n\t\terrors.PrintError(err)\n\t}\n\tfmt.Println(string(b))\n}\n"
  },
  {
    "path": "pkg/lib/docker/docker.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage docker\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/archive\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/cron\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/print\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\tdockertypes \"github.com/docker/docker/api/types\"\n\tdockerclient \"github.com/docker/docker/client\"\n\t\"github.com/docker/docker/pkg/jsonmessage\"\n\t\"github.com/docker/docker/pkg/term\"\n)\n\nvar NoAuth string\n\nvar _cachedClient *Client\n\nfunc init() {\n\tNoAuth, _ = EncodeAuthConfig(dockertypes.AuthConfig{})\n}\n\ntype Client struct {\n\t*dockerclient.Client\n\tInfo dockertypes.Info\n}\n\nfunc GetDockerClient() (*Client, error) {\n\tif _cachedClient != nil {\n\t\treturn _cachedClient, nil\n\t}\n\n\tbaseClient, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv)\n\tif err != nil {\n\t\treturn nil, WrapDockerError(err)\n\t}\n\n\tbaseClient.NegotiateAPIVersion(context.Background())\n\n\tinfo, err := baseClient.Info(context.Background())\n\tif err != nil {\n\t\treturn nil, WrapDockerError(err)\n\t}\n\n\t_cachedClient = &Client{\n\t\tClient: baseClient,\n\t\tInfo:   info,\n\t}\n\n\treturn _cachedClient, nil\n}\n\nfunc MustDockerClient() *Client {\n\tdockerClient, err := GetDockerClient()\n\tif err != nil {\n\t\texit.Error(err)\n\t}\n\n\treturn dockerClient\n}\n\nfunc AWSAuthConfig(awsClient *aws.Client) (string, error) {\n\tdockerClient, err := GetDockerClient()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tecrAuthConfig, err := awsClient.GetECRAuthConfig()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tauth := dockertypes.AuthConfig{\n\t\tUsername:      ecrAuthConfig.Username,\n\t\tPassword:      ecrAuthConfig.AccessToken,\n\t\tServerAddress: ecrAuthConfig.ProxyEndpoint,\n\t}\n\n\t_, err = dockerClient.RegistryLogin(context.Background(), auth)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tauthConfig, err := EncodeAuthConfig(auth)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn authConfig, nil\n}\n\nfunc WrapDockerError(err error) error {\n\tif dockerclient.IsErrConnectionFailed(err) {\n\t\treturn ErrorConnectToDockerDaemon()\n\t}\n\n\tif strings.Contains(strings.ToLower(err.Error()), \"permission denied\") {\n\t\treturn ErrorDockerPermissions(err)\n\t}\n\n\treturn errors.WithStack(err)\n}\n\ntype PullVerbosity int\n\nconst (\n\tNoPrint PullVerbosity = iota\n\tPrintDots\n\tPrintProgressBars\n)\n\nfunc PullImage(image string, encodedAuthConfig string, pullVerbosity PullVerbosity) (bool, error) {\n\tdockerClient, err := GetDockerClient()\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif err := CheckImageExistsLocally(dockerClient, image); err == nil {\n\t\treturn false, nil\n\t}\n\n\tpullOutput, err := dockerClient.ImagePull(context.Background(), image, dockertypes.ImagePullOptions{\n\t\tRegistryAuth: encodedAuthConfig,\n\t})\n\tif err != nil {\n\t\treturn false, WrapDockerError(err)\n\t}\n\tdefer pullOutput.Close()\n\n\tswitch pullVerbosity {\n\tcase PrintProgressBars:\n\t\ttermFd, isTerm := term.GetFdInfo(os.Stderr)\n\t\tjsonmessage.DisplayJSONMessagesStream(pullOutput, os.Stderr, termFd, isTerm, nil)\n\t\tfmt.Println()\n\tcase PrintDots:\n\t\tvar err error\n\t\tfmt.Printf(\"￮ downloading docker image %s \", image)\n\t\tdefer func() {\n\t\t\tif err == nil {\n\t\t\t\tfmt.Print(\" ✓\\n\")\n\t\t\t} else {\n\t\t\t\tfmt.Print(\" x\\n\")\n\t\t\t}\n\t\t}()\n\t\td := json.NewDecoder(pullOutput)\n\t\tvar result jsonmessage.JSONMessage\n\t\tdotCron := cron.Run(print.Dot, nil, 2*time.Second)\n\t\tdefer dotCron.Cancel()\n\t\tfor {\n\t\t\tif e := d.Decode(&result); e != nil {\n\t\t\t\tif e == io.EOF {\n\t\t\t\t\treturn true, nil\n\t\t\t\t}\n\t\t\t\terr = e\n\t\t\t\treturn false, err\n\t\t\t}\n\n\t\t\tif result.Error != nil {\n\t\t\t\terr = result.Error\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\tdefault:\n\t\t// wait until the pull has completed\n\t\tif _, err := ioutil.ReadAll(pullOutput); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\nfunc StreamDockerLogs(containerID string, containerIDs ...string) error {\n\tcontainerIDs = append([]string{containerID}, containerIDs...)\n\n\tdockerClient, err := GetDockerClient()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfns := make([]func() error, len(containerIDs))\n\tfor i, containerID := range containerIDs {\n\t\tfns[i] = StreamDockerLogsFn(containerID, dockerClient)\n\t}\n\n\terr = parallel.RunFirstErr(fns[0], fns[1:]...)\n\n\tif err != nil {\n\t\treturn WrapDockerError(err)\n\t}\n\n\treturn nil\n}\n\nfunc StreamDockerLogsFn(containerID string, dockerClient *Client) func() error {\n\treturn func() error {\n\t\t// Use ContainerLogs() so lines are only printed once they end in \\n\n\t\tlogsOutput, err := dockerClient.ContainerLogs(context.Background(), containerID, dockertypes.ContainerLogsOptions{\n\t\t\tShowStdout: true,\n\t\t\tShowStderr: true,\n\t\t\tFollow:     true,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn WrapDockerError(err)\n\t\t}\n\n\t\t_, err = io.Copy(os.Stdout, logsOutput)\n\t\tif err != nil && err != io.EOF {\n\t\t\treturn errors.WithStack(err)\n\t\t}\n\n\t\treturn nil\n\t}\n}\n\n// The provided input will be extracted into the container's containerPath directory\nfunc CopyToContainer(containerID string, input *archive.Input, containerPath string) error {\n\tif !strings.HasPrefix(containerPath, \"/\") {\n\t\treturn errors.ErrorUnexpected(\"containerPath must start with /\")\n\t}\n\n\tdockerClient, err := GetDockerClient()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// this is necessary to ensure that missing parent directories are created in the container\n\tinput.AddPrefix = filepath.Join(containerPath, input.AddPrefix)\n\n\tbuf := new(bytes.Buffer)\n\t_, err = archive.TarToWriter(input, buf)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = dockerClient.CopyToContainer(context.Background(), containerID, \"/\", buf, dockertypes.CopyToContainerOptions{\n\t\tAllowOverwriteDirWithFile: true,\n\t})\n\tif err != nil {\n\t\treturn WrapDockerError(err)\n\t}\n\n\treturn nil\n}\n\n// The file or directory name of containerPath will be preserved in localDir\n// For example, if the container has /aaa/zzz.txt,\n//   - CopyFromContainer(_, \"/aaa\", \"~/test\") will create \"~/test/aaa/zzz.txt\"\n//   - CopyFromContainer(_, \"/aaa/zzz.txt\", \"~/test\") will create \"~/test/zzz.txt\"\nfunc CopyFromContainer(containerID string, containerPath string, localDir string) error {\n\tif !strings.HasPrefix(containerPath, \"/\") {\n\t\treturn errors.ErrorUnexpected(\"containerPath must start with /\")\n\t}\n\n\tdockerClient, err := GetDockerClient()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treader, _, err := dockerClient.CopyFromContainer(context.Background(), containerID, containerPath)\n\tif err != nil {\n\t\treturn WrapDockerError(err)\n\t}\n\tdefer reader.Close()\n\n\t_, err = archive.UntarReaderToDir(reader, localDir)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc EncodeAuthConfig(authConfig dockertypes.AuthConfig) (string, error) {\n\tencoded, err := json.Marshal(authConfig)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to encode docker login credentials\")\n\t}\n\tregistryAuth := base64.URLEncoding.EncodeToString(encoded)\n\treturn registryAuth, nil\n}\n\nfunc CheckImageAccessible(dockerClient *Client, dockerImage, authConfig string) error {\n\tif _, err := dockerClient.DistributionInspect(context.Background(), dockerImage, authConfig); err != nil {\n\t\treturn ErrorImageInaccessible(dockerImage, err)\n\t}\n\treturn nil\n}\n\nfunc CheckImageExistsLocally(dockerClient *Client, dockerImage string) error {\n\timages, err := dockerClient.ImageList(context.Background(), dockertypes.ImageListOptions{})\n\tif err != nil {\n\t\treturn WrapDockerError(err)\n\t}\n\n\t// in docker, missing tag implies \"latest\"\n\tif ExtractImageTag(dockerImage) == \"\" {\n\t\tdockerImage = fmt.Sprintf(\"%s:latest\", dockerImage)\n\t}\n\n\tfor _, image := range images {\n\t\tif slices.HasString(image.RepoTags, dockerImage) {\n\t\t\treturn nil\n\t\t}\n\t}\n\n\treturn ErrorImageDoesntExistLocally(dockerImage)\n}\n\nfunc ExtractImageTag(dockerImage string) string {\n\tif colonIndex := strings.LastIndex(dockerImage, \":\"); colonIndex != -1 {\n\t\treturn dockerImage[colonIndex+1:]\n\t}\n\treturn \"\"\n}\n"
  },
  {
    "path": "pkg/lib/docker/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage docker\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrConnectToDockerDaemon   = \"docker.connect_to_docker_daemon\"\n\tErrDockerPermissions       = \"docker.docker_permissions\"\n\tErrImageDoesntExistLocally = \"docker.image_doesnt_exist_locally\"\n\tErrImageInaccessible       = \"docker.image_inaccessible\"\n)\n\nfunc ErrorConnectToDockerDaemon() error {\n\tinstallMsg := \"install it by following the instructions for your operating system: https://docs.docker.com/install\"\n\tif strings.HasPrefix(runtime.GOOS, \"darwin\") {\n\t\tinstallMsg = \"install it here: https://docs.docker.com/docker-for-mac/install\"\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrConnectToDockerDaemon,\n\t\tMessage: fmt.Sprintf(\"unable to connect to the Docker daemon\\n\\nplease confirm Docker is running, or if Docker is not installed, %s\", installMsg),\n\t})\n}\n\nfunc ErrorDockerPermissions(err error) error {\n\terrStr := errors.Message(err)\n\n\tvar groupAddStr string\n\tif strings.HasPrefix(runtime.GOOS, \"linux\") {\n\t\tgroupAddStr = \" (e.g. by running `sudo groupadd docker; sudo gpasswd -a $USER docker` and then restarting your terminal)\"\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDockerPermissions,\n\t\tMessage: errStr + \"\\n\\nyou can re-run this command with `sudo`, or grant your current user access to docker\" + groupAddStr,\n\t})\n}\n\nfunc ErrorImageDoesntExistLocally(image string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrImageDoesntExistLocally,\n\t\tMessage: fmt.Sprintf(\"%s does not exist locally; download it with `docker pull %s` (if your registry is private, run `docker login` first)\", image, image),\n\t})\n}\n\nfunc ErrorImageInaccessible(image string, cause error) error {\n\tmessage := fmt.Sprintf(\"%s is not accessible\", image)\n\n\tif cause != nil {\n\t\tmessage += \"\\n\" + errors.Message(cause) // add \\n because docker client errors are verbose but useful\n\t}\n\n\tif strings.Contains(cause.Error(), \"auth\") {\n\t\tmessage += fmt.Sprintf(\"\\n\\nif you would like to use a private docker registry, see https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor)\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrImageInaccessible,\n\t\tMessage: message,\n\t\tCause:   cause,\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/errors/error.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage errors\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"strings\"\n\n\tpkgerrors \"github.com/pkg/errors\"\n)\n\nconst ErrNotCortexError = \"error\"\n\ntype Error struct {\n\tKind        string\n\tMessage     string\n\tMetadata    interface{} // won't be printed\n\tNoTelemetry bool\n\tNoPrint     bool\n\tCause       error\n\tstack       *stack\n}\n\nfunc (cortexError *Error) Error() string {\n\treturn cortexError.Message\n}\n\nfunc (cortexError *Error) StackTrace() pkgerrors.StackTrace {\n\tstackTrace := make([]pkgerrors.Frame, len(*cortexError.stack))\n\tfor i := 0; i < len(stackTrace); i++ {\n\t\tstackTrace[i] = pkgerrors.Frame((*cortexError.stack)[i])\n\t}\n\treturn stackTrace\n}\n\nfunc WithStack(err error) error {\n\tif err == nil {\n\t\treturn nil\n\t}\n\n\tcortexError := getCortexError(err)\n\n\tif cortexError == nil {\n\t\tcortexError = &Error{\n\t\t\tKind:    ErrNotCortexError,\n\t\t\tMessage: strings.TrimSpace(err.Error()),\n\t\t\tCause:   err,\n\t\t}\n\t}\n\n\tif cortexError.stack == nil {\n\t\tcortexError.stack = callers()\n\t}\n\n\treturn cortexError\n}\n\nfunc Wrap(err error, strs ...string) error {\n\tif err == nil {\n\t\treturn nil\n\t}\n\n\tcortexError := WithStack(err).(*Error)\n\n\tstrs = removeEmptyStrs(strs)\n\tstrs = append(strs, cortexError.Message)\n\tcortexError.Message = strings.Join(strs, \": \")\n\n\treturn cortexError\n}\n\nfunc Wrapf(err error, template string, params ...string) error {\n\treturn Wrap(err, fmt.Sprintf(template, params))\n}\n\n// adds to the end of the error message (without adding any whitespace or punctuation)\nfunc Append(err error, str string) error {\n\tif err == nil {\n\t\treturn nil\n\t}\n\n\tcortexError := WithStack(err).(*Error)\n\tcortexError.Message = cortexError.Message + str\n\treturn cortexError\n}\n\nfunc getCortexError(err error) *Error {\n\tif cortexError, ok := err.(*Error); ok {\n\t\treturn cortexError\n\t}\n\treturn nil\n}\n\nfunc GetKind(err error) string {\n\tif cortexError, ok := err.(*Error); ok {\n\t\treturn cortexError.Kind\n\t}\n\treturn ErrNotCortexError\n}\n\nfunc GetMetadata(err error) interface{} {\n\tif cortexError, ok := err.(*Error); ok {\n\t\treturn cortexError.Metadata\n\t}\n\treturn nil\n}\n\nfunc IsNoTelemetry(err error) bool {\n\tif cortexError, ok := err.(*Error); ok {\n\t\treturn cortexError.NoTelemetry\n\t}\n\treturn false\n}\n\nfunc SetNoTelemetry(err error) error {\n\tcortexError := WithStack(err).(*Error)\n\tcortexError.NoTelemetry = true\n\treturn cortexError\n}\n\nfunc IsNoPrint(err error) bool {\n\tif cortexError, ok := err.(*Error); ok {\n\t\treturn cortexError.NoPrint\n\t}\n\treturn false\n}\n\nfunc SetNoPrint(err error) error {\n\tcortexError := WithStack(err).(*Error)\n\tcortexError.NoPrint = true\n\treturn cortexError\n}\n\n// Returns nil if no cause\nfunc Cause(err error) error {\n\tif cortexError, ok := err.(*Error); ok {\n\t\treturn cortexError.Cause\n\t}\n\treturn nil\n}\n\nfunc CauseOrSelf(err error) error {\n\tif cortexError, ok := err.(*Error); ok {\n\t\tcause := cortexError.Cause\n\t\tif cause != nil {\n\t\t\treturn cause\n\t\t}\n\t}\n\treturn err\n}\n\nfunc PrintStacktrace(err error) {\n\tfmt.Printf(\"%+v\\n\", err)\n}\n\nfunc (cortexError *Error) Format(s fmt.State, verb rune) {\n\tswitch verb {\n\tcase 'v':\n\t\tif s.Flag('+') {\n\t\t\tio.WriteString(s, cortexError.Message)\n\t\t\tcortexError.stack.Format(s, verb)\n\t\t\treturn\n\t\t}\n\t\tfallthrough\n\tcase 's':\n\t\tio.WriteString(s, cortexError.Message)\n\tcase 'q':\n\t\tfmt.Fprintf(s, \"%q\", cortexError.Message)\n\t}\n}\n\nfunc CastRecoverError(errInterface interface{}, strs ...string) error {\n\tvar err error\n\tvar ok bool\n\terr, ok = errInterface.(error)\n\tif !ok {\n\t\terr = &Error{\n\t\t\tKind:    ErrNotCortexError,\n\t\t\tMessage: fmt.Sprint(errInterface),\n\t\t}\n\t}\n\treturn Wrap(err, strs...)\n}\n\nfunc removeEmptyStrs(strs []string) []string {\n\tvar cleanStrs []string\n\tfor _, str := range strs {\n\t\tif str != \"\" {\n\t\t\tcleanStrs = append(cleanStrs, str)\n\t\t}\n\t}\n\treturn cleanStrs\n}\n"
  },
  {
    "path": "pkg/lib/errors/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage errors\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrUnexpected = \"errors.unexpected\"\n)\n\nfunc ErrorUnexpected(msgs ...interface{}) error {\n\tstrs := make([]string, len(msgs))\n\tfor i, msg := range msgs {\n\t\tstrs[i] = s.ObjFlatNoQuotes(msg)\n\t}\n\n\treturn WithStack(&Error{\n\t\tKind:    ErrUnexpected,\n\t\tMessage: strings.Join(strs, \": \"),\n\t})\n}\n\nfunc ListOfErrors(errKind string, shouldPrint bool, errors ...error) error {\n\tvar errorsContents string\n\tfor i, err := range errors {\n\t\tif err != nil {\n\t\t\terrorsContents += fmt.Sprintf(\"error #%d: %s\\n\", i+1, err.Error())\n\t\t}\n\t}\n\tif errorsContents == \"\" {\n\t\treturn nil\n\t}\n\treturn WithStack(&Error{\n\t\tKind:    errKind,\n\t\tMessage: errorsContents,\n\t\tNoPrint: !shouldPrint,\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/errors/message.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage errors\n\nimport (\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/print\"\n)\n\nfunc PrintError(err error, strs ...string) {\n\tprint.StderrPrintln(ErrorStr(err, strs...))\n\t// PrintStacktrace(err)\n}\n\nfunc PrintErrorForUser(err error, strs ...string) {\n\tprint.StderrBoldFirstLine(ErrorStr(err, strs...))\n\t// PrintStacktrace(err)\n}\n\nfunc ErrorStr(err error, strs ...string) string {\n\twrappedErr := Wrap(err, strs...)\n\treturn \"error: \" + strings.TrimSpace(Message(wrappedErr))\n}\n\nfunc Message(err error, strs ...string) string {\n\twrappedErr := Wrap(err, strs...)\n\terrStr := wrappedErr.Error()\n\treturn strings.TrimSpace(errStr)\n}\n\nfunc MessageFirstLine(err error, strs ...string) string {\n\twrappedErr := Wrap(err, strs...)\n\n\tvar errStr string\n\tif _, ok := CauseOrSelf(wrappedErr).(awserr.Error); ok {\n\t\terrStr = strings.Split(strings.TrimSpace(wrappedErr.Error()), \"\\n\")[0]\n\t} else {\n\t\terrStr = wrappedErr.Error()\n\t}\n\n\treturn strings.TrimSpace(errStr)\n}\n"
  },
  {
    "path": "pkg/lib/errors/multi.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage errors\n\nfunc AddError(errs []error, err error, strs ...string) ([]error, bool) {\n\tok := false\n\tif err != nil {\n\t\terrs = append(errs, Wrap(err, strs...))\n\t\tok = true\n\t}\n\treturn errs, ok\n}\n\nfunc AddErrors(errs []error, newErrs []error, strs ...string) ([]error, bool) {\n\tok := false\n\tfor _, err := range newErrs {\n\t\tif err != nil {\n\t\t\terrs = append(errs, Wrap(err, strs...))\n\t\t\tok = true\n\t\t}\n\t}\n\treturn errs, ok\n}\n\nfunc WrapAll(errs []error, strs ...string) []error {\n\tif !HasError(errs) {\n\t\treturn nil\n\t}\n\twrappedErrs := make([]error, len(errs))\n\tfor i, err := range errs {\n\t\twrappedErrs[i] = Wrap(err, strs...)\n\t}\n\treturn wrappedErrs\n}\n\nfunc HasError(errs []error) bool {\n\tfor _, err := range errs {\n\t\tif err != nil {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc AreAllErrors(errs []error) bool {\n\tfor _, err := range errs {\n\t\tif err == nil {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc FirstError(errs ...error) error {\n\tfor _, err := range errs {\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc MapHasError(errs map[string]error) bool {\n\tfor _, err := range errs {\n\t\tif err != nil {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc FirstErrorInMap(errs map[string]error) error {\n\tfor _, err := range errs {\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc FirstKeyInErrorMap(errs map[string]error) string {\n\tfor k, err := range errs {\n\t\tif err != nil {\n\t\t\treturn k\n\t\t}\n\t}\n\treturn \"\"\n}\n\nfunc NonNilErrorMapKeys(errs map[string]error) []string {\n\tvar keys []string\n\tfor k, err := range errs {\n\t\tif err != nil {\n\t\t\tkeys = append(keys, k)\n\t\t}\n\t}\n\treturn keys\n}\n"
  },
  {
    "path": "pkg/lib/errors/stack.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage errors\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n\n\tpkgerrors \"github.com/pkg/errors\"\n)\n\ntype stack []uintptr\n\nfunc callers() *stack {\n\tconst depth = 32\n\tvar pcs [depth]uintptr\n\tn := runtime.Callers(3, pcs[:])\n\tvar st stack = pcs[0:n]\n\treturn &st\n}\n\nfunc (s *stack) Format(st fmt.State, verb rune) {\n\tswitch verb {\n\tcase 'v':\n\t\tswitch {\n\t\tcase st.Flag('+'):\n\t\t\tfor _, pc := range *s {\n\t\t\t\tf := pkgerrors.Frame(pc)\n\t\t\t\tfmt.Fprintf(st, \"\\n%+v\", f)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/exit/exit.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage exit\n\nimport (\n\t\"os\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n)\n\nfunc Ok() {\n\ttelemetry.Close()\n\tos.Exit(0)\n}\n\nfunc Error(err error, wrapStrs ...string) {\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\tif err != nil && !errors.IsNoTelemetry(err) {\n\t\ttelemetry.Error(err)\n\t}\n\n\tif err != nil && !errors.IsNoPrint(err) {\n\t\terrors.PrintErrorForUser(err)\n\t}\n\n\ttelemetry.Close()\n\n\tos.Exit(1)\n}\n\nfunc Panic(err error, wrapStrs ...string) {\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\tif err != nil && !errors.IsNoTelemetry(err) {\n\t\ttelemetry.Error(err)\n\t}\n\n\ttelemetry.Close()\n\n\tpanic(err)\n}\n\nfunc RecoverAndExit(strs ...string) {\n\tif errInterface := recover(); errInterface != nil {\n\t\terr := errors.CastRecoverError(errInterface, strs...)\n\t\tPanic(err)\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/files/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage files\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrCreateDir                    = \"files.create_dir\"\n\tErrDeleteDir                    = \"files.delete_dir\"\n\tErrReadFormFile                 = \"files.read_form_file\"\n\tErrCreateFile                   = \"files.create_file\"\n\tErrReadDir                      = \"files.read_dir\"\n\tErrReadFile                     = \"files.read_file\"\n\tErrFileAlreadyExists            = \"files.file_already_exists\"\n\tErrInsufficientMemoryToReadFile = \"files.insufficient_memory_to_read_file\"\n\tErrFileSizeLimit                = \"files.file_size_limit\"\n\tErrProjectSizeLimit             = \"files.project_size_limit\"\n\tErrUnexpected                   = \"files.unexpected\"\n\tErrFileDoesNotExist             = \"files.file_does_not_exist\"\n\tErrDirDoesNotExist              = \"files.dir_does_not_exist\"\n\tErrNotAFile                     = \"files.not_a_file\"\n\tErrNotADir                      = \"files.not_a_dir\"\n)\n\nfunc ErrorCreateDir(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCreateDir,\n\t\tMessage: fmt.Sprintf(\"%s: unable to create directory\", path),\n\t})\n}\n\nfunc ErrorDeleteDir(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDeleteDir,\n\t\tMessage: fmt.Sprintf(\"%s: unable to delete directory\", path),\n\t})\n}\n\nfunc ErrorReadFormFile(fileName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrReadFormFile,\n\t\tMessage: fmt.Sprintf(\"unable to read request form file %s\", s.UserStr(fileName)),\n\t})\n}\n\nfunc ErrorCreateFile(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCreateFile,\n\t\tMessage: fmt.Sprintf(\"%s: unable to create file\", path),\n\t})\n}\n\nfunc ErrorReadDir(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrReadDir,\n\t\tMessage: fmt.Sprintf(\"%s: unable to read directory\", path),\n\t})\n}\n\nfunc ErrorReadFile(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrReadFile,\n\t\tMessage: fmt.Sprintf(\"%s: unable to read file\", path),\n\t})\n}\n\nfunc ErrorFileAlreadyExists(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFileAlreadyExists,\n\t\tMessage: fmt.Sprintf(\"%s: file already exists\", path),\n\t})\n}\n\nfunc ErrorInsufficientMemoryToReadFile(fileSizeBytes, availableMemBytes int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInsufficientMemoryToReadFile,\n\t\tMessage: fmt.Sprintf(\"unable to read file due to insufficient system memory; needs %s but is only allowed to use %s\", s.Int64ToBase2Byte(fileSizeBytes), s.Int64ToBase2Byte(availableMemBytes)),\n\t})\n}\n\nfunc ErrorFileSizeLimit(maxFileSizeBytes int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFileSizeLimit,\n\t\tMessage: fmt.Sprintf(\"file size cannot be greater than %s\", s.Int64ToBase2Byte(maxFileSizeBytes)),\n\t})\n}\n\nfunc ErrorProjectSizeLimit(maxProjectSizeBytes int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrProjectSizeLimit,\n\t\tMessage: fmt.Sprintf(\"project size cannot exceed %s\", s.Int64ToBase2Byte(maxProjectSizeBytes)),\n\t})\n}\n\nfunc ErrorUnexpected() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUnexpected,\n\t\tMessage: \"an unexpected error occurred\",\n\t})\n}\n\nfunc ErrorFileDoesNotExist(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFileDoesNotExist,\n\t\tMessage: fmt.Sprintf(\"%s: file does not exist\", path),\n\t})\n}\n\nfunc ErrorDirDoesNotExist(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDirDoesNotExist,\n\t\tMessage: fmt.Sprintf(\"%s: directory does not exist\", path),\n\t})\n}\n\nfunc ErrorNotAFile(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNotAFile,\n\t\tMessage: fmt.Sprintf(\"%s: no such file\", path),\n\t})\n}\n\nfunc ErrorNotADir(path string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNotADir,\n\t\tMessage: fmt.Sprintf(\"%s: not a directory path\", path),\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/files/files.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage files\n\nimport (\n\t\"bytes\"\n\t\"crypto/md5\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/prompt\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/denormal/go-gitignore\"\n\t\"github.com/mitchellh/go-homedir\"\n\t\"github.com/shirou/gopsutil/mem\"\n\t\"github.com/xlab/treeprint\"\n)\n\nvar (\n\t_homeDir string\n)\n\n// the returned file should be closed by the caller\nfunc Open(path string) (*os.File, error) {\n\tcleanPath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfile, err := os.Open(cleanPath)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorReadFile(path)))\n\t}\n\n\treturn file, nil\n}\n\n// the returned file should be closed by the caller\nfunc OpenFile(path string, flag int, perm os.FileMode) (*os.File, error) {\n\tcleanPath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfile, err := os.OpenFile(cleanPath, flag, perm)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorCreateFile(path)))\n\t}\n\n\treturn file, err\n}\n\n// the returned file should be closed by the caller\nfunc Create(path string) (*os.File, error) {\n\tcleanPath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfile, err := os.Create(cleanPath)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorCreateFile(path)))\n\t}\n\n\treturn file, nil\n}\n\nfunc ReadFile(path string) (string, error) {\n\tfileBytes, err := ReadFileBytes(path)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn string(fileBytes), nil\n}\n\nfunc ReadFileBytes(path string) ([]byte, error) {\n\treturn ReadFileBytesErrPath(path, path)\n}\n\nfunc ReadFileBytesErrPath(path string, errMsgPath string) ([]byte, error) {\n\tpath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := CheckFileErrPath(path, errMsgPath); err != nil {\n\t\treturn nil, err\n\t}\n\n\tfileBytes, err := ioutil.ReadFile(path)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorReadFile(errMsgPath)))\n\t}\n\n\treturn fileBytes, nil\n}\n\nfunc CreateFile(path string) error {\n\tcleanPath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfile, err := os.Create(cleanPath)\n\tif err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorCreateFile(path)))\n\t}\n\tdefer file.Close()\n\n\treturn nil\n}\n\nfunc WriteFileFromReader(reader io.Reader, path string) error {\n\tcleanPath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfile, err := os.Create(cleanPath)\n\tif err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorCreateFile(path)))\n\t}\n\tdefer file.Close()\n\n\t_, err = io.Copy(file, reader)\n\tif err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorCreateFile(path)))\n\t}\n\n\treturn nil\n}\n\nfunc WriteFile(data []byte, path string) error {\n\tcleanPath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif err := ioutil.WriteFile(cleanPath, data, 0664); err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorCreateFile(path)))\n\t}\n\n\treturn nil\n}\n\nfunc IsAbsOrTildePrefixed(path string) bool {\n\treturn strings.HasPrefix(path, \"/\") || strings.HasPrefix(path, \"~/\")\n}\n\n// e.g. ~/path -> /home/ubuntu/path\n// returns original path if there was an error\nfunc EscapeTilde(path string) (string, error) {\n\tif !(path == \"~\" || strings.HasPrefix(path, \"~/\")) {\n\t\treturn path, nil\n\t}\n\n\tif _homeDir == \"\" {\n\t\thomeDir, err := homedir.Dir()\n\t\tif err != nil {\n\t\t\treturn path, err\n\t\t}\n\t\tif homeDir == \"\" || homeDir == \"/\" {\n\t\t\treturn path, nil\n\t\t}\n\t\t_homeDir = homeDir\n\t}\n\n\tif path == \"~\" {\n\t\treturn _homeDir, nil\n\t}\n\n\t// path starts with \"~/\"\n\treturn filepath.Join(_homeDir, path[2:]), nil\n}\n\n// e.g. ~/path/../path2 -> /home/ubuntu/path2\n// returns without escaping tilde if there was an error\nfunc Clean(path string) (string, error) {\n\tpath, err := EscapeTilde(path)\n\tpath = filepath.Clean(path)\n\tif err != nil {\n\t\treturn path, err\n\t}\n\treturn path, nil\n}\n\n// e.g. /home/ubuntu/path -> ~/path\nfunc ReplacePathWithTilde(absPath string) string {\n\tif !strings.HasPrefix(absPath, \"/\") {\n\t\treturn absPath\n\t}\n\n\tif _homeDir == \"\" {\n\t\thomeDir, err := homedir.Dir()\n\t\tif err != nil || homeDir == \"\" || homeDir == \"/\" {\n\t\t\treturn absPath\n\t\t}\n\t\t_homeDir = homeDir\n\t}\n\n\ttrimmedHomeDir := strings.TrimSuffix(s.EnsurePrefix(_homeDir, \"/\"), \"/\")\n\n\tif strings.Index(absPath, trimmedHomeDir) == 0 {\n\t\treturn \"~\" + absPath[len(trimmedHomeDir):]\n\t}\n\n\treturn absPath\n}\n\n// e.g. /home/ubuntu/path -> /\n// or e.g. home/ubuntu/path -> home\nfunc GetTopLevelDirectory(path string) string {\n\tif strings.HasPrefix(path, \"/\") {\n\t\treturn \"/\"\n\t}\n\n\tif path == \"\" {\n\t\treturn \".\"\n\t}\n\n\tsplitList := strings.Split(path, \"/\")\n\tif len(splitList) == 1 {\n\t\treturn \".\"\n\t}\n\n\treturn splitList[0]\n}\n\nfunc TrimDirPrefix(fullPath string, dirPath string) string {\n\tif !strings.HasSuffix(dirPath, \"/\") {\n\t\tdirPath = dirPath + \"/\"\n\t}\n\treturn strings.TrimPrefix(fullPath, dirPath)\n}\n\nfunc RelToAbsPath(relativePath string, baseDir string) string {\n\tif !IsAbsOrTildePrefixed(relativePath) {\n\t\trelativePath = filepath.Join(baseDir, relativePath)\n\t}\n\tcleanPath, _ := Clean(relativePath)\n\treturn cleanPath\n}\n\nfunc UserRelToAbsPath(relativePath string) string {\n\tcwd, err := os.Getwd()\n\tif err != nil {\n\t\treturn relativePath\n\t}\n\treturn RelToAbsPath(relativePath, cwd)\n}\n\nfunc PathRelativeToCWD(absPath string) string {\n\tcwd, err := os.Getwd()\n\tif err != nil {\n\t\treturn absPath\n\t}\n\treturn PathRelativeToDir(absPath, cwd)\n}\n\nfunc PathRelativeToDir(absPath string, dir string) string {\n\tif !IsAbsOrTildePrefixed(absPath) {\n\t\treturn absPath\n\t}\n\tabsPath, _ = EscapeTilde(absPath)\n\tdir, _ = EscapeTilde(dir)\n\tdir = s.EnsureSuffix(dir, \"/\")\n\treturn strings.TrimPrefix(absPath, dir)\n}\n\nfunc DirPathRelativeToCWD(absPath string) string {\n\treturn s.EnsureSuffix(PathRelativeToCWD(absPath), \"/\")\n}\n\nfunc DirPathRelativeToDir(absPath string, dir string) string {\n\treturn s.EnsureSuffix(PathRelativeToDir(absPath, dir), \"/\")\n}\n\nfunc IsFileOrDir(path string) bool {\n\tpath, _ = EscapeTilde(path)\n\t_, err := os.Stat(path)\n\treturn err == nil\n}\n\nfunc IsDir(path string) bool {\n\tif err := CheckDir(path); err != nil {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// CheckDir returns nil if the path is a directory\nfunc CheckDir(dirPath string) error {\n\treturn CheckDirErrPath(dirPath, dirPath)\n}\n\n// CheckDir returns nil if the path is a directory\nfunc CheckDirErrPath(dirPath string, errMsgPath string) error {\n\tdirPath, err := EscapeTilde(dirPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfileInfo, err := os.Stat(dirPath)\n\tif err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorDirDoesNotExist(errMsgPath)))\n\t}\n\n\tif !fileInfo.IsDir() {\n\t\treturn ErrorNotADir(errMsgPath)\n\t}\n\n\treturn nil\n}\n\nfunc IsFile(path string) bool {\n\tif err := CheckFile(path); err != nil {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// CheckFile returns nil if the path is a file\nfunc CheckFile(path string) error {\n\treturn CheckFileErrPath(path, path)\n}\n\n// CheckFile returns nil if the path is a file\nfunc CheckFileErrPath(path string, errMsgPath string) error {\n\tpath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfileInfo, err := os.Stat(path)\n\tif err != nil {\n\t\treturn ErrorFileDoesNotExist(errMsgPath)\n\t}\n\tif fileInfo.IsDir() {\n\t\treturn ErrorNotAFile(errMsgPath)\n\t}\n\n\treturn nil\n}\n\nfunc CreateDir(path string) error {\n\tcleanPath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif err := os.MkdirAll(cleanPath, os.ModePerm); err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorCreateDir(path)))\n\t}\n\n\treturn nil\n}\n\nfunc CreateDirIfMissing(path string) (bool, error) {\n\tif IsDir(path) {\n\t\treturn false, nil\n\t}\n\n\tif IsFile(path) {\n\t\treturn false, ErrorFileAlreadyExists(path)\n\t}\n\n\terr := CreateDir(path)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn true, nil\n}\n\nfunc DeleteDir(path string) error {\n\tcleanPath, err := EscapeTilde(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif err := os.RemoveAll(cleanPath); err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorDeleteDir(path)))\n\t}\n\n\treturn nil\n}\n\nfunc DeleteDirIfPresent(path string) (bool, error) {\n\tif IsFile(path) {\n\t\treturn false, ErrorNotADir(path)\n\t}\n\n\tif !IsDir(path) {\n\t\treturn false, nil\n\t}\n\n\terr := DeleteDir(path)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn true, nil\n}\n\nfunc TmpDir() (string, error) {\n\tfilePath, err := ioutil.TempDir(\"\", \"\")\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err)\n\t}\n\treturn filePath, nil\n}\n\nfunc ParentDir(dir string) string {\n\treturn filepath.Clean(filepath.Join(dir, \"..\"))\n}\n\nfunc SearchForFile(filename string, dir string) (string, error) {\n\tdir, err := Clean(dir)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tfor true {\n\t\tfiles, err := ioutil.ReadDir(dir)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrap(err, errors.Message(ErrorReadDir(dir)))\n\t\t}\n\n\t\tfor _, file := range files {\n\t\t\tif file.Name() == filename {\n\t\t\t\treturn filepath.Join(dir, filename), nil\n\t\t\t}\n\t\t}\n\n\t\tif dir == \"/\" {\n\t\t\treturn \"\", nil\n\t\t}\n\n\t\tdir = ParentDir(dir)\n\t}\n\n\treturn \"\", ErrorUnexpected()\n}\n\nfunc MakeEmptyFile(path string) error {\n\tcleanPath, err := Clean(path)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = os.MkdirAll(filepath.Dir(cleanPath), os.ModePerm)\n\tif err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorCreateDir(filepath.Dir(path))))\n\t}\n\tf, err := os.OpenFile(cleanPath, os.O_RDONLY|os.O_CREATE, 0666)\n\tif err != nil {\n\t\treturn errors.Wrap(err, errors.Message(ErrorCreateFile(path)))\n\t}\n\tdefer f.Close()\n\treturn nil\n}\n\nfunc MakeEmptyFiles(path string, paths ...string) error {\n\tallPaths := append(paths, path)\n\tfor _, path := range allPaths {\n\t\tif err := MakeEmptyFile(path); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc MakeEmptyFilesInDir(dir string, path string, paths ...string) error {\n\tallPaths := append(paths, path)\n\tfor _, path := range allPaths {\n\t\tfullPath := filepath.Join(dir, path)\n\t\tif err := MakeEmptyFile(fullPath); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc IsFilePathYAML(path string) bool {\n\text := filepath.Ext(path)\n\treturn ext == \".yaml\" || ext == \".yml\"\n}\n\nfunc IsFilePathPython(path string) bool {\n\text := filepath.Ext(path)\n\treturn ext == \".py\"\n}\n\n// IgnoreFn if passed a dir, returning true will ignore all subdirs of dir\ntype IgnoreFn func(string, os.FileInfo) (bool, error)\n\nfunc IgnoreHiddenFiles(path string, fi os.FileInfo) (bool, error) {\n\tif !fi.IsDir() && strings.HasPrefix(fi.Name(), \".\") {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc IgnoreCortexYAML(path string, fi os.FileInfo) (bool, error) {\n\tif !fi.IsDir() && fi.Name() == \"cortex.yaml\" {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc IgnoreCortexDebug(path string, fi os.FileInfo) (bool, error) {\n\tif strings.HasPrefix(fi.Name(), \"cortex-debug-\") {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc IgnoreHiddenFolders(path string, fi os.FileInfo) (bool, error) {\n\tif fi.IsDir() && strings.HasPrefix(fi.Name(), \".\") {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc IgnorePythonGeneratedFiles(path string, fi os.FileInfo) (bool, error) {\n\tif fi.IsDir() && fi.Name() == \"__pycache__\" {\n\t\treturn true, nil\n\t}\n\tif !fi.IsDir() {\n\t\text := filepath.Ext(path)\n\t\treturn ext == \".pyc\" || ext == \".pyo\" || ext == \".pyd\", nil\n\t}\n\treturn false, nil\n}\n\nfunc IgnoreNonPython(path string, fi os.FileInfo) (bool, error) {\n\tif !fi.IsDir() && !IsFilePathPython(path) {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc IgnoreNonYAML(path string, fi os.FileInfo) (bool, error) {\n\tif !fi.IsDir() && !IsFilePathYAML(path) {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc IgnoreSpecificFiles(absPaths ...string) IgnoreFn {\n\tabsPathsSet := strset.New(absPaths...)\n\treturn func(path string, fi os.FileInfo) (bool, error) {\n\t\treturn absPathsSet.Has(path), nil\n\t}\n}\n\nfunc GitIgnoreFn(gitIgnorePath string) (IgnoreFn, error) {\n\tgitIgnoreDir := filepath.Dir(gitIgnorePath)\n\n\tignore, err := gitignore.NewFromFile(gitIgnorePath)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, gitIgnorePath)\n\t}\n\n\treturn func(path string, fi os.FileInfo) (bool, error) {\n\t\tif path == gitIgnoreDir {\n\t\t\t// This is to avoid a bug in ignore.Ignore()\n\t\t\treturn false, nil\n\t\t}\n\t\treturn ignore.Ignore(path), nil\n\t}, nil\n}\n\n// promptMsgTemplate should have two placeholders: the first is for the file path and the second is for the file size\nfunc PromptForFilesAboveSize(size int, promptMsgTemplate string) IgnoreFn {\n\tif promptMsgTemplate == \"\" {\n\t\tpromptMsgTemplate = \"do you want to zip %s (%s)?\"\n\t}\n\n\treturn func(path string, fi os.FileInfo) (bool, error) {\n\t\tif !fi.IsDir() && fi.Size() > int64(size) {\n\t\t\tpromptMsg := fmt.Sprintf(promptMsgTemplate, PathRelativeToCWD(path), s.Int64ToBase2Byte(fi.Size()))\n\t\t\treturn !prompt.YesOrNo(promptMsg, \"\", \"\"), nil\n\t\t}\n\t\treturn false, nil\n\t}\n}\n\nfunc ErrorOnBigFilesFn(maxFileSizeBytes int64) IgnoreFn {\n\treturn func(path string, fi os.FileInfo) (bool, error) {\n\t\tif !fi.IsDir() {\n\t\t\tfileSizeBytes := fi.Size()\n\t\t\tvirtual, _ := mem.VirtualMemory()\n\t\t\tif fileSizeBytes > int64(virtual.Available) {\n\t\t\t\treturn false, errors.Wrap(\n\t\t\t\t\tErrorInsufficientMemoryToReadFile(fileSizeBytes, int64(virtual.Available)),\n\t\t\t\t\tpath,\n\t\t\t\t)\n\t\t\t}\n\t\t\tif fileSizeBytes > maxFileSizeBytes {\n\t\t\t\treturn false, errors.Wrap(ErrorFileSizeLimit(maxFileSizeBytes), path)\n\t\t\t}\n\t\t}\n\n\t\treturn false, nil\n\t}\n}\n\nfunc ErrorOnProjectSizeLimit(maxProjectSizeBytes int64) IgnoreFn {\n\tfilesSizeSum := int64(0)\n\treturn func(path string, fi os.FileInfo) (bool, error) {\n\t\tif !fi.IsDir() {\n\t\t\tfilesSizeSum += fi.Size()\n\t\t\tif filesSizeSum > maxProjectSizeBytes {\n\t\t\t\treturn false, errors.Wrap(ErrorProjectSizeLimit(maxProjectSizeBytes), path)\n\t\t\t}\n\t\t}\n\t\treturn false, nil\n\t}\n}\n\n// Retrieves the longest common path given a list of paths.\nfunc LongestCommonPath(paths ...string) string {\n\n\t// Handle special cases.\n\tswitch len(paths) {\n\tcase 0:\n\t\treturn \"\"\n\tcase 1:\n\t\treturn path.Clean(paths[0])\n\t}\n\n\tstartsWithSlash := false\n\tallStartWithSlash := true\n\n\tvar splitPaths [][]string\n\tshortestPathLength := -1\n\tfor _, path := range paths {\n\t\tif strings.HasPrefix(path, \"/\") {\n\t\t\tstartsWithSlash = true\n\t\t} else {\n\t\t\tallStartWithSlash = false\n\t\t}\n\n\t\tsplitPath := slices.RemoveEmpties(strings.Split(path, \"/\"))\n\t\tsplitPaths = append(splitPaths, splitPath)\n\n\t\tif len(splitPath) < shortestPathLength || shortestPathLength == -1 {\n\t\t\tshortestPathLength = len(splitPath)\n\t\t}\n\t}\n\n\tcommonPath := \"\"\n\tnumPaths := len(splitPaths)\n\n\tfor level := 0; level < shortestPathLength; level++ {\n\t\telement := splitPaths[0][level]\n\t\tcounter := 1\n\t\tfor _, splitPath := range splitPaths[1:] {\n\t\t\tif splitPath[level] != element {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tcounter++\n\t\t}\n\n\t\tif counter != numPaths {\n\t\t\tbreak\n\t\t}\n\n\t\tcommonPath = filepath.Join(commonPath, element)\n\t}\n\tif commonPath != \"\" && startsWithSlash {\n\t\tcommonPath = s.EnsurePrefix(commonPath, \"/\")\n\t\tcommonPath = s.EnsureSuffix(commonPath, \"/\")\n\t}\n\tif commonPath == \"\" && allStartWithSlash {\n\t\treturn \"/\"\n\t}\n\n\treturn commonPath\n}\n\nfunc FilterPathsWithDirPrefix(paths []string, prefix string) []string {\n\tprefix = s.EnsureSuffix(prefix, \"/\")\n\n\tvar filteredPaths []string\n\tfor _, path := range paths {\n\t\tif strings.HasPrefix(path, prefix) {\n\t\t\tfilteredPaths = append(filteredPaths, path)\n\t\t}\n\t}\n\n\treturn filteredPaths\n}\n\ntype DirsOrder string\n\nvar DirsSorted DirsOrder = \"sorted\"\nvar DirsOnTop DirsOrder = \"top\"\nvar DirsOnBottom DirsOrder = \"bottom\"\n\nfunc SortFilePaths(paths []string, dirsOrder DirsOrder) []string {\n\tif dirsOrder == DirsSorted {\n\t\tsort.Strings(paths)\n\t\treturn paths\n\t}\n\n\tdirsSortChar := \"\"\n\tif dirsOrder == DirsOnTop {\n\t\tdirsSortChar = \" \"\n\t}\n\tif dirsOrder == DirsOnBottom {\n\t\tdirsSortChar = \"|\"\n\t}\n\n\tfor i, path := range paths {\n\t\tdirPath := filepath.Dir(path)\n\t\tif dirPath == \".\" || dirPath == \"/\" {\n\t\t\tcontinue\n\t\t}\n\t\treplacedDir := strings.Replace(dirPath, \"/\", \"/\"+dirsSortChar, -1)\n\t\tpaths[i] = dirsSortChar + replacedDir + \"/\" + filepath.Base(path)\n\t}\n\tsort.Strings(paths)\n\tfor i, path := range paths {\n\t\tpaths[i] = strings.Replace(path, dirsSortChar, \"\", -1)\n\t}\n\n\treturn paths\n}\n\nfunc FileTree(paths []string, cwd string, dirsOrder DirsOrder) string {\n\tif len(paths) == 0 {\n\t\treturn \".\"\n\t}\n\n\tpaths = SortFilePaths(paths, dirsOrder)\n\tdirPaths := DirPaths(paths, true)\n\n\tdidTrimCwd := false\n\tif cwd != \"\" {\n\t\tcwd = s.EnsureSuffix(cwd, \"/\")\n\t\tpaths, didTrimCwd = s.TrimPrefixIfPresentInAll(paths, cwd)\n\t\tdirPaths = DirPaths(paths, true)\n\t}\n\n\tcommonPrefix := LongestCommonPath(dirPaths...)\n\tpaths, _ = s.TrimPrefixIfPresentInAll(paths, commonPrefix)\n\n\tvar header string\n\n\tif didTrimCwd && commonPrefix == \"\" {\n\t\theader = \".\\n\"\n\t} else if !didTrimCwd && commonPrefix == \"\" {\n\t\theader = \"\"\n\t} else if didTrimCwd && commonPrefix != \"\" {\n\t\theader = \"./\" + commonPrefix\n\t\theader = s.EnsureSingleOccurrenceCharSuffix(header, \"/\") + \"\\n\"\n\t} else if !didTrimCwd && commonPrefix != \"\" {\n\t\theader = commonPrefix + \"/\"\n\t\theader = s.EnsureSingleOccurrenceCharSuffix(header, \"/\") + \"\\n\"\n\t}\n\n\ttree := treeprint.New()\n\n\tcachedTrees := make(map[string]treeprint.Tree)\n\tcachedTrees[\".\"] = tree\n\tcachedTrees[\"/\"] = tree\n\tcachedTrees[\"\"] = tree\n\n\tfor _, path := range paths {\n\t\tdir := filepath.Dir(path)\n\t\tbranch := getTreeBranch(dir, cachedTrees)\n\t\tbranch.AddNode(filepath.Base(path))\n\t}\n\n\ttreeStr := tree.String()\n\treturn header + treeStr[2:]\n}\n\nfunc getTreeBranch(dir string, cachedTrees map[string]treeprint.Tree) treeprint.Tree {\n\tdir = s.TrimPrefixAndSuffix(dir, \"/\")\n\tif cachedTree, ok := cachedTrees[dir]; ok {\n\t\treturn cachedTree\n\t}\n\n\tvar parentDir, lastDir string\n\n\tlastIndex := strings.LastIndex(dir, \"/\")\n\tif lastIndex == -1 {\n\t\tparentDir = \".\"\n\t\tlastDir = dir\n\t} else {\n\t\tparentDir = s.TrimPrefixAndSuffix(dir[:lastIndex], \"/\")\n\t\tlastDir = s.TrimPrefixAndSuffix(dir[lastIndex:], \"/\")\n\t}\n\n\tparentBranch := getTreeBranch(parentDir, cachedTrees)\n\tbranch := parentBranch.AddBranch(lastDir)\n\tcachedTrees[dir] = branch\n\treturn branch\n}\n\n// Return the path to the directory containing the provided path (with a trailing slash)\nfunc Dir(path string) string {\n\treturn s.EnsureSuffix(filepath.Dir(path), \"/\")\n}\n\nfunc DirPaths(paths []string, addTrailingSlash bool) []string {\n\tsuffix := \"\"\n\tif addTrailingSlash {\n\t\tsuffix = \"/\"\n\t}\n\tdirPaths := make([]string, len(paths))\n\tfor i, path := range paths {\n\t\tdirPaths[i] = s.EnsureSuffix(filepath.Dir(path), suffix)\n\t}\n\treturn dirPaths\n}\n\nfunc ListDirRecursive(dir string, relative bool, ignoreFns ...IgnoreFn) ([]string, error) {\n\tcleanDir, err := Clean(dir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcleanDir = strings.TrimSuffix(cleanDir, \"/\")\n\n\tif err := CheckDir(cleanDir); err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar fileList []string\n\twalkErr := filepath.Walk(cleanDir, func(path string, fi os.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, path)\n\t\t}\n\n\t\tfor _, ignoreFn := range ignoreFns {\n\t\t\tignore, err := ignoreFn(path, fi)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif ignore {\n\t\t\t\tif fi.IsDir() {\n\t\t\t\t\treturn filepath.SkipDir\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\n\t\tif !fi.IsDir() {\n\t\t\tif relative && dir != \".\" {\n\t\t\t\tpath = path[len(cleanDir)+1:]\n\t\t\t}\n\t\t\tfileList = append(fileList, path)\n\t\t}\n\t\treturn nil\n\t})\n\n\tif walkErr != nil {\n\t\treturn nil, walkErr\n\t}\n\n\treturn fileList, nil\n}\n\nfunc ListDir(dir string, relative bool) ([]string, error) {\n\tcleanDir, err := Clean(dir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcleanDir = strings.TrimSuffix(cleanDir, \"/\")\n\n\tif err := CheckDir(cleanDir); err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar filenames []string\n\tfileInfo, err := ioutil.ReadDir(cleanDir)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorReadDir(dir)))\n\t}\n\tfor _, file := range fileInfo {\n\t\tfilename := file.Name()\n\t\tif !relative {\n\t\t\tfilename = filepath.Join(cleanDir, filename)\n\t\t}\n\t\tfilenames = append(filenames, filename)\n\t}\n\treturn filenames, nil\n}\n\nfunc CopyFileOverwrite(src string, dest string) error {\n\tsrcFile, err := Open(src)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer srcFile.Close()\n\n\tdestFile, err := Create(dest)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer destFile.Close()\n\n\tif _, err = io.Copy(destFile, srcFile); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc CopyDirOverwrite(src string, dest string, ignoreFns ...IgnoreFn) error {\n\tsrcRelFilePaths, err := ListDirRecursive(src, true, ignoreFns...)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif _, err := CreateDirIfMissing(dest); err != nil {\n\t\treturn err\n\t}\n\tcreatedDirs := strset.New(dest)\n\n\tfor _, srcRelFilePath := range srcRelFilePaths {\n\t\tsrcFilePath := filepath.Join(src, srcRelFilePath)\n\t\tdestFilePath := filepath.Join(dest, srcRelFilePath)\n\n\t\tdestFileDir := filepath.Dir(destFilePath)\n\t\tif !createdDirs.Has(destFileDir) {\n\t\t\tif _, err := CreateDirIfMissing(destFileDir); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tcreatedDirs.Add(destFileDir)\n\t\t}\n\n\t\tif err := CopyFileOverwrite(srcFilePath, destFilePath); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc CopyRecursiveShell(src string, dest string) error {\n\tcleanSrc, err := EscapeTilde(src)\n\tif err != nil {\n\t\treturn err\n\t}\n\tcleanDest, err := EscapeTilde(dest)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tcmd := exec.Command(\"cp\", \"-r\", cleanSrc, cleanDest)\n\tvar out bytes.Buffer\n\tcmd.Stdout = &out\n\tcmd.Stderr = &out\n\n\tif err := cmd.Run(); err != nil {\n\t\treturn errors.Wrap(err, strings.TrimSpace(out.String()))\n\t}\n\n\treturn nil\n}\n\nfunc HashFile(path string, paths ...string) (string, error) {\n\tmd5Hash := md5.New()\n\n\tallPaths := append(paths, path)\n\tfor _, path := range allPaths {\n\t\tf, err := Open(path)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\t// Skip directories\n\t\tfileInfo, err := f.Stat()\n\t\tif err != nil {\n\t\t\tf.Close()\n\t\t\treturn \"\", errors.WithStack(err)\n\t\t}\n\t\tif fileInfo.IsDir() {\n\t\t\tf.Close()\n\t\t\tcontinue\n\t\t}\n\n\t\tif _, err := io.Copy(md5Hash, f); err != nil {\n\t\t\tf.Close()\n\t\t\treturn \"\", errors.Wrap(err, path)\n\t\t}\n\n\t\tio.WriteString(md5Hash, path)\n\t\tf.Close()\n\t}\n\n\treturn hex.EncodeToString(md5Hash.Sum(nil)), nil\n}\n\nfunc HashDirectory(dir string, ignoreFns ...IgnoreFn) (string, error) {\n\tmd5Hash := md5.New()\n\n\twalkErr := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, path)\n\t\t}\n\n\t\tfor _, ignoreFn := range ignoreFns {\n\t\t\tignore, err := ignoreFn(path, fi)\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, path)\n\t\t\t}\n\t\t\tif ignore {\n\t\t\t\tif fi.IsDir() {\n\t\t\t\t\treturn filepath.SkipDir\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tif fi.IsDir() {\n\t\t\treturn nil\n\t\t}\n\t\tf, err := os.Open(path)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, path)\n\t\t}\n\t\tdefer f.Close()\n\n\t\tif _, err := io.Copy(md5Hash, f); err != nil {\n\t\t\treturn errors.Wrap(err, path)\n\t\t}\n\n\t\tio.WriteString(md5Hash, path)\n\t\treturn nil\n\t})\n\n\tif walkErr != nil {\n\t\treturn \"\", walkErr\n\t}\n\n\treturn hex.EncodeToString(md5Hash.Sum(nil)), nil\n}\n\nfunc CloseSilent(closer io.Closer, closers ...io.Closer) {\n\tallClosers := append(closers, closer)\n\tfor _, closer := range allClosers {\n\t\tcloser.Close()\n\t}\n}\n\n// ReadReqFile returns nil if no file\nfunc ReadReqFile(r *http.Request, fileName string) ([]byte, error) {\n\tmpFile, _, err := r.FormFile(fileName)\n\tif err != nil {\n\t\tif strings.Contains(errors.Message(err), \"no such file\") {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorReadFormFile(fileName)))\n\t}\n\tdefer mpFile.Close()\n\tfileBytes, err := ioutil.ReadAll(mpFile)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorReadFormFile(fileName)))\n\t}\n\treturn fileBytes, nil\n}\n"
  },
  {
    "path": "pkg/lib/files/files_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage files\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc IgnoreDir3(path string, fi os.FileInfo) (bool, error) {\n\tif fi.IsDir() && fi.Name() == \"3\" {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\nfunc TestPrintFileTree(t *testing.T) {\n\tvar filesList []string\n\tvar cwd string\n\tvar expectedTree string\n\tvar expectedHeader string\n\n\tfilesList = []string{\n\t\t\"/1/2/3.txt\",\n\t\t\"/1/2/3/5.txt\",\n\t\t\"/1/2/4/5/6.txt\",\n\t\t\"/1/2/3/4.txt\",\n\t}\n\texpectedTree = `\n├── 3.txt\n├── 3\n│   ├── 4.txt\n│   └── 5.txt\n└── 4\n    └── 5\n        └── 6.txt\n`\n\n\tcwd = \"\"\n\texpectedHeader = \"/1/2/\"\n\trequire.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted))\n\n\tcwd = \"/missing\"\n\texpectedHeader = \"/1/2/\"\n\trequire.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted))\n\n\tcwd = \"/1/2\"\n\texpectedHeader = \".\"\n\trequire.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted))\n\n\tcwd = \"/1/2/\"\n\texpectedHeader = \".\"\n\trequire.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted))\n\n\tcwd = \"/1\"\n\texpectedHeader = \"./2/\"\n\trequire.Equal(t, expectedHeader+expectedTree, FileTree(filesList, cwd, DirsSorted))\n\n\tfilesList = []string{\n\t\t\"/1\",\n\t\t\"/2\",\n\t\t\"/1/1\",\n\t\t\"/1/2\",\n\t\t\"/2/1\",\n\t\t\"/2/2\",\n\t\t\"/1/1/1\",\n\t\t\"/1/1/2\",\n\t\t\"/1/2/1\",\n\t\t\"/1/2/2\",\n\t\t\"/2/1/1\",\n\t\t\"/2/1/2\",\n\t\t\"/2/2/1\",\n\t\t\"/2/2/2\",\n\t}\n\n\texpectedTree = `\n├── 1\n├── 1\n│   ├── 1\n│   ├── 1\n│   │   ├── 1\n│   │   └── 2\n│   ├── 2\n│   └── 2\n│       ├── 1\n│       └── 2\n├── 2\n└── 2\n    ├── 1\n    ├── 1\n    │   ├── 1\n    │   └── 2\n    ├── 2\n    └── 2\n        ├── 1\n        └── 2\n`\n\trequire.Equal(t, \"/\"+expectedTree, FileTree(filesList, cwd, DirsSorted))\n\n\texpectedTree = `\n├── 1\n├── 2\n├── 1\n│   ├── 1\n│   ├── 2\n│   ├── 1\n│   │   ├── 1\n│   │   └── 2\n│   └── 2\n│       ├── 1\n│       └── 2\n└── 2\n    ├── 1\n    ├── 2\n    ├── 1\n    │   ├── 1\n    │   └── 2\n    └── 2\n        ├── 1\n        └── 2\n`\n\trequire.Equal(t, \"/\"+expectedTree, FileTree(filesList, cwd, DirsOnBottom))\n\n\texpectedTree = `\n├── 1\n│   ├── 1\n│   │   ├── 1\n│   │   └── 2\n│   ├── 2\n│   │   ├── 1\n│   │   └── 2\n│   ├── 1\n│   └── 2\n├── 2\n│   ├── 1\n│   │   ├── 1\n│   │   └── 2\n│   ├── 2\n│   │   ├── 1\n│   │   └── 2\n│   ├── 1\n│   └── 2\n├── 1\n└── 2\n`\n\trequire.Equal(t, \"/\"+expectedTree, FileTree(filesList, cwd, DirsOnTop))\n}\n\nfunc TestListDirRecursive(t *testing.T) {\n\t_, err := ListDirRecursive(\"/home/path/to/fake/dir\", false)\n\trequire.Error(t, err)\n\n\ttmpDir, err := TmpDir()\n\tdefer os.RemoveAll(tmpDir)\n\trequire.NoError(t, err)\n\n\tfilesList := []string{\n\t\tfilepath.Join(tmpDir, \"1.txt\"),\n\t\tfilepath.Join(tmpDir, \"2.py\"),\n\t\tfilepath.Join(tmpDir, \"3/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/2.txt\"),\n\t\tfilepath.Join(tmpDir, \"3/2/4.md\"),\n\t\tfilepath.Join(tmpDir, \"3/2/3/.tmp\"),\n\t\tfilepath.Join(tmpDir, \"4/1.yaml\"),\n\t\tfilepath.Join(tmpDir, \"4/2.pyc\"),\n\t\tfilepath.Join(tmpDir, \"4/3.md\"),\n\t\tfilepath.Join(tmpDir, \"4/.git/HEAD\"),\n\t\tfilepath.Join(tmpDir, \"README.md\"),\n\t\tfilepath.Join(tmpDir, \".ignore\"),\n\t}\n\n\tignoreContents := `\n*.md\n*.txt\n4/.git\n3/2/1.py\n!README.md\n  `\n\tWriteFile([]byte(ignoreContents), filepath.Join(tmpDir, \".ignore\"))\n\n\tignoreFn, err := GitIgnoreFn(filepath.Join(tmpDir, \".ignore\"))\n\trequire.NoError(t, err)\n\n\terr = MakeEmptyFiles(filesList[0], filesList[1:]...)\n\trequire.NoError(t, err)\n\n\tvar filesListRecursive []string\n\tvar expected []string\n\n\tfilesListRecursive, err = ListDirRecursive(tmpDir, false)\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, filesList, filesListRecursive)\n\n\tfilesListRecursive, err = ListDirRecursive(tmpDir, false, IgnoreHiddenFiles)\n\texpected = []string{\n\t\tfilepath.Join(tmpDir, \"1.txt\"),\n\t\tfilepath.Join(tmpDir, \"2.py\"),\n\t\tfilepath.Join(tmpDir, \"3/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/2.txt\"),\n\t\tfilepath.Join(tmpDir, \"3/2/4.md\"),\n\t\tfilepath.Join(tmpDir, \"4/1.yaml\"),\n\t\tfilepath.Join(tmpDir, \"4/2.pyc\"),\n\t\tfilepath.Join(tmpDir, \"4/3.md\"),\n\t\tfilepath.Join(tmpDir, \"4/.git/HEAD\"),\n\t\tfilepath.Join(tmpDir, \"README.md\"),\n\t}\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, filesListRecursive)\n\n\tfilesListRecursive, err = ListDirRecursive(tmpDir, false, IgnoreHiddenFiles, IgnoreHiddenFolders)\n\texpected = []string{\n\t\tfilepath.Join(tmpDir, \"1.txt\"),\n\t\tfilepath.Join(tmpDir, \"2.py\"),\n\t\tfilepath.Join(tmpDir, \"3/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/2.txt\"),\n\t\tfilepath.Join(tmpDir, \"3/2/4.md\"),\n\t\tfilepath.Join(tmpDir, \"4/1.yaml\"),\n\t\tfilepath.Join(tmpDir, \"4/2.pyc\"),\n\t\tfilepath.Join(tmpDir, \"4/3.md\"),\n\t\tfilepath.Join(tmpDir, \"README.md\"),\n\t}\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, filesListRecursive)\n\n\tfilesListRecursive, err = ListDirRecursive(tmpDir, false, IgnoreHiddenFiles, IgnoreDir3, IgnorePythonGeneratedFiles)\n\texpected = []string{\n\t\tfilepath.Join(tmpDir, \"1.txt\"),\n\t\tfilepath.Join(tmpDir, \"2.py\"),\n\t\tfilepath.Join(tmpDir, \"4/1.yaml\"),\n\t\tfilepath.Join(tmpDir, \"4/3.md\"),\n\t\tfilepath.Join(tmpDir, \"4/.git/HEAD\"),\n\t\tfilepath.Join(tmpDir, \"README.md\"),\n\t}\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, filesListRecursive)\n\n\tfilesListRecursive, err = ListDirRecursive(tmpDir, false, IgnoreNonPython)\n\texpected = []string{\n\t\tfilepath.Join(tmpDir, \"2.py\"),\n\t\tfilepath.Join(tmpDir, \"3/1.py\"),\n\t\tfilepath.Join(tmpDir, \"3/2/1.py\"),\n\t}\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, filesListRecursive)\n\n\tfilesListRecursive, err = ListDirRecursive(tmpDir, true, IgnoreNonPython)\n\texpected = []string{\n\t\tfilepath.Join(\"2.py\"),\n\t\tfilepath.Join(\"3/1.py\"),\n\t\tfilepath.Join(\"3/2/1.py\"),\n\t}\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, filesListRecursive)\n\n\tfilesListRecursive, err = ListDirRecursive(tmpDir, true, ignoreFn)\n\texpected = []string{\n\t\t\"2.py\",\n\t\t\"3/1.py\",\n\t\t\"3/2/3/.tmp\",\n\t\t\"4/1.yaml\",\n\t\t\"4/2.pyc\",\n\t\t\"README.md\",\n\t\t\".ignore\",\n\t}\n\trequire.NoError(t, err)\n\trequire.ElementsMatch(t, expected, filesListRecursive)\n}\n"
  },
  {
    "path": "pkg/lib/hash/hash.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage hash\n\nimport (\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\n// Bytes will trim to 63 characters because e.g. K8s labels must be < 64\nfunc Bytes(bytes []byte) string {\n\thash := sha256.New()\n\thash.Write(bytes)\n\tstr := hex.EncodeToString(hash.Sum(nil))\n\treturn str[:63]\n}\n\nfunc String(str string) string {\n\treturn Bytes([]byte(str))\n}\n\nfunc Strings(strs ...string) string {\n\treturn String(strings.Join(strs, \",\"))\n}\n\nfunc Any(obj interface{}) string {\n\treturn String(s.Obj(obj))\n}\n\nfunc File(path string) (string, error) {\n\tfileBytes, err := files.ReadFileBytes(path)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn Bytes(fileBytes), nil\n}\n"
  },
  {
    "path": "pkg/lib/json/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage json\n\nconst (\n\terrStrMarshalJSON   = \"invalid json\"\n\terrStrUnmarshalJSON = \"invalid json cannot be serialized\"\n)\n"
  },
  {
    "path": "pkg/lib/json/json.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage json\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"path/filepath\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n)\n\nfunc MarshalIndent(obj interface{}) ([]byte, error) {\n\tjsonBytes, err := json.MarshalIndent(obj, \"\", \"  \")\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errStrMarshalJSON)\n\t}\n\treturn jsonBytes, nil\n}\n\nfunc Marshal(obj interface{}) ([]byte, error) {\n\tjsonBytes, err := json.Marshal(obj)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errStrMarshalJSON)\n\t}\n\treturn jsonBytes, nil\n}\n\nfunc Unmarshal(jsonBytes []byte, dst interface{}) error {\n\tif err := json.Unmarshal(jsonBytes, dst); err != nil {\n\t\treturn errors.Wrap(err, errStrUnmarshalJSON)\n\t}\n\treturn nil\n}\n\nfunc DecodeWithNumber(jsonBytes []byte, dst interface{}) error {\n\td := json.NewDecoder(bytes.NewReader(jsonBytes))\n\td.UseNumber()\n\tif err := d.Decode(&dst); err != nil {\n\t\treturn errors.Wrap(err, errStrUnmarshalJSON)\n\t}\n\n\treturn nil\n}\n\nfunc MarshalJSONStr(obj interface{}) (string, error) {\n\tjsonBytes, err := Marshal(obj)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn string(jsonBytes), nil\n}\n\nfunc WriteJSON(obj interface{}, outPath string) error {\n\tjsonBytes, err := Marshal(obj)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := files.CreateDir(filepath.Dir(outPath)); err != nil {\n\t\treturn err\n\t}\n\n\tif err := files.WriteFile(jsonBytes, outPath); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc Pretty(obj interface{}) (string, error) {\n\tb, err := json.MarshalIndent(obj, \"\", \"  \")\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, errStrMarshalJSON)\n\t}\n\n\treturn string(b), nil\n}\n"
  },
  {
    "path": "pkg/lib/k8s/configmap.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nvar _configMapTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"v1\",\n\tKind:       \"ConfigMap\",\n}\n\ntype ConfigMapSpec struct {\n\tName        string\n\tData        map[string]string // Data and BinaryData must not have overlapping keys\n\tBinaryData  map[string][]byte // Data and BinaryData must not have overlapping keys\n\tLabels      map[string]string\n\tAnnotations map[string]string\n}\n\nfunc ConfigMap(spec *ConfigMapSpec) *kcore.ConfigMap {\n\tconfigMap := &kcore.ConfigMap{\n\t\tTypeMeta: _configMapTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.Name,\n\t\t\tLabels:      spec.Labels,\n\t\t\tAnnotations: spec.Annotations,\n\t\t},\n\t\tData:       spec.Data,\n\t\tBinaryData: spec.BinaryData,\n\t}\n\treturn configMap\n}\n\nfunc (c *Client) CreateConfigMap(configMap *kcore.ConfigMap) (*kcore.ConfigMap, error) {\n\tconfigMap.TypeMeta = _configMapTypeMeta\n\tconfigMap, err := c.configMapClient.Create(context.Background(), configMap, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn configMap, nil\n}\n\nfunc (c *Client) UpdateConfigMap(configMap *kcore.ConfigMap) (*kcore.ConfigMap, error) {\n\tconfigMap.TypeMeta = _configMapTypeMeta\n\tconfigMap, err := c.configMapClient.Update(context.Background(), configMap, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn configMap, nil\n}\n\nfunc (c *Client) ApplyConfigMap(configMap *kcore.ConfigMap) (*kcore.ConfigMap, error) {\n\texisting, err := c.GetConfigMap(configMap.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreateConfigMap(configMap)\n\t}\n\treturn c.UpdateConfigMap(configMap)\n}\n\nfunc (c *Client) GetConfigMap(name string) (*kcore.ConfigMap, error) {\n\tconfigMap, err := c.configMapClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tconfigMap.TypeMeta = _configMapTypeMeta\n\treturn configMap, nil\n}\n\nfunc (c *Client) GetConfigMapData(name string) (map[string]string, map[string][]byte, error) {\n\tconfigMap, err := c.GetConfigMap(name)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tif configMap == nil {\n\t\treturn nil, nil, nil\n\t}\n\treturn configMap.Data, configMap.BinaryData, nil\n}\n\nfunc (c *Client) DeleteConfigMap(name string) (bool, error) {\n\terr := c.configMapClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) ListConfigMaps(opts *kmeta.ListOptions) ([]kcore.ConfigMap, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tconfigMapList, err := c.configMapClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range configMapList.Items {\n\t\tconfigMapList.Items[i].TypeMeta = _configMapTypeMeta\n\t}\n\treturn configMapList.Items, nil\n}\n\nfunc (c *Client) ListConfigMapsByLabels(labels map[string]string) ([]kcore.ConfigMap, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListConfigMaps(opts)\n}\n\nfunc (c *Client) ListConfigMapsByLabel(labelKey string, labelValue string) ([]kcore.ConfigMap, error) {\n\treturn c.ListConfigMapsByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListConfigMapsWithLabelKeys(labelKeys ...string) ([]kcore.ConfigMap, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListConfigMaps(opts)\n}\n\nfunc ConfigMapMap(configMaps []kcore.ConfigMap) map[string]kcore.ConfigMap {\n\tconfigMapMap := map[string]kcore.ConfigMap{}\n\tfor _, configMap := range configMaps {\n\t\tconfigMapMap[configMap.Name] = configMap\n\t}\n\treturn configMapMap\n}\n"
  },
  {
    "path": "pkg/lib/k8s/deployment.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nvar _deploymentTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"apps/v1\",\n\tKind:       \"Deployment\",\n}\n\ntype DeploymentSpec struct {\n\tName           string\n\tReplicas       int32\n\tPodSpec        PodSpec\n\tMaxSurge       *string // Can be a percentage (e.g. 10%) or an absolute number (e.g. 2)\n\tMaxUnavailable *string // Can be a percentage (e.g. 10%) or an absolute number (e.g. 2)\n\tSelector       map[string]string\n\tLabels         map[string]string\n\tAnnotations    map[string]string\n}\n\nfunc Deployment(spec *DeploymentSpec) *kapps.Deployment {\n\tif spec.PodSpec.Name == \"\" {\n\t\tspec.PodSpec.Name = spec.Name\n\t}\n\tif spec.Selector == nil {\n\t\tspec.Selector = spec.PodSpec.Labels\n\t}\n\n\tvar maxSurge *intstr.IntOrString\n\tif spec.MaxSurge != nil {\n\t\tintStr := intstr.Parse(*spec.MaxSurge)\n\t\tmaxSurge = &intStr\n\t}\n\n\tvar maxUnavailable *intstr.IntOrString\n\tif spec.MaxUnavailable != nil {\n\t\tintStr := intstr.Parse(*spec.MaxUnavailable)\n\t\tmaxUnavailable = &intStr\n\t}\n\n\tdeployment := &kapps.Deployment{\n\t\tTypeMeta: _deploymentTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.Name,\n\t\t\tLabels:      spec.Labels,\n\t\t\tAnnotations: spec.Annotations,\n\t\t},\n\t\tSpec: kapps.DeploymentSpec{\n\t\t\tReplicas: &spec.Replicas,\n\t\t\tStrategy: kapps.DeploymentStrategy{\n\t\t\t\tType: kapps.RollingUpdateDeploymentStrategyType,\n\t\t\t\tRollingUpdate: &kapps.RollingUpdateDeployment{\n\t\t\t\t\tMaxSurge:       maxSurge,\n\t\t\t\t\tMaxUnavailable: maxUnavailable,\n\t\t\t\t},\n\t\t\t},\n\t\t\tTemplate: kcore.PodTemplateSpec{\n\t\t\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\t\t\tName:        spec.PodSpec.Name,\n\t\t\t\t\tLabels:      spec.PodSpec.Labels,\n\t\t\t\t\tAnnotations: spec.PodSpec.Annotations,\n\t\t\t\t},\n\t\t\t\tSpec: spec.PodSpec.K8sPodSpec,\n\t\t\t},\n\t\t\tSelector: &kmeta.LabelSelector{\n\t\t\t\tMatchLabels: spec.Selector,\n\t\t\t},\n\t\t},\n\t}\n\treturn deployment\n}\n\nfunc (c *Client) CreateDeployment(deployment *kapps.Deployment) (*kapps.Deployment, error) {\n\tdeployment.TypeMeta = _deploymentTypeMeta\n\tdeployment, err := c.deploymentClient.Create(context.Background(), deployment, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn deployment, nil\n}\n\nfunc (c *Client) UpdateDeployment(deployment *kapps.Deployment) (*kapps.Deployment, error) {\n\tdeployment.TypeMeta = _deploymentTypeMeta\n\tdeployment, err := c.deploymentClient.Update(context.Background(), deployment, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn deployment, nil\n}\n\nfunc (c *Client) ApplyDeployment(deployment *kapps.Deployment) (*kapps.Deployment, error) {\n\texisting, err := c.GetDeployment(deployment.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreateDeployment(deployment)\n\t}\n\treturn c.UpdateDeployment(deployment)\n}\n\nfunc (c *Client) GetDeployment(name string) (*kapps.Deployment, error) {\n\tdeployment, err := c.deploymentClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdeployment.TypeMeta = _deploymentTypeMeta\n\treturn deployment, nil\n}\n\nfunc (c *Client) DeleteDeployment(name string) (bool, error) {\n\terr := c.deploymentClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) ListDeployments(opts *kmeta.ListOptions) ([]kapps.Deployment, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tdeploymentList, err := c.deploymentClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range deploymentList.Items {\n\t\tdeploymentList.Items[i].TypeMeta = _deploymentTypeMeta\n\t}\n\treturn deploymentList.Items, nil\n}\n\nfunc (c *Client) ListDeploymentsByLabels(labels map[string]string) ([]kapps.Deployment, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListDeployments(opts)\n}\n\nfunc (c *Client) ListDeploymentsByLabel(labelKey string, labelValue string) ([]kapps.Deployment, error) {\n\treturn c.ListDeploymentsByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListDeploymentsWithLabelKeys(labelKeys ...string) ([]kapps.Deployment, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListDeployments(opts)\n}\n\nfunc DeploymentMap(deployments []kapps.Deployment) map[string]kapps.Deployment {\n\tdeploymentMap := map[string]kapps.Deployment{}\n\tfor _, deployment := range deployments {\n\t\tdeploymentMap[deployment.Name] = deployment\n\t}\n\treturn deploymentMap\n}\n\nfunc DeploymentStartTime(deployment *kapps.Deployment) *time.Time {\n\tif deployment == nil {\n\t\treturn nil\n\t}\n\tt := deployment.CreationTimestamp.Time\n\tif t.IsZero() {\n\t\treturn nil\n\t}\n\treturn &deployment.CreationTimestamp.Time\n}\n\nfunc DeploymentStrategiesMatch(s1, s2 kapps.DeploymentStrategy) bool {\n\tif s1.Type != s2.Type {\n\t\treturn false\n\t}\n\tif s1.RollingUpdate == nil && s2.RollingUpdate == nil {\n\t\treturn true\n\t}\n\tif s1.RollingUpdate == nil || s2.RollingUpdate == nil {\n\t\treturn false\n\t}\n\tif !intOrStrPtrsMatch(s1.RollingUpdate.MaxUnavailable, s2.RollingUpdate.MaxUnavailable) {\n\t\treturn false\n\t}\n\tif !intOrStrPtrsMatch(s1.RollingUpdate.MaxSurge, s2.RollingUpdate.MaxSurge) {\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc intOrStrPtrsMatch(intStr1, intStr2 *intstr.IntOrString) bool {\n\tif intStr1 == nil && intStr2 == nil {\n\t\treturn true\n\t}\n\tif intStr1 == nil || intStr2 == nil {\n\t\treturn false\n\t}\n\treturn (*intStr1).String() == (*intStr2).String()\n}\n"
  },
  {
    "path": "pkg/lib/k8s/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrLabelNotFound      = \"k8s.label_not_found\"\n\tErrAnnotationNotFound = \"k8s.annotation_not_found\"\n\tErrParseLabel         = \"k8s.parse_label\"\n\tErrParseAnnotation    = \"k8s.parse_annotation\"\n\tErrParseQuantity      = \"k8s.parse_quantity\"\n\tErrMissingMetrics     = \"k8s.missing_metrics\"\n\tErrServiceNotFound    = \"k8s.service_not_found\"\n)\n\nfunc ErrorLabelNotFound(labelName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrLabelNotFound,\n\t\tMessage: fmt.Sprintf(\"label %s not found\", s.UserStr(labelName)),\n\t})\n}\n\nfunc ErrorAnnotationNotFound(annotationName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAnnotationNotFound,\n\t\tMessage: fmt.Sprintf(\"annotation %s not found\", s.UserStr(annotationName)),\n\t})\n}\n\nfunc ErrorParseLabel(labelName string, labelVal string, desiredType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrParseLabel,\n\t\tMessage: fmt.Sprintf(\"unable to parse value %s from label %s as type %s\", s.UserStr(labelVal), labelName, desiredType),\n\t})\n}\n\nfunc ErrorParseAnnotation(annotationName string, annotationVal string, desiredType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrParseAnnotation,\n\t\tMessage: fmt.Sprintf(\"unable to parse value %s from annotation %s as type %s\", s.UserStr(annotationVal), annotationName, desiredType),\n\t})\n}\n\nfunc ErrorParseQuantity(qtyStr string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrParseQuantity,\n\t\tMessage: fmt.Sprintf(\"%s: invalid kubernetes quantity, some valid examples are 1, 200m, 500Mi, 2G (see here for more information: https://docs.cortexlabs.com/v/%s/)\", qtyStr, consts.CortexVersionMinor),\n\t})\n}\n\nfunc ErrorMissingMetrics() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMissingMetrics,\n\t\tMessage: \"must specify at least one metric\",\n\t})\n}\n\nfunc ErrorServiceNotFound(serviceName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrServiceNotFound,\n\t\tMessage: fmt.Sprintf(\"service %s couldn't be found\", serviceName),\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/k8s/hpa.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tkautoscaling \"k8s.io/api/autoscaling/v2beta2\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nvar _hpaTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"autoscaling/v1\",\n\tKind:       \"HorizontalPodAutoscaler\",\n}\n\ntype HPASpec struct {\n\tDeploymentName       string\n\tMinReplicas          int32\n\tMaxReplicas          int32\n\tTargetCPUUtilization int32\n\tTargetMemUtilization int32\n\tLabels               map[string]string\n\tAnnotations          map[string]string\n}\n\nfunc HPA(spec *HPASpec) (*kautoscaling.HorizontalPodAutoscaler, error) {\n\tmetrics := []kautoscaling.MetricSpec{}\n\tif spec.TargetCPUUtilization > 0 {\n\t\tmetrics = append(metrics, kautoscaling.MetricSpec{\n\t\t\tType: kautoscaling.ResourceMetricSourceType,\n\t\t\tResource: &kautoscaling.ResourceMetricSource{\n\t\t\t\tName: kcore.ResourceCPU,\n\t\t\t\tTarget: kautoscaling.MetricTarget{\n\t\t\t\t\tType:               kautoscaling.UtilizationMetricType,\n\t\t\t\t\tAverageUtilization: &spec.TargetCPUUtilization,\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\tif spec.TargetMemUtilization > 0 {\n\t\tmetrics = append(metrics, kautoscaling.MetricSpec{\n\t\t\tType: kautoscaling.ResourceMetricSourceType,\n\t\t\tResource: &kautoscaling.ResourceMetricSource{\n\t\t\t\tName: kcore.ResourceMemory,\n\t\t\t\tTarget: kautoscaling.MetricTarget{\n\t\t\t\t\tType:               kautoscaling.UtilizationMetricType,\n\t\t\t\t\tAverageUtilization: &spec.TargetMemUtilization,\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\tif len(metrics) == 0 {\n\t\treturn nil, ErrorMissingMetrics()\n\t}\n\n\thpa := &kautoscaling.HorizontalPodAutoscaler{\n\t\tTypeMeta: _hpaTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.DeploymentName,\n\t\t\tLabels:      spec.Labels,\n\t\t\tAnnotations: spec.Annotations,\n\t\t},\n\t\tSpec: kautoscaling.HorizontalPodAutoscalerSpec{\n\t\t\tMinReplicas: &spec.MinReplicas,\n\t\t\tMaxReplicas: spec.MaxReplicas,\n\t\t\tMetrics:     metrics,\n\t\t\tScaleTargetRef: kautoscaling.CrossVersionObjectReference{\n\t\t\t\tKind:       _deploymentTypeMeta.Kind,\n\t\t\t\tName:       spec.DeploymentName,\n\t\t\t\tAPIVersion: _deploymentTypeMeta.APIVersion,\n\t\t\t},\n\t\t},\n\t}\n\treturn hpa, nil\n}\n\nfunc (c *Client) CreateHPA(hpa *kautoscaling.HorizontalPodAutoscaler) (*kautoscaling.HorizontalPodAutoscaler, error) {\n\thpa.TypeMeta = _hpaTypeMeta\n\thpa, err := c.hpaClient.Create(context.Background(), hpa, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn hpa, nil\n}\n\nfunc (c *Client) UpdateHPA(hpa *kautoscaling.HorizontalPodAutoscaler) (*kautoscaling.HorizontalPodAutoscaler, error) {\n\thpa.TypeMeta = _hpaTypeMeta\n\thpa, err := c.hpaClient.Update(context.Background(), hpa, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn hpa, nil\n}\n\nfunc (c *Client) ApplyHPA(hpa *kautoscaling.HorizontalPodAutoscaler) (*kautoscaling.HorizontalPodAutoscaler, error) {\n\texisting, err := c.GetHPA(hpa.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreateHPA(hpa)\n\t}\n\treturn c.UpdateHPA(hpa)\n}\n\nfunc (c *Client) GetHPA(name string) (*kautoscaling.HorizontalPodAutoscaler, error) {\n\thpa, err := c.hpaClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\thpa.TypeMeta = _hpaTypeMeta\n\treturn hpa, nil\n}\n\nfunc (c *Client) DeleteHPA(name string) (bool, error) {\n\terr := c.hpaClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) ListHPAs(opts *kmeta.ListOptions) ([]kautoscaling.HorizontalPodAutoscaler, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\thpaList, err := c.hpaClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range hpaList.Items {\n\t\thpaList.Items[i].TypeMeta = _hpaTypeMeta\n\t}\n\treturn hpaList.Items, nil\n}\n\nfunc (c *Client) ListHPAsByLabels(labels map[string]string) ([]kautoscaling.HorizontalPodAutoscaler, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListHPAs(opts)\n}\n\nfunc (c *Client) ListHPAsByLabel(labelKey string, labelValue string) ([]kautoscaling.HorizontalPodAutoscaler, error) {\n\treturn c.ListHPAsByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListHPAsWithLabelKeys(labelKeys ...string) ([]kautoscaling.HorizontalPodAutoscaler, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListHPAs(opts)\n}\n\nfunc HPAMap(hpas []kautoscaling.HorizontalPodAutoscaler) map[string]kautoscaling.HorizontalPodAutoscaler {\n\thpaMap := map[string]kautoscaling.HorizontalPodAutoscaler{}\n\tfor _, hpa := range hpas {\n\t\thpaMap[hpa.Name] = hpa\n\t}\n\treturn hpaMap\n}\n\nfunc IsHPAUpToDate(hpa *kautoscaling.HorizontalPodAutoscaler, minReplicas, maxReplicas, targetCPUUtilization, targetMemUtilization int32) bool {\n\tif hpa == nil {\n\t\treturn false\n\t}\n\n\tif hpa.Spec.MinReplicas == nil || *hpa.Spec.MinReplicas != minReplicas {\n\t\treturn false\n\t}\n\n\tif hpa.Spec.MaxReplicas != maxReplicas {\n\t\treturn false\n\t}\n\n\tif len(hpa.Spec.Metrics) != 2 {\n\t\treturn false\n\t}\n\n\tfor _, metric := range hpa.Spec.Metrics {\n\t\tif metric.Type != kautoscaling.ResourceMetricSourceType || metric.Resource == nil {\n\t\t\treturn false\n\t\t}\n\t\tif metric.Resource.Target.Type != kautoscaling.UtilizationMetricType || metric.Resource.Target.AverageUtilization == nil {\n\t\t\treturn false\n\t\t}\n\t\tif metric.Resource.Name != kcore.ResourceCPU && metric.Resource.Name != kcore.ResourceMemory {\n\t\t\treturn false\n\t\t}\n\t\tif metric.Resource.Name == kcore.ResourceCPU && *metric.Resource.Target.AverageUtilization != targetCPUUtilization {\n\t\t\treturn false\n\t\t}\n\t\tif metric.Resource.Name == kcore.ResourceMemory && *metric.Resource.Target.AverageUtilization != targetMemUtilization {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n"
  },
  {
    "path": "pkg/lib/k8s/ingress.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tkextensions \"k8s.io/api/extensions/v1beta1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nvar _ingressTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"extensions/v1beta1\",\n\tKind:       \"Ingress\",\n}\n\ntype IngressSpec struct {\n\tName         string\n\tIngressClass string\n\tServiceName  string\n\tServicePort  int32\n\tPath         string\n\tLabels       map[string]string\n\tAnnotations  map[string]string\n}\n\nfunc Ingress(spec *IngressSpec) *kextensions.Ingress {\n\tif spec.Annotations == nil {\n\t\tspec.Annotations = make(map[string]string)\n\t}\n\tspec.Annotations[\"kubernetes.io/ingress.class\"] = spec.IngressClass\n\n\tingress := &kextensions.Ingress{\n\t\tTypeMeta: _ingressTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.Name,\n\t\t\tAnnotations: spec.Annotations,\n\t\t\tLabels:      spec.Labels,\n\t\t},\n\t\tSpec: kextensions.IngressSpec{\n\t\t\tRules: []kextensions.IngressRule{\n\t\t\t\t{\n\t\t\t\t\tIngressRuleValue: kextensions.IngressRuleValue{\n\t\t\t\t\t\tHTTP: &kextensions.HTTPIngressRuleValue{\n\t\t\t\t\t\t\tPaths: []kextensions.HTTPIngressPath{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tPath: spec.Path,\n\t\t\t\t\t\t\t\t\tBackend: kextensions.IngressBackend{\n\t\t\t\t\t\t\t\t\t\tServiceName: spec.ServiceName,\n\t\t\t\t\t\t\t\t\t\tServicePort: intstr.IntOrString{\n\t\t\t\t\t\t\t\t\t\t\tIntVal: spec.ServicePort,\n\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\treturn ingress\n}\n\nfunc (c *Client) CreateIngress(ingress *kextensions.Ingress) (*kextensions.Ingress, error) {\n\tingress.TypeMeta = _ingressTypeMeta\n\tingress, err := c.ingressClient.Create(context.Background(), ingress, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn ingress, nil\n}\n\nfunc (c *Client) UpdateIngress(ingress *kextensions.Ingress) (*kextensions.Ingress, error) {\n\tingress.TypeMeta = _ingressTypeMeta\n\tingress, err := c.ingressClient.Update(context.Background(), ingress, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn ingress, nil\n}\n\nfunc (c *Client) ApplyIngress(ingress *kextensions.Ingress) (*kextensions.Ingress, error) {\n\texisting, err := c.GetIngress(ingress.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreateIngress(ingress)\n\t}\n\treturn c.UpdateIngress(ingress)\n}\n\nfunc (c *Client) GetIngress(name string) (*kextensions.Ingress, error) {\n\tingress, err := c.ingressClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tingress.TypeMeta = _ingressTypeMeta\n\treturn ingress, nil\n}\n\nfunc (c *Client) DeleteIngress(name string) (bool, error) {\n\terr := c.ingressClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) ListIngresses(opts *kmeta.ListOptions) ([]kextensions.Ingress, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tingressList, err := c.ingressClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range ingressList.Items {\n\t\tingressList.Items[i].TypeMeta = _ingressTypeMeta\n\t}\n\treturn ingressList.Items, nil\n}\n\nfunc (c *Client) ListIngressesByLabels(labels map[string]string) ([]kextensions.Ingress, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListIngresses(opts)\n}\n\nfunc (c *Client) ListIngressesByLabel(labelKey string, labelValue string) ([]kextensions.Ingress, error) {\n\treturn c.ListIngressesByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListIngressesWithLabelKeys(labelKeys ...string) ([]kextensions.Ingress, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListIngresses(opts)\n}\n\nfunc IngressMap(ingresses []kextensions.Ingress) map[string]kextensions.Ingress {\n\tingressMap := map[string]kextensions.Ingress{}\n\tfor _, ingress := range ingresses {\n\t\tingressMap[ingress.Name] = ingress\n\t}\n\treturn ingressMap\n}\n"
  },
  {
    "path": "pkg/lib/k8s/job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nvar _jobTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"batch/v1\",\n\tKind:       \"Job\",\n}\n\ntype JobSpec struct {\n\tName         string\n\tNamespace    string\n\tPodSpec      PodSpec\n\tParallelism  int32\n\tBackoffLimit int32\n\tLabels       map[string]string\n\tAnnotations  map[string]string\n}\n\nfunc Job(spec *JobSpec) *kbatch.Job {\n\tif spec.PodSpec.Name == \"\" {\n\t\tspec.PodSpec.Name = spec.Name\n\t}\n\n\tjob := &kbatch.Job{\n\t\tTypeMeta: _jobTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.Name,\n\t\t\tNamespace:   spec.Namespace,\n\t\t\tLabels:      spec.Labels,\n\t\t\tAnnotations: spec.Annotations,\n\t\t},\n\t\tSpec: kbatch.JobSpec{\n\t\t\tBackoffLimit: &spec.BackoffLimit,\n\t\t\tParallelism:  &spec.Parallelism,\n\t\t\tTemplate: kcore.PodTemplateSpec{\n\t\t\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\t\t\tName:        spec.PodSpec.Name,\n\t\t\t\t\tLabels:      spec.PodSpec.Labels,\n\t\t\t\t\tAnnotations: spec.PodSpec.Annotations,\n\t\t\t\t},\n\t\t\t\tSpec: spec.PodSpec.K8sPodSpec,\n\t\t\t},\n\t\t},\n\t}\n\treturn job\n}\n\nfunc (c *Client) CreateJob(job *kbatch.Job) (*kbatch.Job, error) {\n\tjob.TypeMeta = _jobTypeMeta\n\tjob, err := c.jobClient.Create(context.Background(), job, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn job, nil\n}\n\nfunc (c *Client) UpdateJob(job *kbatch.Job) (*kbatch.Job, error) {\n\tjob.TypeMeta = _jobTypeMeta\n\tjob, err := c.jobClient.Update(context.Background(), job, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn job, nil\n}\n\nfunc (c *Client) ApplyJob(job *kbatch.Job) (*kbatch.Job, error) {\n\texisting, err := c.GetJob(job.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreateJob(job)\n\t}\n\treturn c.UpdateJob(job)\n}\n\nfunc (c *Client) GetJob(name string) (*kbatch.Job, error) {\n\tjob, err := c.jobClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tjob.TypeMeta = _jobTypeMeta\n\treturn job, nil\n}\n\nfunc (c *Client) DeleteJob(name string) (bool, error) {\n\terr := c.jobClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) DeleteJobs(opts *kmeta.ListOptions) (bool, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\n\terr := c.jobClient.DeleteCollection(context.Background(), _deleteOpts, *opts)\n\tif err != nil {\n\t\treturn false, errors.WithStack(err)\n\t}\n\n\treturn true, nil\n}\n\nfunc (c *Client) ListJobs(opts *kmeta.ListOptions) ([]kbatch.Job, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tjobList, err := c.jobClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range jobList.Items {\n\t\tjobList.Items[i].TypeMeta = _jobTypeMeta\n\t}\n\treturn jobList.Items, nil\n}\n\nfunc (c *Client) ListJobsByLabels(labels map[string]string) ([]kbatch.Job, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListJobs(opts)\n}\n\nfunc (c *Client) ListJobsByLabel(labelKey string, labelValue string) ([]kbatch.Job, error) {\n\treturn c.ListJobsByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListJobsWithLabelKeys(labelKeys ...string) ([]kbatch.Job, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListJobs(opts)\n}\n\nfunc JobMap(jobs []kbatch.Job) map[string]kbatch.Job {\n\tjobMap := map[string]kbatch.Job{}\n\tfor _, job := range jobs {\n\t\tjobMap[job.Name] = job\n\t}\n\treturn jobMap\n}\n\nfunc (c *Client) IsJobRunning(name string) (bool, error) {\n\tjob, err := c.GetJob(name)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tif job == nil {\n\t\treturn false, nil\n\t}\n\treturn job.Status.CompletionTime == nil, nil\n}\n"
  },
  {
    "path": "pkg/lib/k8s/k8s.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"path\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/random\"\n\tistioclient \"istio.io/client-go/pkg/clientset/versioned\"\n\tistionetworkingclient \"istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\tkclientdynamic \"k8s.io/client-go/dynamic\"\n\tkclientset \"k8s.io/client-go/kubernetes\"\n\tkclientapps \"k8s.io/client-go/kubernetes/typed/apps/v1\"\n\tkclientautoscaling \"k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2\"\n\tkclientbatch \"k8s.io/client-go/kubernetes/typed/batch/v1\"\n\tkclientcore \"k8s.io/client-go/kubernetes/typed/core/v1\"\n\tkclientextensions \"k8s.io/client-go/kubernetes/typed/extensions/v1beta1\"\n\t_ \"k8s.io/client-go/plugin/pkg/client/auth/gcp\"\n\tkclientrest \"k8s.io/client-go/rest\"\n\tkclientcmd \"k8s.io/client-go/tools/clientcmd\"\n\tkclienthomedir \"k8s.io/client-go/util/homedir\"\n\tctrl \"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\nvar (\n\t_home         = kclienthomedir.HomeDir()\n\t_deletePolicy = kmeta.DeletePropagationBackground\n\t_deleteOpts   = kmeta.DeleteOptions{\n\t\tPropagationPolicy: &_deletePolicy,\n\t}\n)\n\ntype Client struct {\n\tctrl.Client\n\tRestConfig           *kclientrest.Config\n\tclientSet            *kclientset.Clientset\n\tistioClientSet       *istioclient.Clientset\n\tdynamicClient        kclientdynamic.Interface\n\tpodClient            kclientcore.PodInterface\n\tnodeClient           kclientcore.NodeInterface\n\tserviceClient        kclientcore.ServiceInterface\n\tconfigMapClient      kclientcore.ConfigMapInterface\n\tsecretClient         kclientcore.SecretInterface\n\tdeploymentClient     kclientapps.DeploymentInterface\n\tjobClient            kclientbatch.JobInterface\n\tingressClient        kclientextensions.IngressInterface\n\thpaClient            kclientautoscaling.HorizontalPodAutoscalerInterface\n\tvirtualServiceClient istionetworkingclient.VirtualServiceInterface\n\tNamespace            string\n}\n\nfunc New(namespace string, inCluster bool, restConfig *kclientrest.Config, scheme *runtime.Scheme) (*Client, error) {\n\tvar err error\n\tclient := &Client{\n\t\tNamespace: namespace,\n\t}\n\tif restConfig != nil {\n\t\tclient.RestConfig = restConfig\n\t} else if inCluster {\n\t\tclient.RestConfig, err = kclientrest.InClusterConfig()\n\t} else {\n\t\tkubeConfig := path.Join(_home, \".kube\", \"config\")\n\t\tclient.RestConfig, err = kclientcmd.BuildConfigFromFlags(\"\", kubeConfig)\n\t}\n\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"kubeconfig\")\n\t}\n\n\tclient.clientSet, err = kclientset.NewForConfig(client.RestConfig)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"kubeconfig\")\n\t}\n\n\tclient.dynamicClient, err = kclientdynamic.NewForConfig(client.RestConfig)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"kubeconfig\")\n\t}\n\n\tclient.Client, err = ctrl.New(client.RestConfig, ctrl.Options{Scheme: scheme})\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"kubeconfig\")\n\t}\n\n\tclient.istioClientSet, err = istioclient.NewForConfig(client.RestConfig)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"kubeconfig\")\n\t}\n\tclient.virtualServiceClient = client.istioClientSet.NetworkingV1beta1().VirtualServices(namespace)\n\n\tclient.podClient = client.clientSet.CoreV1().Pods(namespace)\n\tclient.nodeClient = client.clientSet.CoreV1().Nodes()\n\tclient.serviceClient = client.clientSet.CoreV1().Services(namespace)\n\tclient.configMapClient = client.clientSet.CoreV1().ConfigMaps(namespace)\n\tclient.secretClient = client.clientSet.CoreV1().Secrets(namespace)\n\tclient.deploymentClient = client.clientSet.AppsV1().Deployments(namespace)\n\tclient.jobClient = client.clientSet.BatchV1().Jobs(namespace)\n\tclient.ingressClient = client.clientSet.ExtensionsV1beta1().Ingresses(namespace)\n\tclient.hpaClient = client.clientSet.AutoscalingV2beta2().HorizontalPodAutoscalers(namespace)\n\treturn client, nil\n}\n\nfunc (c *Client) ClientSet() *kclientset.Clientset {\n\treturn c.clientSet\n}\n\nfunc (c *Client) IstioClientSet() *istioclient.Clientset {\n\treturn c.istioClientSet\n}\n\n// to be safe, k8s sometimes needs all characters to be lower case, and the first to be a letter\nfunc RandomName() string {\n\treturn random.LowercaseLetters(1) + random.LowercaseString(62)\n}\n\n// ValidName ensures name contains only lower case alphanumeric, '-', or '.'\nfunc ValidName(name string) string {\n\tre := regexp.MustCompile(`[^a-zA-Z0-9\\-\\.]`)\n\tname = re.ReplaceAllLiteralString(name, \"-\")\n\tname = strings.ToLower(name)\n\treturn name\n}\n\n// ValidNameContainer ensures name contains only lower case alphanumeric or '-', must start with alphabetic, end with alphanumeric\nfunc ValidNameContainer(name string) string {\n\tname = ValidName(name)\n\n\tdots := regexp.MustCompile(`[\\.]`)\n\tname = dots.ReplaceAllLiteralString(name, \"-\")\n\n\tleading := regexp.MustCompile(`^[^a-z]*`)\n\tname = leading.ReplaceAllLiteralString(name, \"\")\n\n\ttrailing := regexp.MustCompile(`[^a-z0-9]*$`)\n\tname = trailing.ReplaceAllLiteralString(name, \"\")\n\n\tif len(name) == 0 {\n\t\tname = \"x\"\n\t}\n\n\treturn name\n}\n\nfunc CPU(cpu string) kresource.Quantity {\n\treturn kresource.MustParse(cpu)\n}\n\nfunc Mem(mem string) kresource.Quantity {\n\treturn kresource.MustParse(mem)\n}\n\nfunc LabelExistsSelector(labelKeys ...string) string {\n\tif len(labelKeys) == 0 {\n\t\treturn \"\"\n\t}\n\n\treturn strings.Join(labelKeys, \",\")\n}\n\nfunc FieldSelectorNotIn(key string, values []string) string {\n\tselectors := make([]string, len(values))\n\tfor i, value := range values {\n\t\tselectors[i] = key + \"!=\" + value\n\t}\n\treturn strings.Join(selectors, \",\")\n}\n"
  },
  {
    "path": "pkg/lib/k8s/node.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tkcore \"k8s.io/api/core/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nvar _nodeTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"v1\",\n\tKind:       \"Node\",\n}\n\nfunc (c *Client) ListNodes(opts *kmeta.ListOptions) ([]kcore.Node, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tnodeList, err := c.nodeClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range nodeList.Items {\n\t\tnodeList.Items[i].TypeMeta = _nodeTypeMeta\n\t}\n\treturn nodeList.Items, nil\n}\n\nfunc (c *Client) ListNodesByLabels(labels map[string]string) ([]kcore.Node, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListNodes(opts)\n}\n\nfunc (c *Client) ListNodesByLabel(labelKey string, labelValue string) ([]kcore.Node, error) {\n\treturn c.ListNodesByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListNodesWithLabelKeys(labelKeys ...string) ([]kcore.Node, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListNodes(opts)\n}\n\nfunc HowManyPodsFitOnNode(podSpec kcore.PodSpec, node kcore.Node, cpuReserved resource.Quantity, memoryReserved resource.Quantity) int64 {\n\tcpuQty := node.Status.Allocatable[v1.ResourceCPU]\n\tmemoryQty := node.Status.Allocatable[v1.ResourceMemory]\n\tgpuQty := node.Status.Allocatable[\"nvidia.com/gpu\"]\n\tinfQty := node.Status.Allocatable[\"aws.amazon.com/neuron\"]\n\tpodsQty := node.Status.Allocatable[v1.ResourcePods]\n\n\tcpuQty.Sub(cpuReserved)\n\tmemoryReserved.Sub(memoryReserved)\n\n\tcpuInt64 := cpuQty.MilliValue()\n\tmemoryInt64 := memoryQty.MilliValue()\n\tgpuInt64 := gpuQty.Value()\n\tinfInt64 := infQty.Value()\n\tmaxPodsInt64 := podsQty.Value()\n\n\tcpuPodQty, memoryPodQty, podGPUInt64, podInfInt64 := TotalPodCompute(&podSpec)\n\tpodCPUInt64 := cpuPodQty.MilliValue()\n\tpodMemoryInt64 := memoryPodQty.MilliValue()\n\n\tif podCPUInt64 > 0 && float64(cpuInt64)/float64(podCPUInt64) < float64(maxPodsInt64) {\n\t\tmaxPodsInt64 = int64(float64(cpuInt64) / float64(podCPUInt64))\n\t}\n\tif podMemoryInt64 > 0 && float64(memoryInt64)/float64(podMemoryInt64) < float64(maxPodsInt64) {\n\t\tmaxPodsInt64 = int64(float64(memoryInt64) / float64(podMemoryInt64))\n\t}\n\tif podGPUInt64 > 0 && float64(gpuInt64)/float64(podGPUInt64) < float64(maxPodsInt64) {\n\t\tmaxPodsInt64 = int64(float64(gpuInt64) / float64(podGPUInt64))\n\t}\n\tif podInfInt64 > 0 && float64(infInt64)/float64(podInfInt64) < float64(maxPodsInt64) {\n\t\tmaxPodsInt64 = int64(float64(infInt64) / float64(podInfInt64))\n\t}\n\n\treturn maxPodsInt64\n}\n"
  },
  {
    "path": "pkg/lib/k8s/parsers.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"time\"\n\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t_ \"k8s.io/client-go/plugin/pkg/client/auth/gcp\"\n)\n\nfunc GetLabel(obj kmeta.Object, key string) (string, error) {\n\tlabels := obj.GetLabels()\n\tif labels == nil {\n\t\treturn \"\", ErrorLabelNotFound(key)\n\t}\n\tval, ok := labels[key]\n\tif !ok {\n\t\treturn \"\", ErrorLabelNotFound(key)\n\t}\n\treturn val, nil\n}\n\nfunc GetAnnotation(obj kmeta.Object, key string) (string, error) {\n\tannotations := obj.GetAnnotations()\n\tif annotations == nil {\n\t\treturn \"\", ErrorAnnotationNotFound(key)\n\t}\n\tval, ok := annotations[key]\n\tif !ok {\n\t\treturn \"\", ErrorAnnotationNotFound(key)\n\t}\n\treturn val, nil\n}\n\nfunc ParseBoolLabel(obj kmeta.Object, key string) (bool, error) {\n\tval, err := GetLabel(obj, key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tcasted, ok := s.ParseBool(val)\n\tif !ok {\n\t\treturn false, ErrorParseLabel(key, val, \"bool\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseBoolAnnotation(obj kmeta.Object, key string) (bool, error) {\n\tval, err := GetAnnotation(obj, key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tcasted, ok := s.ParseBool(val)\n\tif !ok {\n\t\treturn false, ErrorParseAnnotation(key, val, \"bool\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseIntLabel(obj kmeta.Object, key string) (int, error) {\n\tval, err := GetLabel(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseInt(val)\n\tif !ok {\n\t\treturn 0, ErrorParseLabel(key, val, \"int\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseIntAnnotation(obj kmeta.Object, key string) (int, error) {\n\tval, err := GetAnnotation(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseInt(val)\n\tif !ok {\n\t\treturn 0, ErrorParseAnnotation(key, val, \"int\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseInt32Label(obj kmeta.Object, key string) (int32, error) {\n\tval, err := GetLabel(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseInt32(val)\n\tif !ok {\n\t\treturn 0, ErrorParseLabel(key, val, \"int32\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseInt32Annotation(obj kmeta.Object, key string) (int32, error) {\n\tval, err := GetAnnotation(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseInt32(val)\n\tif !ok {\n\t\treturn 0, ErrorParseAnnotation(key, val, \"int32\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseInt64Label(obj kmeta.Object, key string) (int64, error) {\n\tval, err := GetLabel(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseInt64(val)\n\tif !ok {\n\t\treturn 0, ErrorParseLabel(key, val, \"int64\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseInt64Annotation(obj kmeta.Object, key string) (int64, error) {\n\tval, err := GetAnnotation(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseInt64(val)\n\tif !ok {\n\t\treturn 0, ErrorParseAnnotation(key, val, \"int64\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseFloat32Label(obj kmeta.Object, key string) (float32, error) {\n\tval, err := GetLabel(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseFloat32(val)\n\tif !ok {\n\t\treturn 0, ErrorParseLabel(key, val, \"float32\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseFloat32Annotation(obj kmeta.Object, key string) (float32, error) {\n\tval, err := GetAnnotation(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseFloat32(val)\n\tif !ok {\n\t\treturn 0, ErrorParseAnnotation(key, val, \"float32\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseFloat64Label(obj kmeta.Object, key string) (float64, error) {\n\tval, err := GetLabel(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseFloat64(val)\n\tif !ok {\n\t\treturn 0, ErrorParseLabel(key, val, \"float64\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseFloat64Annotation(obj kmeta.Object, key string) (float64, error) {\n\tval, err := GetAnnotation(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, ok := s.ParseFloat64(val)\n\tif !ok {\n\t\treturn 0, ErrorParseAnnotation(key, val, \"float64\")\n\t}\n\treturn casted, nil\n}\n\nfunc ParseDurationLabel(obj kmeta.Object, key string) (time.Duration, error) {\n\tval, err := GetLabel(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, err := time.ParseDuration(val)\n\tif err != nil {\n\t\treturn 0, ErrorParseLabel(key, val, \"duration\")\n\t}\n\treturn casted, nil\n}\nfunc ParseDurationAnnotation(obj kmeta.Object, key string) (time.Duration, error) {\n\tval, err := GetAnnotation(obj, key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tcasted, err := time.ParseDuration(val)\n\tif err != nil {\n\t\treturn 0, ErrorParseAnnotation(key, val, \"duration\")\n\t}\n\treturn casted, nil\n}\n"
  },
  {
    "path": "pkg/lib/k8s/pod.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"regexp\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n\tkscheme \"k8s.io/client-go/kubernetes/scheme\"\n\tkremotecommand \"k8s.io/client-go/tools/remotecommand\"\n)\n\nvar _podTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"v1\",\n\tKind:       \"Pod\",\n}\n\n// pod termination reasons\n// https://github.com/kubernetes/kube-state-metrics/blob/master/docs/pod-metrics.md\nconst (\n\tReasonEvicted   = \"Evicted\"\n\tReasonOOMKilled = \"OOMKilled\"\n\tReasonCompleted = \"Completed\"\n)\n\ntype PodSpec struct {\n\tName        string\n\tK8sPodSpec  kcore.PodSpec\n\tLabels      map[string]string\n\tAnnotations map[string]string\n}\n\ntype PodStatus string\n\nconst (\n\tPodStatusPending      PodStatus = \"Pending\"\n\tPodStatusCreating     PodStatus = \"Creating\"\n\tPodStatusNotReady     PodStatus = \"NotReady\"\n\tPodStatusReady        PodStatus = \"Ready\"\n\tPodStatusErrImagePull PodStatus = \"ErrImagePull\"\n\tPodStatusTerminating  PodStatus = \"Terminating\"\n\tPodStatusFailed       PodStatus = \"Failed\"\n\tPodStatusKilled       PodStatus = \"Killed\"\n\tPodStatusKilledOOM    PodStatus = \"KilledOOM\"\n\tPodStatusStalled      PodStatus = \"Stalled\"\n\tPodStatusSucceeded    PodStatus = \"Succeeded\"\n\tPodStatusUnknown      PodStatus = \"Unknown\"\n)\n\nvar (\n\t_killStatuses = map[int32]bool{\n\t\t137: true, // SIGKILL\n\t\t143: true, // SIGTERM\n\t\t130: true, // SIGINT\n\t\t129: true, // SIGHUP\n\t}\n\n\t_evictedMemoryMessageRegex = regexp.MustCompile(`(?i)low\\W+on\\W+resource\\W+memory`)\n\n\t// https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/images/types.go#L27\n\t_imagePullErrorStrings = strset.New(\"ErrImagePull\", \"ImagePullBackOff\", \"RegistryUnavailable\")\n\n\t// https://github.com/kubernetes/kubernetes/blob/9f47110aa29094ed2878cf1d85874cb59214664a/staging/src/k8s.io/api/core/v1/types.go#L76-L77\n\t_creatingReasons = strset.New(\"ContainerCreating\", \"PodInitializing\")\n\n\t_waitForCreatingPodTimeout = time.Minute * 15\n)\n\nfunc Pod(spec *PodSpec) *kcore.Pod {\n\tpod := &kcore.Pod{\n\t\tTypeMeta: _podTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.Name,\n\t\t\tLabels:      spec.Labels,\n\t\t\tAnnotations: spec.Annotations,\n\t\t},\n\t\tSpec: spec.K8sPodSpec,\n\t}\n\treturn pod\n}\n\nfunc GetPodConditionOf(pod *kcore.Pod, podType kcore.PodConditionType) (*bool, *kcore.PodCondition) {\n\tif pod == nil {\n\t\treturn nil, nil\n\t}\n\n\tvar conditionState *bool\n\tvar condition *kcore.PodCondition\n\tfor i := range pod.Status.Conditions {\n\t\tif pod.Status.Conditions[i].Type == podType {\n\t\t\tif pod.Status.Conditions[i].Status == kcore.ConditionTrue {\n\t\t\t\tconditionState = pointer.Bool(true)\n\t\t\t}\n\t\t\tif pod.Status.Conditions[i].Status == kcore.ConditionFalse {\n\t\t\t\tconditionState = pointer.Bool(false)\n\t\t\t}\n\t\t\tcondition = &pod.Status.Conditions[i]\n\t\t\tbreak\n\t\t}\n\t}\n\treturn conditionState, condition\n}\n\nfunc (c *Client) CreatePod(pod *kcore.Pod) (*kcore.Pod, error) {\n\tpod.TypeMeta = _podTypeMeta\n\tpod, err := c.podClient.Create(context.Background(), pod, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn pod, nil\n}\n\nfunc (c *Client) UpdatePod(pod *kcore.Pod) (*kcore.Pod, error) {\n\tpod.TypeMeta = _podTypeMeta\n\tpod, err := c.podClient.Update(context.Background(), pod, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn pod, nil\n}\n\nfunc (c *Client) ApplyPod(pod *kcore.Pod) (*kcore.Pod, error) {\n\texisting, err := c.GetPod(pod.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreatePod(pod)\n\t}\n\treturn c.UpdatePod(pod)\n}\n\nfunc IsPodReady(pod *kcore.Pod) bool {\n\tif GetPodStatus(pod) != PodStatusReady {\n\t\treturn false\n\t}\n\n\tpodConditionState, _ := GetPodConditionOf(pod, kcore.PodReady)\n\tif podConditionState != nil && *podConditionState {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\nfunc IsPodStalled(pod *kcore.Pod) bool {\n\tif GetPodStatus(pod) != PodStatusPending {\n\t\treturn false\n\t}\n\n\tpodConditionState, podCondition := GetPodConditionOf(pod, kcore.PodScheduled)\n\tif podConditionState != nil && !*podConditionState && !podCondition.LastTransitionTime.Time.IsZero() && time.Since(podCondition.LastTransitionTime.Time) >= _waitForCreatingPodTimeout {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\nfunc GetPodReadyTime(pod *kcore.Pod) *time.Time {\n\tfor i := range pod.Status.Conditions {\n\t\tcondition := pod.Status.Conditions[i]\n\n\t\tif condition.Type == kcore.PodReady && condition.Status == kcore.ConditionTrue {\n\t\t\tif condition.LastTransitionTime.Time.IsZero() {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn &condition.LastTransitionTime.Time\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc WasPodOOMKilled(pod *kcore.Pod) bool {\n\tif pod.Status.Reason == ReasonEvicted && _evictedMemoryMessageRegex.MatchString(pod.Status.Message) {\n\t\treturn true\n\t}\n\tfor _, containerStatus := range pod.Status.ContainerStatuses {\n\t\tvar reason string\n\t\tif containerStatus.LastTerminationState.Terminated != nil {\n\t\t\treason = containerStatus.LastTerminationState.Terminated.Reason\n\t\t} else if containerStatus.State.Terminated != nil {\n\t\t\treason = containerStatus.State.Terminated.Reason\n\t\t}\n\t\tif reason == ReasonOOMKilled {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc GetPodStatus(pod *kcore.Pod) PodStatus {\n\tif pod == nil {\n\t\treturn PodStatusUnknown\n\t}\n\n\tswitch pod.Status.Phase {\n\tcase kcore.PodPending:\n\t\tpodConditionState, podCondition := GetPodConditionOf(pod, kcore.PodScheduled)\n\t\tif podConditionState != nil && !*podConditionState && !podCondition.LastTransitionTime.Time.IsZero() && time.Since(podCondition.LastTransitionTime.Time) >= _waitForCreatingPodTimeout {\n\t\t\treturn PodStatusStalled\n\t\t}\n\t\treturn PodStatusFromContainerStatuses(append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...))\n\tcase kcore.PodSucceeded:\n\t\treturn PodStatusSucceeded\n\tcase kcore.PodFailed:\n\t\tif pod.Status.Reason == ReasonEvicted && _evictedMemoryMessageRegex.MatchString(pod.Status.Message) {\n\t\t\treturn PodStatusKilledOOM\n\t\t}\n\n\t\tfor _, containerStatus := range pod.Status.ContainerStatuses {\n\t\t\tvar reason string\n\t\t\tvar exitCode int32\n\t\t\tif containerStatus.LastTerminationState.Terminated != nil {\n\t\t\t\treason = containerStatus.LastTerminationState.Terminated.Reason\n\t\t\t\texitCode = containerStatus.LastTerminationState.Terminated.ExitCode\n\t\t\t} else if containerStatus.State.Terminated != nil {\n\t\t\t\treason = containerStatus.State.Terminated.Reason\n\t\t\t\texitCode = containerStatus.State.Terminated.ExitCode\n\t\t\t}\n\t\t\tif reason == ReasonOOMKilled {\n\t\t\t\treturn PodStatusKilledOOM\n\t\t\t} else if _killStatuses[exitCode] {\n\t\t\t\treturn PodStatusKilled\n\t\t\t}\n\n\t\t}\n\t\treturn PodStatusFailed\n\tcase kcore.PodRunning:\n\t\tif pod.ObjectMeta.DeletionTimestamp != nil {\n\t\t\treturn PodStatusTerminating\n\t\t}\n\n\t\tpodConditionState, _ := GetPodConditionOf(pod, kcore.PodReady)\n\t\tif podConditionState != nil && *podConditionState {\n\t\t\treturn PodStatusReady\n\t\t}\n\n\t\tstatus := PodStatusFromContainerStatuses(pod.Status.ContainerStatuses)\n\t\tif status == PodStatusReady {\n\t\t\treturn PodStatusNotReady\n\t\t}\n\n\t\treturn status\n\tdefault:\n\t\treturn PodStatusUnknown\n\t}\n}\n\nfunc PodStatusFromContainerStatuses(containerStatuses []kcore.ContainerStatus) PodStatus {\n\tnumContainers := len(containerStatuses)\n\tnumWaiting := 0\n\tnumCreating := 0\n\tnumNotReady := 0\n\tnumReady := 0\n\tnumSucceeded := 0\n\tnumFailed := 0\n\tnumKilled := 0\n\tnumKilledOOM := 0\n\n\tif len(containerStatuses) == 0 {\n\t\treturn PodStatusPending\n\t}\n\tfor _, containerStatus := range containerStatuses {\n\t\tif containerStatus.State.Running != nil && containerStatus.Ready {\n\t\t\tnumReady++\n\t\t} else if containerStatus.State.Running != nil && !containerStatus.Ready {\n\t\t\tnumNotReady++\n\t\t} else if containerStatus.State.Terminated != nil {\n\t\t\texitCode := containerStatus.State.Terminated.ExitCode\n\t\t\treason := containerStatus.State.Terminated.Reason\n\t\t\tif reason == ReasonOOMKilled {\n\t\t\t\tnumKilledOOM++\n\t\t\t} else if exitCode == 0 {\n\t\t\t\tnumSucceeded++\n\t\t\t} else if _killStatuses[exitCode] {\n\t\t\t\tnumKilled++\n\t\t\t} else {\n\t\t\t\tnumFailed++\n\t\t\t}\n\t\t} else if containerStatus.LastTerminationState.Terminated != nil {\n\t\t\texitCode := containerStatus.LastTerminationState.Terminated.ExitCode\n\t\t\treason := containerStatus.LastTerminationState.Terminated.Reason\n\t\t\tif reason == ReasonOOMKilled {\n\t\t\t\tnumKilledOOM++\n\t\t\t} else if exitCode == 0 {\n\t\t\t\tnumSucceeded++\n\t\t\t} else if _killStatuses[exitCode] {\n\t\t\t\tnumKilled++\n\t\t\t} else {\n\t\t\t\tnumFailed++\n\t\t\t}\n\t\t} else if containerStatus.State.Waiting != nil && _imagePullErrorStrings.Has(containerStatus.State.Waiting.Reason) {\n\t\t\treturn PodStatusErrImagePull\n\t\t} else if containerStatus.State.Waiting != nil && _creatingReasons.Has(containerStatus.State.Waiting.Reason) {\n\t\t\tnumCreating++\n\t\t} else {\n\t\t\t// either containerStatus.State.Waiting != nil or all containerStatus.States are nil (which implies waiting)\n\t\t\tnumWaiting++\n\t\t}\n\t}\n\tif numKilledOOM > 0 {\n\t\treturn PodStatusKilledOOM\n\t} else if numKilled > 0 {\n\t\treturn PodStatusKilled\n\t} else if numFailed > 0 {\n\t\treturn PodStatusFailed\n\t} else if numWaiting > 0 {\n\t\treturn PodStatusPending\n\t} else if numSucceeded == numContainers {\n\t\treturn PodStatusSucceeded\n\t} else if numCreating > 0 {\n\t\treturn PodStatusCreating\n\t} else if numNotReady > 0 {\n\t\treturn PodStatusNotReady\n\t} else {\n\t\treturn PodStatusReady\n\t}\n}\n\nfunc (c *Client) WaitForPodRunning(name string, numSeconds int) error {\n\tfor true {\n\t\tpod, err := c.GetPod(name)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif pod != nil && pod.Status.Phase == kcore.PodRunning {\n\t\t\treturn nil\n\t\t}\n\t\ttime.Sleep(time.Duration(numSeconds) * time.Second)\n\t}\n\treturn nil\n}\n\nfunc (c *Client) GetPod(name string) (*kcore.Pod, error) {\n\tpod, err := c.podClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tpod.TypeMeta = _podTypeMeta\n\treturn pod, nil\n}\n\nfunc (c *Client) DeletePod(name string) (bool, error) {\n\terr := c.podClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) ListPods(opts *kmeta.ListOptions) ([]kcore.Pod, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tpodList, err := c.podClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range podList.Items {\n\t\tpodList.Items[i].TypeMeta = _podTypeMeta\n\t}\n\treturn podList.Items, nil\n}\n\nfunc (c *Client) ListPodsByLabels(labels map[string]string) ([]kcore.Pod, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListPods(opts)\n}\n\nfunc (c *Client) ListPodsByLabel(labelKey string, labelValue string) ([]kcore.Pod, error) {\n\treturn c.ListPodsByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListPodsWithLabelKeys(labelKeys ...string) ([]kcore.Pod, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListPods(opts)\n}\n\nfunc PodMap(pods []kcore.Pod) map[string]kcore.Pod {\n\tpodMap := map[string]kcore.Pod{}\n\tfor _, pod := range pods {\n\t\tpodMap[pod.Name] = pod\n\t}\n\treturn podMap\n}\n\nfunc PodComputesEqual(podSpec1, podSpec2 *kcore.PodSpec) bool {\n\tcpu1, mem1, gpu1, inf1 := TotalPodCompute(podSpec1)\n\tcpu2, mem2, gpu2, inf2 := TotalPodCompute(podSpec2)\n\treturn cpu1.Equal(cpu2) && mem1.Equal(mem2) && gpu1 == gpu2 && inf1 == inf2\n}\n\nfunc TotalPodCompute(podSpec *kcore.PodSpec) (Quantity, Quantity, int64, int64) {\n\ttotalCPU := Quantity{}\n\ttotalMem := Quantity{}\n\tvar totalGPU, totalInf int64\n\n\tif podSpec == nil {\n\t\treturn totalCPU, totalMem, totalGPU, totalInf\n\t}\n\n\tfor _, container := range podSpec.Containers {\n\t\trequests := container.Resources.Requests\n\t\tif len(requests) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\ttotalCPU.Add(requests[kcore.ResourceCPU])\n\t\ttotalMem.Add(requests[kcore.ResourceMemory])\n\t\tif gpu, ok := requests[\"nvidia.com/gpu\"]; ok {\n\t\t\ttotalGPU += gpu.Value()\n\t\t}\n\t\tif inf, ok := requests[\"aws.amazon.com/neuron\"]; ok {\n\t\t\ttotalInf += inf.Value()\n\t\t}\n\t}\n\n\treturn totalCPU, totalMem, totalGPU, totalInf\n}\n\n// Example of running a shell command: []string{\"/bin/bash\", \"-c\", \"ps aux | grep my-proc\"}\nfunc (c *Client) Exec(podName string, containerName string, command []string) (string, error) {\n\toptions := &kcore.PodExecOptions{\n\t\tContainer: containerName,\n\t\tCommand:   command,\n\t\tStdin:     false,\n\t\tStdout:    true,\n\t\tStderr:    true,\n\t\tTTY:       true,\n\t}\n\n\treq := c.clientSet.CoreV1().RESTClient().Post().Namespace(c.Namespace).Resource(\"pods\").Name(podName).SubResource(\"exec\")\n\treq.VersionedParams(options, kscheme.ParameterCodec)\n\n\texec, err := kremotecommand.NewSPDYExecutor(c.RestConfig, \"POST\", req.URL())\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tbuf := &bytes.Buffer{}\n\n\terr = exec.Stream(kremotecommand.StreamOptions{\n\t\tStdin:  nil,\n\t\tStdout: buf,\n\t\tStderr: nil, // TTY merges stdout and stderr\n\t\tTty:    true,\n\t})\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn buf.String(), nil\n}\n"
  },
  {
    "path": "pkg/lib/k8s/quantity.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n)\n\ntype Quantity struct {\n\tkresource.Quantity\n\tUserString string\n}\n\ntype QuantityValidation struct {\n\tGreaterThan          *kresource.Quantity\n\tGreaterThanOrEqualTo *kresource.Quantity\n\tLessThan             *kresource.Quantity\n\tLessThanOrEqualTo    *kresource.Quantity\n}\n\nfunc QuantityParser(v *QuantityValidation) func(string) (interface{}, error) {\n\treturn func(str string) (interface{}, error) {\n\t\tk8sQuantity, err := kresource.ParseQuantity(str)\n\t\tif err != nil {\n\t\t\treturn Quantity{}, ErrorParseQuantity(str)\n\t\t}\n\n\t\tif v.GreaterThan != nil {\n\t\t\tif k8sQuantity.Cmp(*v.GreaterThan) <= 0 {\n\t\t\t\treturn nil, configreader.ErrorMustBeGreaterThan(str, *v.GreaterThan)\n\t\t\t}\n\t\t}\n\t\tif v.GreaterThanOrEqualTo != nil {\n\t\t\tif k8sQuantity.Cmp(*v.GreaterThanOrEqualTo) < 0 {\n\t\t\t\treturn nil, configreader.ErrorMustBeGreaterThanOrEqualTo(str, *v.GreaterThanOrEqualTo)\n\t\t\t}\n\t\t}\n\t\tif v.LessThan != nil {\n\t\t\tif k8sQuantity.Cmp(*v.LessThan) >= 0 {\n\t\t\t\treturn nil, configreader.ErrorMustBeLessThan(str, *v.LessThan)\n\t\t\t}\n\t\t}\n\t\tif v.LessThanOrEqualTo != nil {\n\t\t\tif k8sQuantity.Cmp(*v.LessThanOrEqualTo) > 0 {\n\t\t\t\treturn nil, configreader.ErrorMustBeLessThanOrEqualTo(str, *v.LessThanOrEqualTo)\n\t\t\t}\n\t\t}\n\n\t\treturn Quantity{\n\t\t\tQuantity:   k8sQuantity,\n\t\t\tUserString: str,\n\t\t}, nil\n\t}\n}\n\nfunc WrapQuantity(k8sQuantity kresource.Quantity) *Quantity {\n\treturn &Quantity{\n\t\tQuantity: k8sQuantity,\n\t}\n}\n\nfunc NewQuantity(value int64) *Quantity {\n\tk8sQuantity := kresource.NewQuantity(value, kresource.DecimalSI)\n\n\treturn &Quantity{\n\t\tQuantity: *k8sQuantity,\n\t}\n}\n\nfunc NewMilliQuantity(milliValue int64) *Quantity {\n\tk8sQuantity := kresource.NewMilliQuantity(milliValue, kresource.DecimalSI)\n\n\treturn &Quantity{\n\t\tQuantity:   *k8sQuantity,\n\t\tUserString: s.Int64(milliValue) + \"m\",\n\t}\n}\n\n// Returns nil if no quantities are passed in\nfunc NewSummed(quantities ...kresource.Quantity) *Quantity {\n\tif len(quantities) == 0 {\n\t\treturn nil\n\t}\n\n\tk8sQuantity := kresource.Quantity{}\n\tfor _, q := range quantities {\n\t\tk8sQuantity.Add(q)\n\t}\n\n\treturn &Quantity{\n\t\tQuantity: k8sQuantity,\n\t}\n}\n\nfunc (quantity *Quantity) MilliString() string {\n\treturn s.Int64(quantity.Quantity.MilliValue()) + \"m\"\n}\n\nfunc (quantity *Quantity) ToFloat32() float32 {\n\treturn float32(quantity.Quantity.MilliValue()) / float32(1000)\n}\n\nfunc ToKiRounded(k8sQuantity kresource.Quantity) int64 {\n\tkiFloat := float64(k8sQuantity.Value()) / float64(1024)\n\treturn int64(math.Round(kiFloat))\n}\nfunc ToKiRoundedStr(k8sQuantity kresource.Quantity) string {\n\treturn s.Int64(ToKiRounded(k8sQuantity)) + \"Ki\"\n}\nfunc (quantity *Quantity) ToKiRounded() int64 {\n\treturn ToKiRounded(quantity.Quantity)\n}\nfunc (quantity *Quantity) ToKiRoundedStr() string {\n\treturn ToKiRoundedStr(quantity.Quantity)\n}\n\nfunc ToKiCeil(k8sQuantity kresource.Quantity) int64 {\n\tkiFloat := float64(k8sQuantity.Value()) / float64(1024)\n\treturn int64(math.Ceil(kiFloat))\n}\nfunc ToKiCeilStr(k8sQuantity kresource.Quantity) string {\n\treturn s.Int64(ToKiCeil(k8sQuantity)) + \"Ki\"\n}\nfunc (quantity *Quantity) ToKiCeil() int64 {\n\treturn ToKiCeil(quantity.Quantity)\n}\nfunc (quantity *Quantity) ToKiCeilStr() string {\n\treturn ToKiCeilStr(quantity.Quantity)\n}\n\nfunc ToKiFloor(k8sQuantity kresource.Quantity) int64 {\n\tkiFloat := float64(k8sQuantity.Value()) / float64(1024)\n\treturn int64(math.Floor(kiFloat))\n}\nfunc ToKiFloorStr(k8sQuantity kresource.Quantity) string {\n\treturn s.Int64(ToKiFloor(k8sQuantity)) + \"Ki\"\n}\nfunc (quantity *Quantity) ToKiFloor() int64 {\n\treturn ToKiFloor(quantity.Quantity)\n}\nfunc (quantity *Quantity) ToKiFloorStr() string {\n\treturn ToKiFloorStr(quantity.Quantity)\n}\n\nfunc ToMiRounded(k8sQuantity kresource.Quantity) int64 {\n\tmiFloat := float64(k8sQuantity.Value()) / float64(1024*1024)\n\treturn int64(math.Round(miFloat))\n}\nfunc ToMiRoundedStr(k8sQuantity kresource.Quantity) string {\n\treturn s.Int64(ToMiRounded(k8sQuantity)) + \"Mi\"\n}\nfunc (quantity *Quantity) ToMiRounded() int64 {\n\treturn ToMiRounded(quantity.Quantity)\n}\nfunc (quantity *Quantity) ToMiRoundedStr() string {\n\treturn ToMiRoundedStr(quantity.Quantity)\n}\n\nfunc ToMiCeil(k8sQuantity kresource.Quantity) int64 {\n\tmiFloat := float64(k8sQuantity.Value()) / float64(1024*1024)\n\treturn int64(math.Ceil(miFloat))\n}\nfunc ToMiCeilStr(k8sQuantity kresource.Quantity) string {\n\treturn s.Int64(ToMiCeil(k8sQuantity)) + \"Mi\"\n}\nfunc (quantity *Quantity) ToMiCeil() int64 {\n\treturn ToMiCeil(quantity.Quantity)\n}\nfunc (quantity *Quantity) ToMiCeilStr() string {\n\treturn ToMiCeilStr(quantity.Quantity)\n}\n\nfunc ToMiFloor(k8sQuantity kresource.Quantity) int64 {\n\tmiFloat := float64(k8sQuantity.Value()) / float64(1024*1024)\n\treturn int64(math.Floor(miFloat))\n}\nfunc ToMiFloorStr(k8sQuantity kresource.Quantity) string {\n\treturn s.Int64(ToMiFloor(k8sQuantity)) + \"Mi\"\n}\nfunc (quantity *Quantity) ToMiFloor() int64 {\n\treturn ToMiFloor(quantity.Quantity)\n}\nfunc (quantity *Quantity) ToMiFloorStr() string {\n\treturn ToMiFloorStr(quantity.Quantity)\n}\n\nfunc (quantity *Quantity) Sub(q2 kresource.Quantity) {\n\tquantity.Quantity.Sub(q2)\n\tquantity.UserString = \"\"\n}\n\nfunc (quantity *Quantity) SubQty(q2 Quantity) {\n\tquantity.Quantity.Sub(q2.Quantity)\n\tquantity.UserString = \"\"\n}\n\nfunc (quantity *Quantity) Add(q2 kresource.Quantity) {\n\tquantity.Quantity.Add(q2)\n\tquantity.UserString = \"\"\n}\n\nfunc (quantity *Quantity) AddQty(q2 Quantity) {\n\tquantity.Quantity.Add(q2.Quantity)\n\tquantity.UserString = \"\"\n}\n\nfunc (quantity *Quantity) String() string {\n\tif quantity == nil {\n\t\treturn \"<nil>\"\n\t}\n\tif quantity.UserString != \"\" {\n\t\treturn quantity.UserString\n\t}\n\treturn quantity.Quantity.String()\n}\n\nfunc (quantity *Quantity) Equal(quantity2 Quantity) bool {\n\treturn quantity.Quantity.Cmp(quantity2.Quantity) == 0\n}\n\nfunc (quantity *Quantity) ID() string {\n\treturn s.Int64(quantity.MilliValue())\n}\n\nfunc (quantity *Quantity) DeepCopy() Quantity {\n\treturn Quantity{\n\t\tQuantity:   quantity.Quantity.DeepCopy(),\n\t\tUserString: quantity.UserString,\n\t}\n}\n\nfunc QuantityPtr(k8sQuantity kresource.Quantity) *kresource.Quantity {\n\treturn &k8sQuantity\n}\n\nfunc QuantityPtrID(quantity *Quantity) string {\n\tif quantity == nil {\n\t\treturn \"nil\"\n\t}\n\treturn quantity.ID()\n}\n\nfunc QuantityPtrsEqual(quantity *Quantity, quantity2 *Quantity) bool {\n\tif quantity == nil && quantity2 == nil {\n\t\treturn true\n\t}\n\tif quantity == nil || quantity2 == nil {\n\t\treturn false\n\t}\n\treturn quantity.Equal(*quantity2)\n}\n\ntype quantityMarshalable struct {\n\tQuantity   kresource.Quantity\n\tUserString string\n}\n\nfunc (quantity Quantity) MarshalYAML() (interface{}, error) {\n\treturn quantity.String(), nil\n}\n\nfunc (quantity *Quantity) UnmarshalYAML(unmarshal func(interface{}) error) error {\n\tvar userString string\n\terr := unmarshal(&userString)\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = quantity.UnmarshalJSON([]byte(userString))\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (quantity Quantity) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(quantity.String())\n}\n\nfunc (quantity *Quantity) UnmarshalJSON(data []byte) error {\n\tvar userString string\n\terr := json.Unmarshal(data, &userString)\n\tquantity.UserString = userString\n\n\tparsedQuantity, err := kresource.ParseQuantity(userString)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tquantity.Quantity = parsedQuantity\n\tquantity.UserString = userString\n\treturn nil\n}\n\nfunc (quantity Quantity) MarshalBinary() ([]byte, error) {\n\treturn json.Marshal(quantity)\n}\n\nfunc (quantity *Quantity) UnmarshalBinary(data []byte) error {\n\treturn json.Unmarshal(data, quantity)\n}\n\nfunc (quantity Quantity) MarshalText() ([]byte, error) {\n\treturn json.Marshal(quantity)\n}\n\nfunc (quantity *Quantity) UnmarshalText(data []byte) error {\n\treturn json.Unmarshal(data, quantity)\n}\n"
  },
  {
    "path": "pkg/lib/k8s/secret.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nvar _secretTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"v1\",\n\tKind:       \"Secret\",\n}\n\ntype SecretSpec struct {\n\tName        string\n\tData        map[string][]byte\n\tLabels      map[string]string\n\tAnnotations map[string]string\n}\n\nfunc Secret(spec *SecretSpec) *kcore.Secret {\n\tsecret := &kcore.Secret{\n\t\tTypeMeta: _secretTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.Name,\n\t\t\tLabels:      spec.Labels,\n\t\t\tAnnotations: spec.Annotations,\n\t\t},\n\t\tData: spec.Data,\n\t}\n\treturn secret\n}\n\nfunc (c *Client) CreateSecret(secret *kcore.Secret) (*kcore.Secret, error) {\n\tsecret.TypeMeta = _secretTypeMeta\n\tsecret, err := c.secretClient.Create(context.Background(), secret, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn secret, nil\n}\n\nfunc (c *Client) UpdateSecret(secret *kcore.Secret) (*kcore.Secret, error) {\n\tsecret.TypeMeta = _secretTypeMeta\n\tsecret, err := c.secretClient.Update(context.Background(), secret, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn secret, nil\n}\n\nfunc (c *Client) ApplySecret(secret *kcore.Secret) (*kcore.Secret, error) {\n\texisting, err := c.GetSecret(secret.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreateSecret(secret)\n\t}\n\treturn c.UpdateSecret(secret)\n}\n\nfunc (c *Client) GetSecret(name string) (*kcore.Secret, error) {\n\tsecret, err := c.secretClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tsecret.TypeMeta = _secretTypeMeta\n\treturn secret, nil\n}\n\nfunc (c *Client) GetSecretData(name string) (map[string][]byte, error) {\n\tsecret, err := c.GetSecret(name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif secret == nil {\n\t\treturn nil, nil\n\t}\n\treturn secret.Data, nil\n}\n\nfunc (c *Client) DeleteSecret(name string) (bool, error) {\n\terr := c.secretClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) ListSecrets(opts *kmeta.ListOptions) ([]kcore.Secret, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tsecretList, err := c.secretClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range secretList.Items {\n\t\tsecretList.Items[i].TypeMeta = _secretTypeMeta\n\t}\n\treturn secretList.Items, nil\n}\n\nfunc (c *Client) ListSecretsByLabels(labels map[string]string) ([]kcore.Secret, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListSecrets(opts)\n}\n\nfunc (c *Client) ListSecretsByLabel(labelKey string, labelValue string) ([]kcore.Secret, error) {\n\treturn c.ListSecretsByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListSecretsWithLabelKeys(labelKeys ...string) ([]kcore.Secret, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListSecrets(opts)\n}\n\nfunc SecretMap(secrets []kcore.Secret) map[string]kcore.Secret {\n\tsecretMap := map[string]kcore.Secret{}\n\tfor _, secret := range secrets {\n\t\tsecretMap[secret.Name] = secret\n\t}\n\treturn secretMap\n}\n"
  },
  {
    "path": "pkg/lib/k8s/service.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nvar _serviceTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"v1\",\n\tKind:       \"Service\",\n}\n\ntype ServiceSpec struct {\n\tName        string\n\tPortName    string\n\tPort        int32\n\tTargetPort  int32\n\tServiceType kcore.ServiceType\n\tSelector    map[string]string\n\tLabels      map[string]string\n\tAnnotations map[string]string\n}\n\nfunc Service(spec *ServiceSpec) *kcore.Service {\n\tservice := &kcore.Service{\n\t\tTypeMeta: _serviceTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.Name,\n\t\t\tLabels:      spec.Labels,\n\t\t\tAnnotations: spec.Annotations,\n\t\t},\n\t\tSpec: kcore.ServiceSpec{\n\t\t\tSelector: spec.Selector,\n\t\t\tType:     spec.ServiceType,\n\t\t\tPorts: []kcore.ServicePort{\n\t\t\t\t{\n\t\t\t\t\tProtocol: kcore.ProtocolTCP,\n\t\t\t\t\tName:     spec.PortName,\n\t\t\t\t\tPort:     spec.Port,\n\t\t\t\t\tTargetPort: intstr.IntOrString{\n\t\t\t\t\t\tIntVal: spec.TargetPort,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\treturn service\n}\n\nfunc (c *Client) CreateService(service *kcore.Service) (*kcore.Service, error) {\n\tservice.TypeMeta = _serviceTypeMeta\n\tservice, err := c.serviceClient.Create(context.Background(), service, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn service, nil\n}\n\nfunc (c *Client) UpdateService(existing, updated *kcore.Service) (*kcore.Service, error) {\n\tupdated.TypeMeta = _serviceTypeMeta\n\tupdated.Spec.ClusterIP = existing.Spec.ClusterIP\n\tupdated.ResourceVersion = existing.ResourceVersion\n\n\tservice, err := c.serviceClient.Update(context.Background(), updated, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn service, nil\n}\n\nfunc (c *Client) ApplyService(service *kcore.Service) (*kcore.Service, error) {\n\texisting, err := c.GetService(service.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreateService(service)\n\t}\n\treturn c.UpdateService(existing, service)\n}\n\nfunc (c *Client) GetService(name string) (*kcore.Service, error) {\n\tservice, err := c.serviceClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tservice.TypeMeta = _serviceTypeMeta\n\treturn service, nil\n}\n\nfunc (c *Client) DeleteService(name string) (bool, error) {\n\terr := c.serviceClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) ListServices(opts *kmeta.ListOptions) ([]kcore.Service, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tserviceList, err := c.serviceClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range serviceList.Items {\n\t\tserviceList.Items[i].TypeMeta = _serviceTypeMeta\n\t}\n\treturn serviceList.Items, nil\n}\n\nfunc (c *Client) ListServicesByLabels(labels map[string]string) ([]kcore.Service, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListServices(opts)\n}\n\nfunc (c *Client) ListServicesByLabel(labelKey string, labelValue string) ([]kcore.Service, error) {\n\treturn c.ListServicesByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListServicesWithLabelKeys(labelKeys ...string) ([]kcore.Service, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListServices(opts)\n}\n\nfunc (c *Client) InternalServiceEndpoint(serviceName string, portNumber int32) string {\n\treturn fmt.Sprintf(\"http://%s.%s.svc.cluster.local:%d\", serviceName, c.Namespace, portNumber)\n}\n\nfunc ServiceMap(services []kcore.Service) map[string]kcore.Service {\n\tserviceMap := map[string]kcore.Service{}\n\tfor _, service := range services {\n\t\tserviceMap[service.Name] = service\n\t}\n\treturn serviceMap\n}\n"
  },
  {
    "path": "pkg/lib/k8s/virtual_service.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\t\"context\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n\tistionetworking \"istio.io/api/networking/v1beta1\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tistionetworkingclient \"istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nvar _virtualServiceTypeMeta = kmeta.TypeMeta{\n\tAPIVersion: \"v1beta1\",\n\tKind:       \"VirtualService\",\n}\n\ntype VirtualServiceSpec struct {\n\tName         string\n\tGateways     []string\n\tExactPath    *string // either this or PrefixPath\n\tPrefixPath   *string // either this or ExactPath\n\tDestinations []Destination\n\tRewrite      *string\n\tLabels       map[string]string\n\tAnnotations  map[string]string\n\tHeaders      *istionetworking.Headers\n\tRetries      *int32\n}\n\ntype Destination struct {\n\tServiceName string\n\tWeight      int32\n\tPort        uint32\n\tShadow      bool\n\tHeaders     *istionetworking.Headers\n}\n\nfunc VirtualService(spec *VirtualServiceSpec) *istioclientnetworking.VirtualService {\n\tdestinations := []*istionetworking.HTTPRouteDestination{}\n\tvar mirror *istionetworking.Destination\n\tvar mirrorWeight *istionetworking.Percent\n\n\tfor _, destination := range spec.Destinations {\n\t\tif destination.Shadow {\n\t\t\tmirror = &istionetworking.Destination{\n\t\t\t\tHost: destination.ServiceName,\n\t\t\t\tPort: &istionetworking.PortSelector{\n\t\t\t\t\tNumber: destination.Port,\n\t\t\t\t},\n\t\t\t}\n\t\t\tmirrorWeight = &istionetworking.Percent{Value: float64(destination.Weight)}\n\t\t} else {\n\t\t\tdestinations = append(destinations, &istionetworking.HTTPRouteDestination{\n\t\t\t\tDestination: &istionetworking.Destination{\n\t\t\t\t\tHost: destination.ServiceName,\n\t\t\t\t\tPort: &istionetworking.PortSelector{\n\t\t\t\t\t\tNumber: destination.Port,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tWeight:  destination.Weight,\n\t\t\t\tHeaders: destination.Headers,\n\t\t\t})\n\t\t}\n\t}\n\n\tvar httpRoutes []*istionetworking.HTTPRoute\n\n\tif spec.ExactPath != nil {\n\t\thttpRoutes = append(httpRoutes, &istionetworking.HTTPRoute{\n\t\t\tMatch: []*istionetworking.HTTPMatchRequest{\n\t\t\t\t{\n\t\t\t\t\tUri: &istionetworking.StringMatch{\n\t\t\t\t\t\tMatchType: &istionetworking.StringMatch_Exact{\n\t\t\t\t\t\t\tExact: urls.CanonicalizeEndpoint(*spec.ExactPath),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tRoute:            destinations,\n\t\t\tMirror:           mirror,\n\t\t\tMirrorPercentage: mirrorWeight,\n\t\t\tHeaders:          spec.Headers,\n\t\t})\n\n\t\tif spec.Rewrite != nil {\n\t\t\thttpRoutes[0].Rewrite = &istionetworking.HTTPRewrite{\n\t\t\t\tUri: urls.CanonicalizeEndpoint(*spec.Rewrite),\n\t\t\t}\n\t\t}\n\t} else {\n\t\texactMatch := &istionetworking.HTTPRoute{\n\t\t\tMatch: []*istionetworking.HTTPMatchRequest{\n\t\t\t\t{\n\t\t\t\t\tUri: &istionetworking.StringMatch{\n\t\t\t\t\t\tMatchType: &istionetworking.StringMatch_Exact{\n\t\t\t\t\t\t\tExact: urls.CanonicalizeEndpoint(*spec.PrefixPath),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tRoute:            destinations,\n\t\t\tMirror:           mirror,\n\t\t\tMirrorPercentage: mirrorWeight,\n\t\t\tHeaders:          spec.Headers,\n\t\t}\n\n\t\tprefixMatch := &istionetworking.HTTPRoute{\n\t\t\tMatch: []*istionetworking.HTTPMatchRequest{\n\t\t\t\t{\n\t\t\t\t\tUri: &istionetworking.StringMatch{\n\t\t\t\t\t\tMatchType: &istionetworking.StringMatch_Prefix{\n\t\t\t\t\t\t\tPrefix: urls.CanonicalizeEndpointWithTrailingSlash(*spec.PrefixPath),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tRoute:            destinations,\n\t\t\tMirror:           mirror,\n\t\t\tMirrorPercentage: mirrorWeight,\n\t\t\tHeaders:          spec.Headers,\n\t\t}\n\n\t\tif spec.Rewrite != nil {\n\t\t\texactMatch.Rewrite = &istionetworking.HTTPRewrite{\n\t\t\t\tUri: urls.CanonicalizeEndpoint(*spec.Rewrite),\n\t\t\t}\n\n\t\t\tprefixMatch.Rewrite = &istionetworking.HTTPRewrite{\n\t\t\t\tUri: urls.CanonicalizeEndpointWithTrailingSlash(*spec.Rewrite),\n\t\t\t}\n\t\t}\n\n\t\thttpRoutes = append(httpRoutes, exactMatch, prefixMatch)\n\t}\n\n\tif spec.Retries != nil {\n\t\tfor i := range httpRoutes {\n\t\t\thttpRoutes[i].Retries = &istionetworking.HTTPRetry{\n\t\t\t\tAttempts: *spec.Retries,\n\t\t\t}\n\t\t}\n\t}\n\n\tvirtualService := &istioclientnetworking.VirtualService{\n\t\tTypeMeta: _virtualServiceTypeMeta,\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:        spec.Name,\n\t\t\tLabels:      spec.Labels,\n\t\t\tAnnotations: spec.Annotations,\n\t\t},\n\t\tSpec: istionetworking.VirtualService{\n\t\t\tHosts:    []string{\"*\"},\n\t\t\tGateways: spec.Gateways,\n\t\t\tHttp:     httpRoutes,\n\t\t},\n\t}\n\n\treturn virtualService\n}\n\nfunc (c *Client) VirtualServiceClient() istionetworkingclient.VirtualServiceInterface {\n\treturn c.virtualServiceClient\n}\n\nfunc (c *Client) CreateVirtualService(virtualService *istioclientnetworking.VirtualService) (*istioclientnetworking.VirtualService, error) {\n\tvirtualService.TypeMeta = _virtualServiceTypeMeta\n\tvirtualService, err := c.virtualServiceClient.Create(context.Background(), virtualService, kmeta.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn virtualService, nil\n}\n\nfunc (c *Client) UpdateVirtualService(existing, updated *istioclientnetworking.VirtualService) (*istioclientnetworking.VirtualService, error) {\n\tupdated.TypeMeta = _virtualServiceTypeMeta\n\tupdated.ResourceVersion = existing.ResourceVersion\n\n\tvirtualService, err := c.virtualServiceClient.Update(context.Background(), updated, kmeta.UpdateOptions{})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn virtualService, nil\n}\n\nfunc (c *Client) ApplyVirtualService(virtualService *istioclientnetworking.VirtualService) (*istioclientnetworking.VirtualService, error) {\n\texisting, err := c.GetVirtualService(virtualService.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif existing == nil {\n\t\treturn c.CreateVirtualService(virtualService)\n\t}\n\treturn c.UpdateVirtualService(existing, virtualService)\n}\n\nfunc (c *Client) GetVirtualService(name string) (*istioclientnetworking.VirtualService, error) {\n\tvirtualService, err := c.virtualServiceClient.Get(context.Background(), name, kmeta.GetOptions{})\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvirtualService.TypeMeta = _virtualServiceTypeMeta\n\treturn virtualService, nil\n}\n\nfunc (c *Client) DeleteVirtualService(name string) (bool, error) {\n\terr := c.virtualServiceClient.Delete(context.Background(), name, _deleteOpts)\n\tif err != nil {\n\t\tif kerrors.IsNotFound(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, errors.WithStack(err)\n\t}\n\treturn true, nil\n}\n\nfunc (c *Client) ListVirtualServices(opts *kmeta.ListOptions) ([]istioclientnetworking.VirtualService, error) {\n\tif opts == nil {\n\t\topts = &kmeta.ListOptions{}\n\t}\n\tvsList, err := c.virtualServiceClient.List(context.Background(), *opts)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor i := range vsList.Items {\n\t\tvsList.Items[i].TypeMeta = _virtualServiceTypeMeta\n\t}\n\treturn vsList.Items, nil\n}\n\nfunc (c *Client) ListVirtualServicesByLabels(labels map[string]string) ([]istioclientnetworking.VirtualService, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(labels).String(),\n\t}\n\treturn c.ListVirtualServices(opts)\n}\n\nfunc (c *Client) ListVirtualServicesByLabel(labelKey string, labelValue string) ([]istioclientnetworking.VirtualService, error) {\n\treturn c.ListVirtualServicesByLabels(map[string]string{labelKey: labelValue})\n}\n\nfunc (c *Client) ListVirtualServicesWithLabelKeys(labelKeys ...string) ([]istioclientnetworking.VirtualService, error) {\n\topts := &kmeta.ListOptions{\n\t\tLabelSelector: LabelExistsSelector(labelKeys...),\n\t}\n\treturn c.ListVirtualServices(opts)\n}\n\nfunc ExtractVirtualServiceGateways(virtualService *istioclientnetworking.VirtualService) strset.Set {\n\treturn strset.FromSlice(virtualService.Spec.Gateways)\n}\n\nfunc ExtractVirtualServiceEndpoints(virtualService *istioclientnetworking.VirtualService) strset.Set {\n\tendpoints := strset.New()\n\tfor _, http := range virtualService.Spec.Http {\n\t\tfor _, match := range http.Match {\n\t\t\tif match.Uri.GetExact() != \"\" {\n\t\t\t\tendpoints.Add(urls.CanonicalizeEndpoint(match.Uri.GetExact()))\n\t\t\t}\n\n\t\t\tif match.Uri.GetPrefix() != \"\" {\n\t\t\t\tendpoints.Add(urls.CanonicalizeEndpoint(match.Uri.GetPrefix()))\n\t\t\t}\n\t\t}\n\t}\n\treturn endpoints\n}\n"
  },
  {
    "path": "pkg/lib/k8s/volume.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage k8s\n\nimport (\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nfunc EmptyDirVolume(volumeName string) kcore.Volume {\n\treturn kcore.Volume{\n\t\tName: volumeName,\n\t\tVolumeSource: kcore.VolumeSource{\n\t\t\tEmptyDir: &kcore.EmptyDirVolumeSource{},\n\t\t},\n\t}\n}\n\nfunc EmptyDirVolumeMount(volumeName string, mountPath string) kcore.VolumeMount {\n\treturn kcore.VolumeMount{\n\t\tName:      volumeName,\n\t\tMountPath: mountPath,\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/logging/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage logging\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrInvalidLogLevel = \"logging.invalid_log_level\"\n)\n\nfunc ErrorInvalidLogLevel(provided string, loglevels []string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidLogLevel,\n\t\tMessage: fmt.Sprintf(\"invalid log level %s; must be one of %s\", provided, s.StrsOr(loglevels)),\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/logging/logging.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage logging\n\nimport (\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"go.uber.org/zap\"\n)\n\nvar logger *zap.SugaredLogger\nvar loggerLock sync.Mutex\n\nfunc initializeLogger() {\n\tlogLevel := strings.ToLower(os.Getenv(\"CORTEX_LOG_LEVEL\"))\n\tif logLevel == \"\" {\n\t\tlogLevel = \"info\"\n\t}\n\n\tcortexLogLevel := userconfig.LogLevelFromString(logLevel)\n\tif cortexLogLevel == userconfig.UnknownLogLevel {\n\t\tpanic(ErrorInvalidLogLevel(logLevel, userconfig.LogLevelTypes()))\n\t}\n\n\tzapConfig := DefaultZapConfig(cortexLogLevel)\n\n\tdisableJSONLogging := strings.ToLower(os.Getenv(\"CORTEX_DISABLE_JSON_LOGGING\"))\n\tif disableJSONLogging == \"true\" {\n\t\tzapConfig.Encoding = \"console\"\n\t}\n\n\tzapLogger, err := zapConfig.Build()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlogger = zapLogger.Sugar()\n}\n\nfunc GetLogger() *zap.SugaredLogger {\n\tloggerLock.Lock()\n\tdefer loggerLock.Unlock()\n\n\tif logger == nil {\n\t\tinitializeLogger()\n\t}\n\treturn logger\n}\n\nfunc DefaultZapConfig(level userconfig.LogLevel, fields ...map[string]interface{}) zap.Config {\n\tencoderConfig := zap.NewProductionEncoderConfig()\n\tencoderConfig.MessageKey = \"message\"\n\n\tlabels := map[string]interface{}{}\n\tfor _, m := range fields {\n\t\tfor k, v := range m {\n\t\t\tlabels[k] = v\n\t\t}\n\t}\n\n\tinitialFields := map[string]interface{}{}\n\tif len(labels) > 0 {\n\t\tinitialFields[\"cortex.labels\"] = labels\n\t}\n\n\treturn zap.Config{\n\t\tLevel:            zap.NewAtomicLevelAt(userconfig.ToZapLogLevel(level)),\n\t\tEncoding:         \"json\",\n\t\tEncoderConfig:    encoderConfig,\n\t\tOutputPaths:      []string{\"stdout\"},\n\t\tErrorOutputPaths: []string{\"stderr\"},\n\t\tInitialFields:    initialFields,\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/maps/interface.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage maps\n\nimport (\n\t\"reflect\"\n\t\"sort\"\n)\n\nfunc InterfaceMapKeys(myMap map[string]interface{}) []string {\n\tkeys := make([]string, len(myMap))\n\ti := 0\n\tfor key := range myMap {\n\t\tkeys[i] = key\n\t\ti++\n\t}\n\treturn keys\n}\n\nfunc InterfaceMapSortedKeys(myMap map[string]interface{}) []string {\n\tkeys := InterfaceMapKeys(myMap)\n\tsort.Strings(keys)\n\treturn keys\n}\n\nfunc InterfaceMapKeysUnsafe(myMap interface{}) []string {\n\tkeyValues := reflect.ValueOf(myMap).MapKeys()\n\tkeys := make([]string, len(keyValues))\n\tfor i := range keyValues {\n\t\tkeys[i] = keyValues[i].String()\n\t}\n\treturn keys\n}\n\nfunc InterfaceMapsKeysMatch(map1 map[string]interface{}, map2 map[string]interface{}) bool {\n\tif len(map1) != len(map2) {\n\t\treturn false\n\t}\n\tfor key := range map1 {\n\t\tif _, ok := map2[key]; !ok {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc MergeStrInterfaceMaps(maps ...map[string]interface{}) map[string]interface{} {\n\tmerged := map[string]interface{}{}\n\tfor _, m := range maps {\n\t\tfor k, v := range m {\n\t\t\tmerged[k] = v\n\t\t}\n\t}\n\treturn merged\n}\n"
  },
  {
    "path": "pkg/lib/maps/string.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage maps\n\nfunc StrMapKeysString(myMap map[string]string) []string {\n\tkeys := make([]string, len(myMap))\n\ti := 0\n\tfor key := range myMap {\n\t\tkeys[i] = key\n\t\ti++\n\t}\n\treturn keys\n}\n\nfunc StrMapValuesString(myMap map[string]string) []string {\n\tvalues := make([]string, len(myMap))\n\ti := 0\n\tfor _, value := range myMap {\n\t\tvalues[i] = value\n\t\ti++\n\t}\n\treturn values\n}\n\nfunc MergeStrMapsString(maps ...map[string]string) map[string]string {\n\tmerged := map[string]string{}\n\tfor _, m := range maps {\n\t\tfor k, v := range m {\n\t\t\tmerged[k] = v\n\t\t}\n\t}\n\treturn merged\n}\n\nfunc StrMapsEqualString(m1, m2 map[string]string) bool {\n\tif len(m1) != len(m2) {\n\t\treturn false\n\t}\n\n\tif len(m1) == 0 && len(m2) == 0 {\n\t\treturn true\n\t}\n\n\tif len(m1) == 0 || len(m2) == 0 {\n\t\treturn false\n\t}\n\n\tfor k, v1 := range m1 {\n\t\tif v2, ok := m2[k]; !ok || v2 != v1 {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc StrMapKeysInt(myMap map[string]int) []string {\n\tkeys := make([]string, len(myMap))\n\ti := 0\n\tfor key := range myMap {\n\t\tkeys[i] = key\n\t\ti++\n\t}\n\treturn keys\n}\n\nfunc StrMapValuesInt(myMap map[string]int) []int {\n\tvalues := make([]int, len(myMap))\n\ti := 0\n\tfor _, value := range myMap {\n\t\tvalues[i] = value\n\t\ti++\n\t}\n\treturn values\n}\n\nfunc MergeStrMapsInt(maps ...map[string]int) map[string]int {\n\tmerged := map[string]int{}\n\tfor _, m := range maps {\n\t\tfor k, v := range m {\n\t\t\tmerged[k] = v\n\t\t}\n\t}\n\treturn merged\n}\n\nfunc StrMapsEqualInt(m1, m2 map[string]int) bool {\n\tif len(m1) != len(m2) {\n\t\treturn false\n\t}\n\n\tif len(m1) == 0 && len(m2) == 0 {\n\t\treturn true\n\t}\n\n\tif len(m1) == 0 || len(m2) == 0 {\n\t\treturn false\n\t}\n\n\tfor k, v1 := range m1 {\n\t\tif v2, ok := m2[k]; !ok || v2 != v1 {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n"
  },
  {
    "path": "pkg/lib/math/float32.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage math\n\nfunc MinFloat32(val float32, vals ...float32) float32 {\n\tmin := val\n\tfor _, v := range vals {\n\t\tif v < min {\n\t\t\tmin = v\n\t\t}\n\t}\n\treturn min\n}\n\nfunc MaxFloat32(val float32, vals ...float32) float32 {\n\tmax := val\n\tfor _, v := range vals {\n\t\tif v > max {\n\t\t\tmax = v\n\t\t}\n\t}\n\treturn max\n}\n"
  },
  {
    "path": "pkg/lib/math/float64.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage math\n\nfunc MinFloat64(val float64, vals ...float64) float64 {\n\tmin := val\n\tfor _, v := range vals {\n\t\tif v < min {\n\t\t\tmin = v\n\t\t}\n\t}\n\treturn min\n}\n\nfunc MaxFloat64(val float64, vals ...float64) float64 {\n\tmax := val\n\tfor _, v := range vals {\n\t\tif v > max {\n\t\t\tmax = v\n\t\t}\n\t}\n\treturn max\n}\n"
  },
  {
    "path": "pkg/lib/math/int.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage math\n\nimport (\n\t\"math\"\n\t\"sort\"\n)\n\nfunc MinInt(val int, vals ...int) int {\n\tmin := val\n\tfor _, v := range vals {\n\t\tif v < min {\n\t\t\tmin = v\n\t\t}\n\t}\n\treturn min\n}\n\nfunc MaxInt(val int, vals ...int) int {\n\tmax := val\n\tfor _, v := range vals {\n\t\tif v > max {\n\t\t\tmax = v\n\t\t}\n\t}\n\treturn max\n}\n\nfunc IsDivisibleByInt(num int, divisor int) bool {\n\treturn num%divisor == 0\n}\n\nfunc FactorsInt(num int) []int {\n\tdivisibleNumbers := []int{}\n\tmaxDivisor := int(math.Sqrt(float64(num)))\n\tincrementer := 1\n\n\t// Skip even numbers if num is odd\n\tif num%2 == 1 {\n\t\tincrementer = 2\n\t}\n\n\tfor divisor := 1; divisor <= maxDivisor; divisor += incrementer {\n\t\tif num%divisor == 0 {\n\t\t\tdivisibleNumbers = append(divisibleNumbers, divisor)\n\t\t\tcomplementaryDivisor := num / divisor\n\t\t\tif divisor != complementaryDivisor {\n\t\t\t\tdivisibleNumbers = append(divisibleNumbers, complementaryDivisor)\n\t\t\t}\n\t\t}\n\t}\n\n\tsort.Ints(divisibleNumbers)\n\treturn divisibleNumbers\n}\n"
  },
  {
    "path": "pkg/lib/math/int32.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage math\n\nimport (\n\t\"math\"\n\t\"sort\"\n)\n\nfunc MinInt32(val int32, vals ...int32) int32 {\n\tmin := val\n\tfor _, v := range vals {\n\t\tif v < min {\n\t\t\tmin = v\n\t\t}\n\t}\n\treturn min\n}\n\nfunc MaxInt32(val int32, vals ...int32) int32 {\n\tmax := val\n\tfor _, v := range vals {\n\t\tif v > max {\n\t\t\tmax = v\n\t\t}\n\t}\n\treturn max\n}\n\nfunc IsDivisibleByInt32(num int32, divisor int32) bool {\n\treturn num%divisor == 0\n}\n\nfunc FactorsInt32(num int32) []int32 {\n\tdivisibleNumbers := []int32{}\n\tmaxDivisor := int32(math.Sqrt(float64(num)))\n\tincrementer := int32(1)\n\n\t// Skip even numbers if num is odd\n\tif num%2 == 1 {\n\t\tincrementer = int32(2)\n\t}\n\n\tfor divisor := int32(1); divisor <= maxDivisor; divisor += incrementer {\n\t\tif num%divisor == 0 {\n\t\t\tdivisibleNumbers = append(divisibleNumbers, divisor)\n\t\t\tcomplementaryDivisor := num / divisor\n\t\t\tif divisor != complementaryDivisor {\n\t\t\t\tdivisibleNumbers = append(divisibleNumbers, complementaryDivisor)\n\t\t\t}\n\t\t}\n\t}\n\n\tsort.Slice(divisibleNumbers, func(i, j int) bool { return divisibleNumbers[i] < divisibleNumbers[j] })\n\treturn divisibleNumbers\n}\n"
  },
  {
    "path": "pkg/lib/math/int64.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage math\n\nimport (\n\t\"math\"\n\t\"sort\"\n)\n\nfunc MinInt64(val int64, vals ...int64) int64 {\n\tmin := val\n\tfor _, v := range vals {\n\t\tif v < min {\n\t\t\tmin = v\n\t\t}\n\t}\n\treturn min\n}\n\nfunc MaxInt64(val int64, vals ...int64) int64 {\n\tmax := val\n\tfor _, v := range vals {\n\t\tif v > max {\n\t\t\tmax = v\n\t\t}\n\t}\n\treturn max\n}\n\nfunc IsDivisibleByInt64(num int64, divisor int64) bool {\n\treturn num%divisor == 0\n}\n\nfunc FactorsInt64(num int64) []int64 {\n\tdivisibleNumbers := []int64{}\n\tmaxDivisor := int64(math.Sqrt(float64(num)))\n\tincrementer := int64(1)\n\n\t// Skip even numbers if num is odd\n\tif num%2 == 1 {\n\t\tincrementer = int64(2)\n\t}\n\n\tfor divisor := int64(1); divisor <= maxDivisor; divisor += incrementer {\n\t\tif num%divisor == 0 {\n\t\t\tdivisibleNumbers = append(divisibleNumbers, divisor)\n\t\t\tcomplementaryDivisor := num / divisor\n\t\t\tif divisor != complementaryDivisor {\n\t\t\t\tdivisibleNumbers = append(divisibleNumbers, complementaryDivisor)\n\t\t\t}\n\t\t}\n\t}\n\n\tsort.Slice(divisibleNumbers, func(i, j int) bool { return divisibleNumbers[i] < divisibleNumbers[j] })\n\treturn divisibleNumbers\n}\n"
  },
  {
    "path": "pkg/lib/msgpack/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage msgpack\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrUnmarshalMsgpack = \"msgpack.unmarshal_msgpack\"\n\tErrMarshalMsgpack   = \"msgpack.marshal_msgpack\"\n)\n\nfunc ErrorUnmarshalMsgpack() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUnmarshalMsgpack,\n\t\tMessage: \"invalid messagepack\",\n\t})\n}\n\nfunc ErrorMarshalMsgpack() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMarshalMsgpack,\n\t\tMessage: \"invalid messagepack cannot be serialized\",\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/msgpack/msgpack.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage msgpack\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/ugorji/go/codec\"\n)\n\nvar _mh codec.MsgpackHandle\n\nfunc init() {\n\t_mh.RawToString = true\n}\n\nfunc Marshal(obj interface{}) ([]byte, error) {\n\tvar bytes []byte\n\tenc := codec.NewEncoderBytes(&bytes, &_mh)\n\terr := enc.Encode(obj)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorMarshalMsgpack()))\n\t}\n\treturn bytes, nil\n}\n\nfunc MustMarshal(obj interface{}) []byte {\n\tmsgpackBytes, err := Marshal(obj)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn msgpackBytes\n}\n\nfunc UnmarshalToInterface(b []byte) (interface{}, error) {\n\tvar obj interface{}\n\terr := Unmarshal(b, &obj)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, errors.Message(ErrorUnmarshalMsgpack()))\n\t}\n\treturn obj, nil\n}\n\nfunc Unmarshal(b []byte, obj interface{}) error {\n\tdec := codec.NewDecoderBytes(b, &_mh)\n\treturn dec.Decode(&obj)\n}\n"
  },
  {
    "path": "pkg/lib/parallel/parallel.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage parallel\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\n// Alternative: https://golang.org/pkg/sync/#WaitGroup (with error channel)\n// Alternative: https://godoc.org/golang.org/x/sync/errgroup\n\nfunc Run(fn func() error, fns ...func() error) []error {\n\tallFns := append(fns, fn)\n\n\terrChannels := make([]chan error, len(allFns))\n\tfor i := range errChannels {\n\t\terrChannels[i] = make(chan error)\n\t}\n\n\tfor i := range allFns {\n\t\tfn := allFns[i]\n\t\terrChannel := errChannels[i]\n\n\t\tif fn == nil {\n\t\t\terrChannel <- nil\n\t\t\tcontinue\n\t\t}\n\n\t\tgo func() {\n\t\t\tdefer func() {\n\t\t\t\tif r := recover(); r != nil {\n\t\t\t\t\terrChannel <- errors.CastRecoverError(r)\n\t\t\t\t}\n\t\t\t}()\n\t\t\terrChannel <- fn()\n\t\t}()\n\t}\n\n\terrors := make([]error, len(allFns))\n\tfor i := range allFns {\n\t\terrors[i] = <-errChannels[i]\n\t}\n\treturn errors\n}\n\nfunc RunFirstErr(fn func() error, fns ...func() error) error {\n\terrs := Run(fn, fns...)\n\treturn errors.FirstError(errs...)\n}\n"
  },
  {
    "path": "pkg/lib/parallel/parallel_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage parallel\n\n//\n// These tests must be run and verified manually:\n// go test github.com/cortexlabs/cortex/pkg/lib/parallel -run TestRunInParallel -v\n//\n\n// import (\n// \t\"testing\"\n\n// \t\"github.com/stretchr/testify/require\"\n\n// \t\"github.com/cortexlabs/cortex/utils/util\"\n// )\n\n// func delayAndPrint(jobID string, secs int) error {\n// \ttime.Sleep(time.Duration(secs) * time.Second)\n// \tt := time.Now()\n// \tfmt.Printf(\"Done with job %s at %d:%d:%s\\n\", jobID, t.Minute(), t.Second(), util.MillisecsStr(t))\n// \treturn nil\n// }\n\n// func delayAndError(jobID string, secs int) error {\n// \tdelayAndPrint(jobID, secs)\n// \treturn errors.New(\"Error in job \" + jobID)\n// }\n\n// func TestRunInParallel(t *testing.T) {\n// \tvar errs []error\n// \tvar err error\n\n// \terrs = util.RunInParallel(\n// \t\tfunc() error {\n// \t\t\treturn delayAndPrint(\"1\", 1)\n// \t\t},\n// \t\tfunc() error {\n// \t\t\treturn delayAndPrint(\"2\", 2)\n// \t\t},\n// \t\tfunc() error {\n// \t\t\treturn delayAndPrint(\"3\", 3)\n// \t\t},\n// \t)\n// \terr = util.FirstError(errs...)\n// \trequire.NoError(t, err)\n\n// \terrs = util.RunInParallel(\n// \t\tfunc() error {\n// \t\t\treturn delayAndPrint(\"3\", 3)\n// \t\t},\n// \t\tfunc() error {\n// \t\t\treturn delayAndPrint(\"2\", 2)\n// \t\t},\n// \t\tfunc() error {\n// \t\t\treturn delayAndPrint(\"1\", 1)\n// \t\t},\n// \t)\n// \terr = util.FirstError(errs...)\n// \trequire.NoError(t, err)\n\n// \terrs = util.RunInParallel(\n// \t\tfunc() error {\n// \t\t\treturn delayAndError(\"3\", 3)\n// \t\t},\n// \t\tfunc() error {\n// \t\t\treturn delayAndError(\"2\", 2)\n// \t\t},\n// \t\tfunc() error {\n// \t\t\treturn delayAndError(\"1\", 1)\n// \t\t},\n// \t)\n// \texpectedErrs := []error{\n// \t\terrors.New(\"Error in job 3\"),\n// \t\terrors.New(\"Error in job 2\"),\n// \t\terrors.New(\"Error in job 1\"),\n// \t}\n// \trequire.Equal(t, expectedErrs, errs)\n// }\n"
  },
  {
    "path": "pkg/lib/pointer/equal.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage pointer\n\nimport (\n\t\"time\"\n)\n\nfunc AreIntsEqual(v1 *int, v2 *int) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreInt8sEqual(v1 *int8, v2 *int8) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreInt16sEqual(v1 *int16, v2 *int16) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreInt32sEqual(v1 *int32, v2 *int32) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreInt64sEqual(v1 *int64, v2 *int64) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreFloat64sEqual(v1 *float64, v2 *float64) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreFloat32sEqual(v1 *float32, v2 *float32) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreStringsEqual(v1 *string, v2 *string) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreBoolsEqual(v1 *bool, v2 *bool) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n\nfunc AreTimesEqual(v1 *time.Time, v2 *time.Time) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn v1.Equal(*v2)\n}\n\nfunc AreDurationsEqual(v1 *time.Duration, v2 *time.Duration) bool {\n\tif v1 == nil && v2 == nil {\n\t\treturn true\n\t}\n\tif v1 == nil || v2 == nil {\n\t\treturn false\n\t}\n\treturn *v1 == *v2\n}\n"
  },
  {
    "path": "pkg/lib/pointer/pointer.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage pointer\n\nimport (\n\t\"reflect\"\n\t\"time\"\n)\n\nfunc Int(val int) *int {\n\treturn &val\n}\n\nfunc Int8(val int8) *int8 {\n\treturn &val\n}\n\nfunc Int16(val int16) *int16 {\n\treturn &val\n}\n\nfunc Int32(val int32) *int32 {\n\treturn &val\n}\n\nfunc Int64(val int64) *int64 {\n\treturn &val\n}\n\nfunc Float64(val float64) *float64 {\n\treturn &val\n}\n\nfunc Float32(val float32) *float32 {\n\treturn &val\n}\n\nfunc String(val string) *string {\n\treturn &val\n}\n\nfunc Bool(val bool) *bool {\n\treturn &val\n}\n\nfunc Time(val time.Time) *time.Time {\n\treturn &val\n}\n\nfunc Duration(val time.Duration) *time.Duration {\n\treturn &val\n}\n\n// IndirectSafe dereferences if obj is a pointer, otherwise no-op\nfunc IndirectSafe(obj interface{}) interface{} {\n\tif obj == nil {\n\t\treturn nil\n\t}\n\treturn reflect.Indirect(reflect.ValueOf(obj)).Interface()\n}\n"
  },
  {
    "path": "pkg/lib/pointer/pointer_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage pointer\n\nimport (\n\t\"testing\"\n)\n\nfunc TestIndirectSafe(t *testing.T) {\n\tIndirectSafe(nil)\n\tIndirectSafe(\"\")\n\tIndirectSafe([]string{})\n\tvar s []string\n\tIndirectSafe(s)\n}\n"
  },
  {
    "path": "pkg/lib/print/print.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage print\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n)\n\nvar _maxBoldLength = 200\n\nfunc BoldFirstLine(msg string) {\n\tmsgParts := strings.Split(msg, \"\\n\")\n\n\tif len(msgParts[0]) > _maxBoldLength {\n\t\tfmt.Println(msg)\n\t\treturn\n\t}\n\n\tfmt.Println(console.Bold(msgParts[0]))\n\n\tif len(msgParts) > 1 {\n\t\tfmt.Println(strings.Join(msgParts[1:], \"\\n\"))\n\t}\n}\n\nfunc StderrBoldFirstLine(msg string) {\n\tmsgParts := strings.Split(msg, \"\\n\")\n\n\tif len(msgParts[0]) > _maxBoldLength {\n\t\tStderrPrintln(msg)\n\t\treturn\n\t}\n\n\tStderrPrintln(console.Bold(msgParts[0]))\n\n\tif len(msgParts) > 1 {\n\t\tStderrPrintln(strings.Join(msgParts[1:], \"\\n\"))\n\t}\n}\n\nfunc BoldFirstBlock(msg string) {\n\tmsgParts := strings.Split(msg, \"\\n\\n\")\n\n\tif len(msgParts[0]) > _maxBoldLength {\n\t\tfmt.Println(msg)\n\t\treturn\n\t}\n\n\tfmt.Println(console.Bold(msgParts[0]))\n\n\tif len(msgParts) > 1 {\n\t\tfmt.Println(\"\\n\" + strings.Join(msgParts[1:], \"\\n\\n\"))\n\t}\n}\n\nfunc StderrBoldFirstBlock(msg string) {\n\tmsgParts := strings.Split(msg, \"\\n\\n\")\n\n\tif len(msgParts[0]) > _maxBoldLength {\n\t\tStderrPrintln(msg)\n\t\treturn\n\t}\n\n\tStderrPrintln(console.Bold(msgParts[0]))\n\n\tif len(msgParts) > 1 {\n\t\tStderrPrintln(\"\\n\" + strings.Join(msgParts[1:], \"\\n\\n\"))\n\t}\n}\n\nfunc Dot() error {\n\tfmt.Print(\".\")\n\treturn nil\n}\n\nfunc StderrPrintln(str string) {\n\tos.Stderr.WriteString(str + \"\\n\")\n}\n"
  },
  {
    "path": "pkg/lib/prompt/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage prompt\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrUserNoContinue = \"prompt.user_no_continue\"\n\tErrUserCtrlC      = \"prompt.user_ctrl_c\"\n)\n\nfunc ErrorUserNoContinue() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:        ErrUserNoContinue,\n\t\tNoPrint:     true,\n\t\tNoTelemetry: true,\n\t})\n}\n\nfunc ErrorUserCtrlC() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:        ErrUserCtrlC,\n\t\tNoPrint:     true,\n\t\tNoTelemetry: true,\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/prompt/prompt.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage prompt\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/exit\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\tinput \"github.com/cortexlabs/go-input\"\n)\n\nvar _ui = &input.UI{\n\tWriter: os.Stdout,\n\tReader: os.Stdin,\n}\n\ntype Options struct {\n\tPrompt              string\n\tDefaultStr          string\n\tHideDefault         bool\n\tMaskDefault         bool\n\tHideTyping          bool\n\tMaskTyping          bool\n\tTypingMaskVal       string\n\tSkipTrailingNewline bool\n}\n\nfunc Prompt(opts *Options) string {\n\tprompt := opts.Prompt\n\n\tif opts.DefaultStr != \"\" && !opts.HideDefault {\n\t\tdefaultStr := opts.DefaultStr\n\t\tif opts.MaskDefault {\n\t\t\tdefaultStr = s.MaskString(defaultStr, 4)\n\t\t}\n\t\tprompt = fmt.Sprintf(\"%s [%s]\", opts.Prompt, defaultStr)\n\t}\n\n\tval, err := _ui.Ask(prompt, &input.Options{\n\t\tDefault:     opts.DefaultStr,\n\t\tHide:        opts.HideTyping,\n\t\tMask:        opts.MaskTyping,\n\t\tMaskVal:     opts.TypingMaskVal,\n\t\tRequired:    false,\n\t\tHideDefault: true,\n\t\tHideOrder:   true,\n\t\tLoop:        false,\n\t\tSkipNewline: opts.SkipTrailingNewline,\n\t})\n\n\tif err != nil {\n\t\tif errors.Message(err) == \"interrupted\" {\n\t\t\texit.Error(ErrorUserCtrlC())\n\t\t}\n\t\tif strings.Contains(errors.Message(err), \"not a terminal\") {\n\t\t\terr = errors.Append(err, \"\\n\\nyou may be able to pass flags into this command to provide all required inputs and/or skip prompts (e.g. via `--yes`)\")\n\t\t}\n\t\texit.Error(err)\n\t}\n\n\treturn val\n}\n\nfunc YesOrExit(prompt string, yesMessage string, noMessage string) {\n\tfor {\n\t\tstr := Prompt(&Options{\n\t\t\tPrompt:      prompt + \" (y/n)\",\n\t\t\tHideDefault: true,\n\t\t})\n\n\t\tif strings.ToLower(str) == \"y\" {\n\t\t\tif yesMessage != \"\" {\n\t\t\t\tfmt.Println(yesMessage)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\n\t\tif strings.ToLower(str) == \"n\" {\n\t\t\tif noMessage != \"\" {\n\t\t\t\tfmt.Println(noMessage)\n\t\t\t}\n\t\t\texit.Error(ErrorUserNoContinue())\n\t\t}\n\n\t\tfmt.Println(\"please enter \\\"y\\\" or \\\"n\\\"\")\n\t\tfmt.Println()\n\t}\n}\n\nfunc YesOrNo(prompt string, yesMessage string, noMessage string) bool {\n\tfor true {\n\t\tstr := Prompt(&Options{\n\t\t\tPrompt:      prompt + \" (y/n)\",\n\t\t\tHideDefault: true,\n\t\t})\n\n\t\tif strings.ToLower(str) == \"y\" {\n\t\t\tif yesMessage != \"\" {\n\t\t\t\tfmt.Println(yesMessage)\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\n\t\tif strings.ToLower(str) == \"n\" {\n\t\t\tif noMessage != \"\" {\n\t\t\t\tfmt.Println(noMessage)\n\t\t\t}\n\t\t\treturn false\n\t\t}\n\n\t\tfmt.Println(\"please enter \\\"y\\\" or \\\"n\\\"\")\n\t\tfmt.Println()\n\t}\n\treturn false\n}\n"
  },
  {
    "path": "pkg/lib/random/random.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage random\n\nimport (\n\t\"math/rand\"\n\t\"time\"\n)\n\nconst (\n\t_uppercaseBytes = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n\t_lowercaseBytes = \"abcdefghijklmnopqrstuvwxyz\"\n\t_letterBytes    = _lowercaseBytes + _uppercaseBytes\n\t_numberBytes    = \"0123456789\"\n\t_stringBytes    = _letterBytes + _numberBytes\n\n\t_letterIdxBits = 6                     // 6 bits to represent a letter index\n\t_letterIdxMask = 1<<_letterIdxBits - 1 // All 1-bits, as many as _letterIdxBits\n\t_letterIdxMax  = 63 / _letterIdxBits   // # of letter indices fitting in 63 bits\n)\n\nfunc randomString(n int, src rand.Source, charset string) string {\n\tb := make([]byte, n)\n\tfor i, cache, remain := n-1, src.Int63(), _letterIdxMax; i >= 0; {\n\t\tif remain == 0 {\n\t\t\tcache, remain = src.Int63(), _letterIdxMax\n\t\t}\n\t\tif idx := int(cache & _letterIdxMask); idx < len(charset) {\n\t\t\tb[i] = charset[idx]\n\t\t\ti--\n\t\t}\n\t\tcache >>= _letterIdxBits\n\t\tremain--\n\t}\n\n\treturn string(b)\n}\n\n// Digits generates a random string containing only digits\nfunc Digits(n int) string {\n\treturn randomString(n, rand.NewSource(time.Now().UnixNano()), _numberBytes)\n}\n\n// Letters generates a random string containing only english alphabet characters (upper and lower case)\nfunc Letters(n int) string {\n\treturn randomString(n, rand.NewSource(time.Now().UnixNano()), _letterBytes)\n}\n\n// LowercaseLetters generates a random string containing only lower case english alphabet characters\nfunc LowercaseLetters(n int) string {\n\treturn randomString(n, rand.NewSource(time.Now().UnixNano()), _lowercaseBytes)\n}\n\n// String generates a random string containing both digits and english alphabet characters (upper and lower)\nfunc String(n int) string {\n\treturn randomString(n, rand.NewSource(time.Now().UnixNano()), _stringBytes)\n}\n\nfunc LowercaseString(n int) string {\n\treturn randomString(n, rand.NewSource(time.Now().UnixNano()), _lowercaseBytes+_numberBytes)\n}\n"
  },
  {
    "path": "pkg/lib/regex/regex.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage regex\n\nimport (\n\t\"regexp\"\n)\n\nfunc MatchAnyRegex(s string, regexes []*regexp.Regexp) bool {\n\tfor _, regex := range regexes {\n\t\tif regex.MatchString(s) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nvar _leadingWhitespaceRegex = regexp.MustCompile(`^\\s+`)\n\nfunc HasLeadingWhitespace(s string) bool {\n\treturn _leadingWhitespaceRegex.MatchString(s)\n}\n\nvar _trailingWhitespaceRegex = regexp.MustCompile(`\\s+$`)\n\nfunc HasTrailingWhitespace(s string) bool {\n\treturn _trailingWhitespaceRegex.MatchString(s)\n}\n\n// letters, numbers, spaces representable in UTF-8, and the following characters: _ . : / + - @\n// = is not supported because it doesn't propagate to the NLB correctly (via the k8s service annotation)\nvar _awsTagRegex = regexp.MustCompile(`^[\\sa-zA-Z0-9_\\-\\.:/+@]+$`)\n\nfunc IsValidAWSTag(s string) bool {\n\treturn _awsTagRegex.MatchString(s)\n}\n\nvar _alphaNumericDashDotUnderscoreRegex = regexp.MustCompile(`^[a-zA-Z0-9_\\-\\.]+$`)\n\nfunc IsAlphaNumericDashDotUnderscore(s string) bool {\n\treturn _alphaNumericDashDotUnderscoreRegex.MatchString(s)\n}\n\nvar _alphaNumericDashUnderscoreRegex = regexp.MustCompile(`^[a-zA-Z0-9_\\-]+$`)\n\nfunc IsAlphaNumericDashUnderscore(s string) bool {\n\treturn _alphaNumericDashUnderscoreRegex.MatchString(s)\n}\n\nvar _alphaNumericDotUnderscoreRegex = regexp.MustCompile(`^[a-zA-Z0-9_\\.]+$`)\n\nfunc IsAlphaNumericDotUnderscore(s string) bool {\n\treturn _alphaNumericDotUnderscoreRegex.MatchString(s)\n}\n\nvar _alphaNumericDashRegex = regexp.MustCompile(`^[a-zA-Z0-9\\-]+$`)\n\nfunc IsAlphaNumericDash(s string) bool {\n\treturn _alphaNumericDashRegex.MatchString(s)\n}\n\n// used the evaluated form of\n// https://github.com/docker/distribution/blob/3150937b9f2b1b5b096b2634d0e7c44d4a0f89fb/reference/regexp.go#L68-L70\nvar _dockerValidImage = regexp.MustCompile(\n\t`^((?:(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])` +\n\t\t`(?:(?:\\.(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9]))+)` +\n\t\t`?(?::[0-9]+)?/)?[a-z0-9]` +\n\t\t`+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)` +\n\t\t`?(?:(?:/[a-z0-9]+(?:(?:(?:[._]|__|[-]*)[a-z0-9]+)+)?)+)?)` +\n\t\t`(?::([\\w][\\w.-]{0,127}))` +\n\t\t`?(?:@([A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}))?$`,\n)\n\nfunc IsValidDockerImage(s string) bool {\n\treturn _dockerValidImage.MatchString(s)\n}\n\nvar _ecrPattern = regexp.MustCompile(\n\t`(^[a-zA-Z0-9][a-zA-Z0-9-_]*)\\.dkr\\.ecr(\\-fips)?\\.([a-zA-Z0-9][a-zA-Z0-9-_]*)\\.amazonaws\\.com(\\.cn)?`,\n)\n\nfunc IsValidECRURL(s string) bool {\n\treturn _ecrPattern.MatchString(s)\n}\n"
  },
  {
    "path": "pkg/lib/regex/regex_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage regex\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\ntype regexpMatch struct {\n\tinput string\n\tmatch bool\n\tsubs  []string\n}\n\nfunc TestHasLeadingWhitespace(t *testing.T) {\n\ttestcases := []regexpMatch{\n\t\t{\n\t\t\tinput: \" test\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\ttest\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\\ntest\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \" test \",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\ttest\t\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\\ntest\\n\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"test\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"te st\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"te\tst\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"te\\nst\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"_\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"test \",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"test\t\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"test\\n\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \" \",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\t\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\\n\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\",\n\t\t\tmatch: false,\n\t\t},\n\t}\n\n\tfor i := range testcases {\n\t\tmatch := HasLeadingWhitespace(testcases[i].input)\n\t\tassert.Equal(t, testcases[i].match, match, \"input: \"+testcases[i].input)\n\t}\n}\n\nfunc TestHasTrailingWhitespace(t *testing.T) {\n\ttestcases := []regexpMatch{\n\t\t{\n\t\t\tinput: \" test\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"\ttest\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"\\ntest\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \" test \",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\ttest\t\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\\ntest\\n\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"test\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"te st\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"te\tst\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"te\\nst\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"_\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"test \",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"test\t\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"test\\n\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \" \",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\t\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\\n\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"\",\n\t\t\tmatch: false,\n\t\t},\n\t}\n\n\tfor i := range testcases {\n\t\tmatch := HasTrailingWhitespace(testcases[i].input)\n\t\tassert.Equal(t, testcases[i].match, match, \"input: \"+testcases[i].input)\n\t}\n}\n\nfunc TestAlphaNumericDashDotUnderscoreRegex(t *testing.T) {\n\ttestcases := []regexpMatch{\n\t\t{\n\t\t\tinput: \"generic.package.com\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"generic_package.com\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"_value-123\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"data.generic.package()\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"!variable\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"assignment=value\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"LibrayPackage@model.net\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"aaaabbbbcccABCDZX-123456789\",\n\t\t\tmatch: true,\n\t\t},\n\t}\n\n\tfor i := range testcases {\n\t\tmatch := _alphaNumericDashDotUnderscoreRegex.MatchString(testcases[i].input)\n\t\tif match != testcases[i].match {\n\t\t\tt.Errorf(\"No match for %q\", testcases[i].input)\n\t\t}\n\t}\n}\n\nfunc TestAlphaNumericDashUnderscoreRegex(t *testing.T) {\n\ttestcases := []regexpMatch{\n\t\t{\n\t\t\tinput: \"generic.package.com\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"generic_package.com\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"_value-123\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"data.generic.package()\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"!variable\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"assignment=value\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"LibrayPackage@model.net\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"aaaabbbbcccABCDZX-123456789\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"word1-word2_word3_word4\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"____-----____\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"(word)\",\n\t\t\tmatch: false,\n\t\t},\n\t}\n\n\tfor i := range testcases {\n\t\tmatch := _alphaNumericDashUnderscoreRegex.MatchString(testcases[i].input)\n\t\tif match != testcases[i].match {\n\t\t\tt.Errorf(\"No match for %q\", testcases[i].input)\n\t\t}\n\t}\n}\n\nfunc TestValidDockerImage(t *testing.T) {\n\ttestcases := []regexpMatch{\n\t\t{\n\t\t\tinput: \"\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"short\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"simple/name\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"library/ubuntu\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"library/ubuntu:latest\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"docker/stevvooe/app\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"aa/aa/aa/aa/aa/aa/aa/aa/aa/bb/bb/bb/bb/bb/bb\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"aa/aa/bb/bb/bb\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"a/a/a/a\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"a/a/a/a/\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"a//a/a\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"a\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"a/aa\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"a/aa/a\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo.com\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo.com/\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo.com:8080/bar\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo.com:http/bar\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo.com/bar\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo.com/bar/baz\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"localhost:8080/bar\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"sub-dom1.foo.com/bar/baz/quux\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"blog.foo.com/bar/baz\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"a^a\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"aa/asdf$$^/aa\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"asdf$$^/aa\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"aa-a/a\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: strings.Repeat(\"a/\", 128) + \"a\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"a-/a/a/a\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo.com/a-/a/a\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"-foo/bar\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo/bar-\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo-/bar\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo/-bar\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"_foo/bar\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo_bar\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo_bar.com\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo_bar.com:8080/app\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"foo.com/foo_bar\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"____/____\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"_docker/_docker\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"docker_/docker_\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"b.gcr.io/test.example.com/my-app\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"xn--n3h.com/myimage\", // ☃.com in punycode\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"xn--7o8h.com/myimage\", // 🐳.com in punycode\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"example.com/xn--7o8h.com/myimage\", // 🐳.com in punycode\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"example.com/some_separator__underscore/myimage\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"example.com/__underscore/myimage\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"example.com/..dots/myimage\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"example.com/.dots/myimage\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"example.com/nodouble..dots/myimage\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"example.com/nodouble..dots/myimage\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"docker./docker\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \".docker/docker\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"docker-/docker\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"-docker/docker\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"do..cker/docker\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"do__cker:8080/docker\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"do__cker/docker\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"b.gcr.io/test.example.com/my-app\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.io/foo/project--id.module--name.ver---sion--name\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.com:8080/myapp:tag\",\n\t\t\tmatch: true,\n\t\t\tsubs:  []string{\"registry.com:8080/myapp\", \"tag\", \"\"},\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.com:8080/myapp@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\",\n\t\t\tmatch: true,\n\t\t\tsubs:  []string{\"registry.com:8080/myapp\", \"\", \"sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\"},\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.com:8080/myapp:tag2@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\",\n\t\t\tmatch: true,\n\t\t\tsubs:  []string{\"registry.com:8080/myapp\", \"tag2\", \"sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\"},\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.com:8080/myapp@sha256:badbadbadbad\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.com:8080/myapp:invalid~tag\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"bad_hostname.com:8080/myapp:tag\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput:// localhost treated as name, missing tag with 8080 as tag\n\t\t\t\"localhost:8080@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\",\n\t\t\tmatch: true,\n\t\t\tsubs:  []string{\"localhost\", \"8080\", \"sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\"},\n\t\t},\n\t\t{\n\t\t\tinput: \"localhost:8080/name@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\",\n\t\t\tmatch: true,\n\t\t\tsubs:  []string{\"localhost:8080/name\", \"\", \"sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\"},\n\t\t},\n\t\t{\n\t\t\tinput: \"localhost:http/name@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\t// localhost will be treated as an image name without a host\n\t\t\tinput: \"localhost@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\",\n\t\t\tmatch: true,\n\t\t\tsubs:  []string{\"localhost\", \"\", \"sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\"},\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.com:8080/myapp@bad\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.com:8080/myapp@2bad\",\n\t\t\tmatch: false, // Support this as valid?\n\t\t},\n\t\t{\n\t\t\tinput: \"680880929103.dkr.ecr.eu-central-1.amazonaws.com/cortexlabs/async-gateway:latest\",\n\t\t\tmatch: true,\n\t\t},\n\t}\n\n\tfor i := range testcases {\n\t\tmatch := _dockerValidImage.MatchString(testcases[i].input)\n\t\tif match != testcases[i].match {\n\t\t\tt.Errorf(\"No match for %q\", testcases[i].input)\n\t\t}\n\t}\n\n}\n\nfunc TestValidECR(t *testing.T) {\n\ttestcases := []regexpMatch{\n\t\t{\n\t\t\tinput: \"\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"library/ubuntu:latest\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"registry.com:8080/myapp:tag\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"680880929102.dkr.ecr.eu-central-1.amazonaws.com/cortexlabs/python-handler-cpu:latest\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"localhost@sha256:be178c0543eb17f5f3043021c9e5fcf30285e557a4fc309cce97ff9ca6182912\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"680880929102.dkr.ecr.eu-central-1.com/cortexlabs/image\",\n\t\t\tmatch: false,\n\t\t},\n\t\t{\n\t\t\tinput: \"680880929102.dkr.ecr.us-east-1.amazonaws.com/registry\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"1234567.dkr.ecr.us-west-1.amazonaws.com/registry/image:123\",\n\t\t\tmatch: true,\n\t\t},\n\t\t{\n\t\t\tinput: \"680880929102.dkr.ecr.us-east-1.amazonaws.com\",\n\t\t\tmatch: true,\n\t\t},\n\t}\n\n\tfor i := range testcases {\n\t\tmatch := _ecrPattern.MatchString(testcases[i].input)\n\t\tif match != testcases[i].match {\n\t\t\tt.Errorf(\"No match for %q\", testcases[i].input)\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/requests/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage requests\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n)\n\nconst (\n\t_errStrCantMakeRequest = \"unable to make request\"\n\t_errStrRead            = \"unable to read\"\n)\n\nfunc errStrFailedToConnect(u url.URL) string {\n\treturn \"failed to connect to \" + urls.TrimQueryParamsURL(u)\n}\n\nconst (\n\tErrResponseUnknown = \"requests.response_unknown\"\n)\n\nfunc ErrorResponseUnknown(body string, statusCode int) error {\n\tmsg := body\n\tif strings.TrimSpace(body) == \"\" {\n\t\tmsg = fmt.Sprintf(\"empty response (status code %d)\", statusCode)\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrResponseUnknown,\n\t\tMessage: msg,\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/requests/requests.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage requests\n\nimport (\n\t\"crypto/tls\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nfunc MakeRequest(request *http.Request) (http.Header, []byte, error) {\n\tclient := http.Client{\n\t\tTimeout: 5 * time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t\t},\n\t}\n\n\tresponse, err := client.Do(request)\n\tif err != nil {\n\t\treturn nil, nil, errors.Wrap(err, errStrFailedToConnect(*request.URL))\n\t}\n\tdefer response.Body.Close()\n\n\tif response.StatusCode != 200 {\n\t\tbodyBytes, err := ioutil.ReadAll(response.Body)\n\t\tif err != nil {\n\t\t\treturn nil, nil, errors.Wrap(err, _errStrRead)\n\t\t}\n\t\treturn nil, nil, ErrorResponseUnknown(string(bodyBytes), response.StatusCode)\n\t}\n\n\tbodyBytes, err := ioutil.ReadAll(response.Body)\n\tif err != nil {\n\t\treturn nil, nil, errors.Wrap(err, _errStrRead)\n\t}\n\treturn response.Header, bodyBytes, nil\n}\n"
  },
  {
    "path": "pkg/lib/sets/strset/strset.go",
    "content": "/*\nCopyright 2017 ScyllaDB\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage strset\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"sort\"\n\t\"strings\"\n)\n\ntype Set map[string]struct{}\n\nvar _keyExists = struct{}{}\n\n// New creates and initializes a new Set.\nfunc New(ts ...string) Set {\n\ts := make(Set)\n\ts.Add(ts...)\n\treturn s\n}\n\nfunc FromSlice(items []string) Set {\n\treturn New(items...)\n}\n\n// NewWithSize creates a new Set and gives make map a size hint.\nfunc NewWithSize(size int) Set {\n\treturn make(Set, size)\n}\n\n// Add includes the specified items (one or more) to the Set. The underlying\n// Set s is modified. If passed nothing it silently returns.\nfunc (s Set) Add(items ...string) {\n\tfor _, item := range items {\n\t\ts[item] = _keyExists\n\t}\n}\n\n// Remove deletes the specified items from the Set. The underlying Set s is\n// modified. If passed nothing it silently returns.\nfunc (s Set) Remove(items ...string) {\n\tfor _, item := range items {\n\t\tdelete(s, item)\n\t}\n}\n\n// GetOne returns an item from the set or \"\" if the set is empty.\nfunc (s Set) GetOne() string {\n\tfor item := range s {\n\t\treturn item\n\t}\n\treturn \"\"\n}\n\n// GetOne2 returns an item from the set. The second value is a bool that is\n// true if an item exists in the set, or false if the set is empty.\nfunc (s Set) GetOne2() (string, bool) {\n\tfor item := range s {\n\t\treturn item, true\n\t}\n\treturn \"\", false\n}\n\n// Pop deletes and returns an item from the Set. The underlying Set s is\n// modified. If Set is empty, the zero value is returned.\nfunc (s Set) Pop() string {\n\tfor item := range s {\n\t\tdelete(s, item)\n\t\treturn item\n\t}\n\treturn \"\"\n}\n\n// Pop2 tries to delete and return an item from the Set. The underlying Set s\n// is modified. The second value is a bool that is true if the item existed in\n// the set, and false if not. If Set is empty, the zero value and false are\n// returned.\nfunc (s Set) Pop2() (string, bool) {\n\tfor item := range s {\n\t\tdelete(s, item)\n\t\treturn item, true\n\t}\n\treturn \"\", false\n}\n\n// Has looks for the existence of items passed. It returns false if nothing is\n// passed. For multiple items it returns true only if all of the items exist.\nfunc (s Set) Has(items ...string) bool {\n\thas := false\n\tfor _, item := range items {\n\t\tif _, has = s[item]; !has {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn has\n}\n\n// HasWithPrefix checks if at least one element of the set is the prefix of any of the passed items.\n// It returns false if nothing is passed.\nfunc (s Set) HasWithPrefix(items ...string) bool {\n\tfor _, prefix := range items {\n\t\tfor k := range s {\n\t\t\tif strings.HasPrefix(prefix, k) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// HasAny looks for the existence of any of the items passed.\n// It returns false if nothing is passed.\n// For multiple items it returns true if any of the items exist.\nfunc (s Set) HasAny(items ...string) bool {\n\thas := false\n\tfor _, item := range items {\n\t\tif _, has = s[item]; has {\n\t\t\tbreak\n\t\t}\n\t}\n\treturn has\n}\n\n// Clear removes all items from the Set.\nfunc (s *Set) Clear() {\n\t*s = make(Set)\n}\n\n// IsEqual test whether s and t are the same in size and have the same items.\nfunc (s Set) IsEqual(t Set) bool {\n\tif len(s) != len(t) {\n\t\treturn false\n\t}\n\n\tfor item := range s {\n\t\tif !t.Has(item) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// IsSubset tests whether t is a subset of s.\nfunc (s Set) IsSubset(t Set) bool {\n\tif len(s) < len(t) {\n\t\treturn false\n\t}\n\n\tfor item := range t {\n\t\tif !s.Has(item) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// IsSuperset tests whether t is a superset of s.\nfunc (s Set) IsSuperset(t Set) bool {\n\treturn t.IsSubset(s)\n}\n\n// Copy returns a new Set with a copy of s.\nfunc (s Set) Copy() Set {\n\tu := make(Set, len(s))\n\tfor item := range s {\n\t\tu[item] = _keyExists\n\t}\n\treturn u\n}\n\n// String returns a string representation of s\nfunc (s Set) String() string {\n\tv := make([]string, 0, len(s))\n\tfor item := range s {\n\t\tv = append(v, fmt.Sprintf(\"%v\", item))\n\t}\n\treturn fmt.Sprintf(\"[%s]\", strings.Join(v, \", \"))\n}\n\n// List returns a slice of all items.\nfunc (s Set) Slice() []string {\n\tv := make([]string, 0, len(s))\n\tfor item := range s {\n\t\tv = append(v, item)\n\t}\n\treturn v\n}\n\n// List returns a sorted slice of all items (a to z).\nfunc (s Set) SliceSorted() []string {\n\tv := s.Slice()\n\tsort.Strings(v)\n\treturn v\n}\n\n// Merge is like Union, however it modifies the current Set it's applied on\n// with the given t Set.\nfunc (s Set) Merge(sets ...Set) {\n\tfor _, set := range sets {\n\t\tfor item := range set {\n\t\t\ts[item] = _keyExists\n\t\t}\n\t}\n}\n\n// Subtract removes the Set items contained in sets from Set s\nfunc (s Set) Subtract(sets ...Set) {\n\tfor _, set := range sets {\n\t\tfor item := range set {\n\t\t\tdelete(s, item)\n\t\t}\n\t}\n}\n\n// Remove items until len(s) <= targetLen\nfunc (s Set) Shrink(targetLen int) {\n\tfor len(s) > targetLen {\n\t\ts.Pop()\n\t}\n}\n\n// remove items alphabetically until len(s) <= targetLen\nfunc (s Set) ShrinkSorted(targetLen int) {\n\tif len(s) <= targetLen {\n\t\treturn\n\t}\n\n\tsorted := s.SliceSorted()\n\textras := sorted[targetLen:]\n\ts.Remove(extras...)\n}\n\n// Union is the merger of multiple sets. It returns a new set with all the\n// elements present in all the sets that are passed.\nfunc Union(sets ...Set) Set {\n\tmaxPos := -1\n\tmaxSize := 0\n\n\t// find which set is the largest and its size\n\tfor i, set := range sets {\n\t\tif l := len(set); l > maxSize {\n\t\t\tmaxSize = l\n\t\t\tmaxPos = i\n\t\t}\n\t}\n\tif maxSize == 0 {\n\t\treturn make(Set)\n\t}\n\n\tu := sets[maxPos].Copy()\n\tfor i, set := range sets {\n\t\tif i == maxPos {\n\t\t\tcontinue\n\t\t}\n\t\tfor item := range set {\n\t\t\tu[item] = _keyExists\n\t\t}\n\t}\n\treturn u\n}\n\n// Difference returns a new set which contains items which are in the first\n// set but not in the others.\nfunc Difference(set1 Set, sets ...Set) Set {\n\ts := set1.Copy()\n\tfor _, set := range sets {\n\t\ts.Subtract(set)\n\t}\n\treturn s\n}\n\n// Intersection returns a new set which contains items that only exist in all\n// given sets.\nfunc Intersection(sets ...Set) Set {\n\tminPos := -1\n\tminSize := math.MaxInt64\n\tfor i, set := range sets {\n\t\tif l := len(set); l < minSize {\n\t\t\tminSize = l\n\t\t\tminPos = i\n\t\t}\n\t}\n\tif minSize == math.MaxInt64 || minSize == 0 {\n\t\treturn make(Set)\n\t}\n\n\tt := sets[minPos].Copy()\n\tfor i, set := range sets {\n\t\tif i == minPos {\n\t\t\tcontinue\n\t\t}\n\t\tfor item := range t {\n\t\t\tif _, has := set[item]; !has {\n\t\t\t\tdelete(t, item)\n\t\t\t}\n\t\t}\n\t}\n\treturn t\n}\n\n// SymmetricDifference returns a new set which s is the difference of items\n// which are in one of either, but not in both.\nfunc SymmetricDifference(s Set, t Set) Set {\n\tu := Difference(s, t)\n\tv := Difference(t, s)\n\treturn Union(u, v)\n}\n"
  },
  {
    "path": "pkg/lib/sets/strset/strset_test.go",
    "content": "/*\nCopyright 2017 ScyllaDB\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage strset_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/stretchr/testify/require\"\n)\n\n// Also tests Add\nfunc TestNew(t *testing.T) {\n\tset := strset.New()\n\trequire.Equal(t, 0, len(set))\n\n\tset = strset.New(\"a\", \"b\", \"a\")\n\trequire.Equal(t, 2, len(set))\n\tif _, ok := set[\"a\"]; !ok {\n\t\trequire.FailNow(t, \"a not found in set\")\n\t}\n\tif _, ok := set[\"b\"]; !ok {\n\t\trequire.FailNow(t, \"b not found in set\")\n\t}\n}\n\nfunc TestAdd(t *testing.T) {\n\tset := strset.New()\n\tset.Add(\"a\")\n\tset.Add(\"b\", \"c\")\n\trequire.Equal(t, set, strset.New(\"a\", \"b\", \"c\"))\n}\n\nfunc TestRemove(t *testing.T) {\n\tset := strset.New(\"a\", \"b\")\n\tset.Remove(\"c\")\n\trequire.Equal(t, set, strset.New(\"a\", \"b\"))\n\n\tset.Remove()\n\trequire.Equal(t, set, strset.New(\"a\", \"b\"))\n\n\tset.Remove(\"a\")\n\trequire.Equal(t, set, strset.New(\"b\"))\n\n\tset.Add(\"a\")\n\tset.Remove(\"a\", \"b\")\n\trequire.Equal(t, set, strset.New())\n}\n\nfunc TestPop(t *testing.T) {\n\tset := strset.New(\"a\", \"b\")\n\tp := set.Pop()\n\trequire.Contains(t, []string{\"a\", \"b\"}, p)\n\trequire.Equal(t, 1, len(set))\n\tp = set.Pop()\n\trequire.Contains(t, []string{\"a\", \"b\"}, p)\n\trequire.Equal(t, 0, len(set))\n\tp = set.Pop()\n\trequire.Equal(t, \"\", p)\n\trequire.Equal(t, 0, len(set))\n}\n\nfunc TestPop2(t *testing.T) {\n\tset := strset.New(\"a\", \"b\")\n\tp, ok := set.Pop2()\n\trequire.Contains(t, []string{\"a\", \"b\"}, p)\n\trequire.Equal(t, 1, len(set))\n\trequire.True(t, ok)\n\tp, ok = set.Pop2()\n\trequire.Contains(t, []string{\"a\", \"b\"}, p)\n\trequire.Equal(t, 0, len(set))\n\trequire.True(t, ok)\n\tp, ok = set.Pop2()\n\trequire.Equal(t, \"\", p)\n\trequire.Equal(t, 0, len(set))\n\trequire.False(t, ok)\n}\n\nfunc TestHas(t *testing.T) {\n\tset := strset.New(\"a\", \"b\", \"c\")\n\trequire.True(t, set.Has(\"a\"))\n\trequire.True(t, set.Has(\"a\", \"b\"))\n\trequire.False(t, set.Has(\"z\"))\n\trequire.False(t, set.Has(\"a\", \"z\"))\n}\n\nfunc TestHasAny(t *testing.T) {\n\tset := strset.New(\"a\", \"b\", \"c\")\n\trequire.True(t, set.HasAny(\"a\"))\n\trequire.True(t, set.HasAny(\"a\", \"b\"))\n\trequire.False(t, set.HasAny(\"z\"))\n\trequire.True(t, set.HasAny(\"a\", \"z\"))\n}\n\nfunc TestClear(t *testing.T) {\n\tset := strset.New(\"a\", \"b\", \"c\")\n\trequire.Equal(t, 3, len(set))\n\tset.Clear()\n\trequire.Equal(t, 0, len(set))\n}\n\nfunc TestIsEqual(t *testing.T) {\n\tset1 := strset.New(\"a\", \"b\", \"c\")\n\tset2 := strset.New(\"a\", \"b\", \"c\")\n\tset3 := strset.New(\"a\", \"b\")\n\tset4 := strset.New(\"a\", \"b\", \"z\")\n\trequire.True(t, set1.IsEqual(set2))\n\trequire.False(t, set1.IsEqual(set3))\n\trequire.False(t, set1.IsEqual(set4))\n}\n\nfunc TestIsSubset(t *testing.T) {\n\tset1 := strset.New(\"a\", \"b\", \"c\")\n\tset2 := strset.New(\"a\", \"b\", \"c\")\n\tset3 := strset.New(\"a\", \"b\")\n\tset4 := strset.New(\"a\", \"b\", \"z\")\n\tset5 := strset.New(\"a\", \"b\", \"c\", \"d\")\n\trequire.True(t, set1.IsSubset(set2))\n\trequire.True(t, set1.IsSubset(set3))\n\trequire.False(t, set1.IsSubset(set4))\n\trequire.False(t, set1.IsSubset(set5))\n}\n\nfunc TestIsSuperset(t *testing.T) {\n\tset1 := strset.New(\"a\", \"b\", \"c\")\n\tset2 := strset.New(\"a\", \"b\", \"c\")\n\tset3 := strset.New(\"a\", \"b\")\n\tset4 := strset.New(\"a\", \"b\", \"z\")\n\tset5 := strset.New(\"a\", \"b\", \"c\", \"d\")\n\trequire.True(t, set1.IsSuperset(set2))\n\trequire.False(t, set1.IsSuperset(set3))\n\trequire.False(t, set1.IsSuperset(set4))\n\trequire.True(t, set1.IsSuperset(set5))\n}\n\nfunc TestCopy(t *testing.T) {\n\tset := strset.New()\n\tcset := set.Copy()\n\trequire.Equal(t, 0, len(cset))\n\n\tset = strset.New(\"a\", \"b\")\n\tcset = set.Copy()\n\trequire.Equal(t, 2, len(cset))\n\tif _, ok := set[\"a\"]; !ok {\n\t\trequire.FailNow(t, \"a not found in set\")\n\t}\n\tif _, ok := set[\"b\"]; !ok {\n\t\trequire.FailNow(t, \"b not found in set\")\n\t}\n}\n\nfunc TestSlice(t *testing.T) {\n\tset := strset.New()\n\trequire.Equal(t, set.Slice(), []string{})\n\n\tset.Add(\"a\")\n\trequire.Equal(t, set.Slice(), []string{\"a\"})\n\n\tset.Add(\"a\", \"b\")\n\trequire.ElementsMatch(t, set.Slice(), []string{\"a\", \"b\"})\n}\n\nfunc TestMerge(t *testing.T) {\n\tset := strset.New()\n\temptySet := strset.New()\n\tset.Merge(emptySet)\n\trequire.Equal(t, 0, len(set))\n\n\tset.Merge(strset.New(\"a\"))\n\trequire.Equal(t, 1, len(set))\n\n\tset.Merge(emptySet)\n\trequire.Equal(t, 1, len(set))\n\n\tset.Add(\"a\", \"b\", \"c\")\n\tset.Merge(strset.New(\"e\", \"e\", \"d\"))\n\trequire.Equal(t, set, strset.New(\"a\", \"b\", \"c\", \"e\", \"d\"))\n\n\tset.Merge(strset.New(\"a\", \"e\"))\n\trequire.Equal(t, set, strset.New(\"a\", \"b\", \"c\", \"e\", \"d\"))\n\tset.Merge(strset.New(\"a\", \"e\", \"i\"), strset.New(\"o\", \"u\"), strset.New(\"sometimes y\"))\n\trequire.Equal(t, set, strset.New(\"a\", \"b\", \"c\", \"e\", \"d\", \"i\", \"o\", \"u\", \"sometimes y\"))\n}\n\nfunc TestSubtract(t *testing.T) {\n\tset := strset.New(\"a\", \"b\", \"c\")\n\tset.Subtract(strset.New(\"z\", \"a\", \"b\"))\n\trequire.Equal(t, set, strset.New(\"c\"))\n\tset.Subtract(strset.New(\"x\"))\n\trequire.Equal(t, set, strset.New(\"c\"))\n}\n\nfunc TestShrink(t *testing.T) {\n\tset := strset.New(\"a\", \"b\", \"c\", \"d\")\n\tset.Shrink(2)\n\trequire.Len(t, set, 2)\n\n\tset = strset.New(\"g\", \"f\", \"e\", \"d\", \"c\", \"b\", \"a\")\n\tset.ShrinkSorted(3)\n\trequire.Len(t, set, 3)\n\n\tset = strset.New(\"a\", \"b\")\n\tset.Shrink(2)\n\trequire.Equal(t, set, strset.New(\"a\", \"b\"))\n\n\tset = strset.New(\"a\")\n\tset.Shrink(2)\n\trequire.Equal(t, set, strset.New(\"a\"))\n\n\tset = strset.New()\n\tset.Shrink(2)\n\trequire.Len(t, set, 0)\n}\n\nfunc TestShrinkSorted(t *testing.T) {\n\tfor i := 0; i < 10; i++ {\n\t\tset := strset.New(\"g\", \"f\", \"e\", \"d\", \"c\", \"b\", \"a\")\n\t\tset.ShrinkSorted(2)\n\t\trequire.Equal(t, set, strset.New(\"a\", \"b\"))\n\n\t\tset = strset.New(\"g\", \"f\", \"e\", \"d\", \"c\", \"b\", \"a\")\n\t\tset.ShrinkSorted(3)\n\t\trequire.Equal(t, set, strset.New(\"a\", \"b\", \"c\"))\n\t}\n\n\tset := strset.New(\"a\", \"b\")\n\tset.ShrinkSorted(2)\n\trequire.Equal(t, set, strset.New(\"a\", \"b\"))\n\n\tset = strset.New(\"a\")\n\tset.ShrinkSorted(2)\n\trequire.Equal(t, set, strset.New(\"a\"))\n\n\tset = strset.New()\n\tset.ShrinkSorted(2)\n\trequire.Len(t, set, 0)\n}\n\nfunc TestUnion(t *testing.T) {\n\trequire.Equal(t, len(strset.Union(strset.New(), strset.New())), 0)\n\trequire.Equal(t, strset.Union(strset.New(\"a\", \"b\"), strset.New()), strset.New(\"a\", \"b\"))\n\trequire.Equal(t, strset.Union(strset.New(), strset.New(\"a\", \"b\")), strset.New(\"a\", \"b\"))\n\trequire.Equal(t, strset.Union(strset.New(\"a\", \"a\"), strset.New(\"a\", \"b\")), strset.New(\"a\", \"b\"))\n\trequire.Equal(t, strset.Union(strset.New(\"a\"), strset.New(\"b\")), strset.New(\"a\", \"b\"))\n}\n\nfunc TestDifference(t *testing.T) {\n\tset1 := strset.New(\"a\", \"b\", \"c\")\n\tset2 := strset.New(\"z\", \"a\", \"b\")\n\n\td := strset.Difference(set1, set2)\n\trequire.Equal(t, d, strset.New(\"c\"))\n\td = strset.Difference(set2, set1)\n\trequire.Equal(t, d, strset.New(\"z\"))\n\td = strset.Difference(set1, set1)\n\trequire.Equal(t, d, strset.New())\n}\n\nfunc TestIntersection(t *testing.T) {\n\tset1 := strset.New(\"a\", \"b\", \"c\")\n\tset2 := strset.New(\"a\", \"x\", \"y\")\n\tset3 := strset.New(\"z\", \"b\", \"c\")\n\tset4 := strset.New(\"z\", \"b\", \"w\")\n\n\td := strset.Intersection(set1, set2)\n\trequire.Equal(t, d, strset.New(\"a\"))\n\td = strset.Intersection(set1, set3)\n\trequire.Equal(t, d, strset.New(\"b\", \"c\"))\n\td = strset.Intersection(set2, set3)\n\trequire.Equal(t, d, strset.New())\n\td = strset.Intersection(set1, set3, set4)\n\trequire.Equal(t, d, strset.New(\"b\"))\n}\n"
  },
  {
    "path": "pkg/lib/sets/strset/threadsafe/strset.go",
    "content": "/*\nCopyright 2017 ScyllaDB\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage threadsafe\n\nimport (\n\t\"sync\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\n// New creates and initializes a new Set.\ntype Set struct {\n\tsync.RWMutex\n\ts strset.Set\n}\n\nfunc New(ts ...string) *Set {\n\tset := Set{}\n\tset.s = strset.New(ts...)\n\treturn &set\n}\n\nfunc FromSlice(items []string) *Set {\n\treturn New(items...)\n}\n\n// NewWithSize creates a new Set and gives make map a size hint.\nfunc NewWithSize(size int) *Set {\n\tset := Set{}\n\tset.s = strset.NewWithSize(size)\n\treturn &set\n}\n\nfunc (s *Set) Len() int {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn len(s.s)\n}\n\nfunc (s *Set) ToStrset() strset.Set {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.Copy()\n}\n\n// Add includes the specified items (one or more) to the Set. The underlying\n// Set s is modified. If passed nothing it silently returns.\nfunc (s *Set) Add(items ...string) {\n\ts.Lock()\n\tdefer s.Unlock()\n\ts.s.Add(items...)\n}\n\n// Remove deletes the specified items from the Set. The underlying Set s is\n// modified. If passed nothing it silently returns.\nfunc (s *Set) Remove(items ...string) {\n\ts.Lock()\n\tdefer s.Unlock()\n\ts.s.Remove(items...)\n}\n\n// GetOne returns an item from the set or \"\" if the set is empty.\nfunc (s *Set) GetOne() string {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.GetOne()\n}\n\n// GetOne2 returns an item from the set. The second value is a bool that is\n// true if an item exists in the set, or false if the set is empty.\nfunc (s *Set) GetOne2() (string, bool) {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.GetOne2()\n}\n\n// Pop deletes and returns an item from the Set. The underlying Set s is\n// modified. If Set is empty, the zero value is returned.\nfunc (s *Set) Pop() string {\n\ts.Lock()\n\tdefer s.Unlock()\n\treturn s.s.Pop()\n}\n\n// Pop2 tries to delete and return an item from the Set. The underlying Set s\n// is modified. The second value is a bool that is true if the item existed in\n// the set, and false if not. If Set is empty, the zero value and false are\n// returned.\nfunc (s *Set) Pop2() (string, bool) {\n\ts.Lock()\n\tdefer s.Unlock()\n\treturn s.s.Pop2()\n}\n\n// Has looks for the existence of items passed. It returns false if nothing is\n// passed. For multiple items it returns true only if all of the items exist.\nfunc (s *Set) Has(items ...string) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.Has(items...)\n}\n\n// HasAny looks for the existence of any of the items passed.\n// It returns false if nothing is passed.\n// For multiple items it returns true if any of the items exist.\nfunc (s *Set) HasAny(items ...string) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.HasAny(items...)\n}\n\n// Clear removes all items from the Set.\nfunc (s *Set) Clear() {\n\ts.Lock()\n\tdefer s.Unlock()\n\ts.s.Clear()\n}\n\n// IsEqual test whether s and t are the same in size and have the same items.\nfunc (s *Set) IsEqual(t strset.Set) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.IsEqual(t)\n}\n\n// IsEqualThreadsafe test whether s and t are the same in size and have the same items.\nfunc (s *Set) IsEqualThreadsafe(t *Set) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\tt.RLock()\n\tdefer t.RUnlock()\n\treturn s.s.IsEqual(t.s)\n}\n\n// IsSubset tests whether t is a subset of s.\nfunc (s *Set) IsSubset(t strset.Set) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.IsSubset(t)\n}\n\n// IsSubsetThreadsafe tests whether t is a subset of s.\nfunc (s *Set) IsSubsetThreadsafe(t *Set) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\tt.RLock()\n\tdefer t.RUnlock()\n\treturn s.s.IsSubset(t.s)\n}\n\n// IsSuperset tests whether t is a superset of s.\nfunc (s *Set) IsSuperset(t strset.Set) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.IsSuperset(t)\n}\n\n// IsSupersetThreadsafe tests whether t is a superset of s.\nfunc (s *Set) IsSupersetThreadsafe(t *Set) bool {\n\ts.RLock()\n\tdefer s.RUnlock()\n\tt.RLock()\n\tdefer t.RUnlock()\n\treturn s.s.IsSuperset(t.s)\n}\n\n// Copy returns a new Set with a copy of s.\nfunc (s *Set) Copy() strset.Set {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.Copy()\n}\n\n// CopyToThreadsafe returns a new Set with a copy of s.\nfunc (s *Set) CopyToThreadsafe() *Set {\n\ts.RLock()\n\tdefer s.RUnlock()\n\n\tnewSet := Set{}\n\tnewSet.s = s.s.Copy()\n\treturn &newSet\n}\n\n// String returns a string representation of s\nfunc (s *Set) String() string {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.String()\n}\n\n// List returns a slice of all items.\nfunc (s *Set) Slice() []string {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.Slice()\n}\n\n// List returns a sorted slice of all items (a to z).\nfunc (s *Set) SliceSorted() []string {\n\ts.RLock()\n\tdefer s.RUnlock()\n\treturn s.s.SliceSorted()\n}\n\n// Merge is like Union, however it modifies the current Set it's applied on\n// with the given t Set.\nfunc (s *Set) Merge(sets ...strset.Set) {\n\ts.Lock()\n\tdefer s.Unlock()\n\ts.s.Merge(sets...)\n}\n\n// MergeThreadsafe is like UnionThreadsafe, however it modifies the current Set it's applied on\n// with the given t Set.\nfunc (s *Set) MergeThreadsafe(sets ...*Set) {\n\ts.Lock()\n\tdefer s.Unlock()\n\n\tfor _, set := range sets {\n\t\tset.RLock()\n\t\ts.s.Merge(set.s)\n\t\tset.RUnlock()\n\t}\n}\n\n// Subtract removes the Set items contained in sets from Set s\nfunc (s *Set) Subtract(sets ...strset.Set) {\n\ts.Lock()\n\tdefer s.Unlock()\n\ts.s.Subtract(sets...)\n}\n\n// SubtractThreadsafe removes the Set items contained in sets from Set s\nfunc (s *Set) SubtractThreadsafe(sets ...*Set) {\n\ts.Lock()\n\tdefer s.Unlock()\n\n\tfor _, set := range sets {\n\t\tset.RLock()\n\t\ts.s.Subtract(set.s)\n\t\tset.RUnlock()\n\t}\n}\n\n// Remove items until len(s) <= targetLen\nfunc (s *Set) Shrink(targetLen int) {\n\ts.Lock()\n\tdefer s.Unlock()\n\ts.s.Shrink(targetLen)\n}\n\n// remove items alphabetically until len(s) <= targetLen\nfunc (s *Set) ShrinkSorted(targetLen int) {\n\ts.Lock()\n\tdefer s.Unlock()\n\ts.s.ShrinkSorted(targetLen)\n}\n\n// Union is the merger of multiple sets. It returns a new set with all the\n// elements present in all the sets that are passed.\nfunc Union(set1 *Set, sets ...strset.Set) *Set {\n\tfinalSet := set1.CopyToThreadsafe()\n\tfor _, set := range sets {\n\t\tfinalSet.s.Merge(set)\n\t}\n\treturn finalSet\n}\n\n// UnionThreadsafe is the merger of multiple sets. It returns a new set with all the\n// elements present in all the sets that are passed.\nfunc UnionThreadsafe(sets ...*Set) *Set {\n\tfinalSet := New()\n\tfor _, set := range sets {\n\t\tset.RLock()\n\t\tfinalSet.s.Merge(set.s)\n\t\tset.RUnlock()\n\t}\n\treturn finalSet\n}\n\n// Difference returns a new set which contains items which are in the first\n// set but not in the others.\nfunc Difference(set1 *Set, sets ...strset.Set) *Set {\n\ts := set1.CopyToThreadsafe()\n\tfor _, set := range sets {\n\t\ts.s.Subtract(set)\n\t}\n\treturn s\n}\n\n// DifferenceThreadsafe returns a new set which contains items which are in in the first\n// set but not in the others.\nfunc DifferenceThreadsafe(set1 *Set, sets ...*Set) *Set {\n\ts := set1.CopyToThreadsafe()\n\tfor _, set := range sets {\n\t\tset.RLock()\n\t\ts.s.Subtract(set.s)\n\t\tset.RUnlock()\n\t}\n\treturn s\n}\n\n// Intersection returns a new set which contains items that only exist in all\n// given sets.\nfunc Intersection(set1 *Set, sets ...strset.Set) *Set {\n\tt := set1.CopyToThreadsafe()\n\tfor _, set := range sets {\n\t\tfor item := range t.s {\n\t\t\tif _, has := set[item]; !has {\n\t\t\t\tdelete(t.s, item)\n\t\t\t}\n\t\t}\n\t}\n\treturn t\n}\n\n// IntersectionThreadsafe returns a new set which contains items that only exist in all\n// given sets.\nfunc IntersectionThreadsafe(set1 *Set, sets ...*Set) *Set {\n\tt := set1.CopyToThreadsafe()\n\tfor _, set := range sets {\n\t\tset.RLock()\n\t\tfor item := range t.s {\n\t\t\tif _, has := set.s[item]; !has {\n\t\t\t\tdelete(t.s, item)\n\t\t\t}\n\t\t}\n\t\tset.RUnlock()\n\t}\n\treturn t\n}\n\n// SymmetricDifferenceThreadsafe returns a new set which s is the difference of items\n// which are in one of either, but not in both.\nfunc SymmetricDifferenceThreadsafe(s *Set, t *Set) *Set {\n\tu := DifferenceThreadsafe(s, t)\n\tv := DifferenceThreadsafe(t, s)\n\treturn UnionThreadsafe(u, v)\n}\n"
  },
  {
    "path": "pkg/lib/sets/strset/threadsafe/strset_test.go",
    "content": "/*\nCopyright 2017 ScyllaDB\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage threadsafe_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset/threadsafe\"\n\t\"github.com/stretchr/testify/require\"\n)\n\n// Also tests Add\nfunc TestNew(t *testing.T) {\n\tset := threadsafe.New()\n\trequire.Equal(t, 0, set.Len())\n\n\tset = threadsafe.New(\"a\", \"b\", \"a\")\n\trequire.Equal(t, 2, set.Len())\n\tif !set.Has(\"a\") {\n\t\trequire.FailNow(t, \"a not found in set\")\n\t}\n\tif !set.Has(\"b\") {\n\t\trequire.FailNow(t, \"b not found in set\")\n\t}\n}\n\nfunc TestAdd(t *testing.T) {\n\tset := threadsafe.New()\n\tset.Add(\"a\")\n\tset.Add(\"b\", \"c\")\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\", \"c\"))\n}\n\nfunc TestRemove(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\")\n\tset.Remove(\"c\")\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\"))\n\n\tset.Remove()\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\"))\n\n\tset.Remove(\"a\")\n\trequire.Equal(t, set, threadsafe.New(\"b\"))\n\n\tset.Add(\"a\")\n\tset.Remove(\"a\", \"b\")\n\trequire.Equal(t, set, threadsafe.New())\n}\n\nfunc TestPop(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\")\n\tp := set.Pop()\n\trequire.Contains(t, []string{\"a\", \"b\"}, p)\n\trequire.Equal(t, 1, set.Len())\n\tp = set.Pop()\n\trequire.Contains(t, []string{\"a\", \"b\"}, p)\n\trequire.Equal(t, 0, set.Len())\n\tp = set.Pop()\n\trequire.Equal(t, \"\", p)\n\trequire.Equal(t, 0, set.Len())\n}\n\nfunc TestPop2(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\")\n\tp, ok := set.Pop2()\n\trequire.Contains(t, []string{\"a\", \"b\"}, p)\n\trequire.Equal(t, 1, set.Len())\n\trequire.True(t, ok)\n\tp, ok = set.Pop2()\n\trequire.Contains(t, []string{\"a\", \"b\"}, p)\n\trequire.Equal(t, 0, set.Len())\n\trequire.True(t, ok)\n\tp, ok = set.Pop2()\n\trequire.Equal(t, \"\", p)\n\trequire.Equal(t, 0, set.Len())\n\trequire.False(t, ok)\n}\n\nfunc TestHas(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\", \"c\")\n\trequire.True(t, set.Has(\"a\"))\n\trequire.True(t, set.Has(\"a\", \"b\"))\n\trequire.False(t, set.Has(\"z\"))\n\trequire.False(t, set.Has(\"a\", \"z\"))\n}\n\nfunc TestHasAny(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\", \"c\")\n\trequire.True(t, set.HasAny(\"a\"))\n\trequire.True(t, set.HasAny(\"a\", \"b\"))\n\trequire.False(t, set.HasAny(\"z\"))\n\trequire.True(t, set.HasAny(\"a\", \"z\"))\n}\n\nfunc TestClear(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\", \"c\")\n\trequire.Equal(t, 3, set.Len())\n\tset.Clear()\n\trequire.Equal(t, 0, set.Len())\n}\n\nfunc TestIsEqual(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := strset.New(\"a\", \"b\", \"c\")\n\tset3 := strset.New(\"a\", \"b\")\n\tset4 := strset.New(\"a\", \"b\", \"z\")\n\trequire.True(t, set1.IsEqual(set2))\n\trequire.False(t, set1.IsEqual(set3))\n\trequire.False(t, set1.IsEqual(set4))\n}\n\nfunc IsEqualThreadsafe(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset3 := threadsafe.New(\"a\", \"b\")\n\tset4 := threadsafe.New(\"a\", \"b\", \"z\")\n\trequire.True(t, set1.IsEqualThreadsafe(set2))\n\trequire.False(t, set1.IsEqualThreadsafe(set3))\n\trequire.False(t, set1.IsEqualThreadsafe(set4))\n}\n\nfunc TestIsSubset(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := strset.New(\"a\", \"b\", \"c\")\n\tset3 := strset.New(\"a\", \"b\")\n\tset4 := strset.New(\"a\", \"b\", \"z\")\n\tset5 := strset.New(\"a\", \"b\", \"c\", \"d\")\n\trequire.True(t, set1.IsSubset(set2))\n\trequire.True(t, set1.IsSubset(set3))\n\trequire.False(t, set1.IsSubset(set4))\n\trequire.False(t, set1.IsSubset(set5))\n}\n\nfunc IsSubsetThreadsafe(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset3 := threadsafe.New(\"a\", \"b\")\n\tset4 := threadsafe.New(\"a\", \"b\", \"z\")\n\tset5 := threadsafe.New(\"a\", \"b\", \"c\", \"d\")\n\trequire.True(t, set1.IsSubsetThreadsafe(set2))\n\trequire.True(t, set1.IsSubsetThreadsafe(set3))\n\trequire.False(t, set1.IsSubsetThreadsafe(set4))\n\trequire.False(t, set1.IsSubsetThreadsafe(set5))\n}\n\nfunc TestIsSuperset(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := strset.New(\"a\", \"b\", \"c\")\n\tset3 := strset.New(\"a\", \"b\")\n\tset4 := strset.New(\"a\", \"b\", \"z\")\n\tset5 := strset.New(\"a\", \"b\", \"c\", \"d\")\n\trequire.True(t, set1.IsSuperset(set2))\n\trequire.False(t, set1.IsSuperset(set3))\n\trequire.False(t, set1.IsSuperset(set4))\n\trequire.True(t, set1.IsSuperset(set5))\n}\n\nfunc IsSupersetThreadsafe(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset3 := threadsafe.New(\"a\", \"b\")\n\tset4 := threadsafe.New(\"a\", \"b\", \"z\")\n\tset5 := threadsafe.New(\"a\", \"b\", \"c\", \"d\")\n\trequire.True(t, set1.IsSupersetThreadsafe(set2))\n\trequire.False(t, set1.IsSupersetThreadsafe(set3))\n\trequire.False(t, set1.IsSupersetThreadsafe(set4))\n\trequire.True(t, set1.IsSupersetThreadsafe(set5))\n}\n\nfunc TestCopy(t *testing.T) {\n\tset := threadsafe.New()\n\tcset := set.Copy()\n\trequire.Equal(t, 0, len(cset))\n\n\tset = threadsafe.New(\"a\", \"b\")\n\tcset = set.Copy()\n\trequire.Equal(t, 2, len(cset))\n\tif !set.Has(\"a\") {\n\t\trequire.FailNow(t, \"a not found in set\")\n\t}\n\tif !set.Has(\"b\") {\n\t\trequire.FailNow(t, \"b not found in set\")\n\t}\n}\n\nfunc TestCopyToThreadsafe(t *testing.T) {\n\tset := threadsafe.New()\n\tcset := set.CopyToThreadsafe()\n\trequire.Equal(t, 0, cset.Len())\n\n\tset = threadsafe.New(\"a\", \"b\")\n\tcset = set.CopyToThreadsafe()\n\trequire.Equal(t, 2, cset.Len())\n\tif !set.Has(\"a\") {\n\t\trequire.FailNow(t, \"a not found in set\")\n\t}\n\tif !set.Has(\"b\") {\n\t\trequire.FailNow(t, \"b not found in set\")\n\t}\n}\n\nfunc TestSlice(t *testing.T) {\n\tset := threadsafe.New()\n\trequire.Equal(t, set.Slice(), []string{})\n\n\tset.Add(\"a\")\n\trequire.Equal(t, set.Slice(), []string{\"a\"})\n\n\tset.Add(\"a\", \"b\")\n\trequire.ElementsMatch(t, set.Slice(), []string{\"a\", \"b\"})\n}\n\nfunc TestMerge(t *testing.T) {\n\tset := threadsafe.New()\n\temptySet := strset.New()\n\tset.Merge(emptySet)\n\trequire.Equal(t, 0, set.Len())\n\n\tset.Merge(strset.New(\"a\"))\n\trequire.Equal(t, 1, set.Len())\n\n\tset.Merge(emptySet)\n\trequire.Equal(t, 1, set.Len())\n\n\tset.Add(\"a\", \"b\", \"c\")\n\tset.Merge(strset.New(\"e\", \"e\", \"d\"))\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\", \"c\", \"e\", \"d\"))\n\n\tset.Merge(strset.New(\"a\", \"e\"))\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\", \"c\", \"e\", \"d\"))\n\tset.Merge(strset.New(\"a\", \"e\", \"i\"), strset.New(\"o\", \"u\"), strset.New(\"sometimes y\"))\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\", \"c\", \"e\", \"d\", \"i\", \"o\", \"u\", \"sometimes y\"))\n}\n\nfunc TestMergeThreadsafe(t *testing.T) {\n\tset := threadsafe.New()\n\temptySet := threadsafe.New()\n\tset.MergeThreadsafe(emptySet)\n\trequire.Equal(t, 0, set.Len())\n\n\tset.MergeThreadsafe(threadsafe.New(\"a\"))\n\trequire.Equal(t, 1, set.Len())\n\n\tset.MergeThreadsafe(emptySet)\n\trequire.Equal(t, 1, set.Len())\n\n\tset.Add(\"a\", \"b\", \"c\")\n\tset.MergeThreadsafe(threadsafe.New(\"e\", \"e\", \"d\"))\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\", \"c\", \"e\", \"d\"))\n\n\tset.MergeThreadsafe(threadsafe.New(\"a\", \"e\"))\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\", \"c\", \"e\", \"d\"))\n\tset.MergeThreadsafe(threadsafe.New(\"a\", \"e\", \"i\"), threadsafe.New(\"o\", \"u\"), threadsafe.New(\"sometimes y\"))\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\", \"c\", \"e\", \"d\", \"i\", \"o\", \"u\", \"sometimes y\"))\n}\n\nfunc TestSubtract(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\", \"c\")\n\tset.Subtract(strset.New(\"z\", \"a\", \"b\"))\n\trequire.Equal(t, set, threadsafe.New(\"c\"))\n\tset.Subtract(strset.New(\"x\"))\n\trequire.Equal(t, set, threadsafe.New(\"c\"))\n}\n\nfunc SubtractThreadsafe(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\", \"c\")\n\tset.SubtractThreadsafe(threadsafe.New(\"z\", \"a\", \"b\"))\n\trequire.Equal(t, set, threadsafe.New(\"c\"))\n\tset.SubtractThreadsafe(threadsafe.New(\"x\"))\n\trequire.Equal(t, set, threadsafe.New(\"c\"))\n}\n\nfunc TestShrink(t *testing.T) {\n\tset := threadsafe.New(\"a\", \"b\", \"c\", \"d\")\n\tset.Shrink(2)\n\trequire.Equal(t, set.Len(), 2)\n\n\tset = threadsafe.New(\"g\", \"f\", \"e\", \"d\", \"c\", \"b\", \"a\")\n\tset.ShrinkSorted(3)\n\trequire.Equal(t, set.Len(), 3)\n\n\tset = threadsafe.New(\"a\", \"b\")\n\tset.Shrink(2)\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\"))\n\n\tset = threadsafe.New(\"a\")\n\tset.Shrink(2)\n\trequire.Equal(t, set, threadsafe.New(\"a\"))\n\n\tset = threadsafe.New()\n\tset.Shrink(2)\n\trequire.Equal(t, set.Len(), 0)\n}\n\nfunc TestShrinkSorted(t *testing.T) {\n\tfor i := 0; i < 10; i++ {\n\t\tset := threadsafe.New(\"g\", \"f\", \"e\", \"d\", \"c\", \"b\", \"a\")\n\t\tset.ShrinkSorted(2)\n\t\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\"))\n\n\t\tset = threadsafe.New(\"g\", \"f\", \"e\", \"d\", \"c\", \"b\", \"a\")\n\t\tset.ShrinkSorted(3)\n\t\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\", \"c\"))\n\t}\n\n\tset := threadsafe.New(\"a\", \"b\")\n\tset.ShrinkSorted(2)\n\trequire.Equal(t, set, threadsafe.New(\"a\", \"b\"))\n\n\tset = threadsafe.New(\"a\")\n\tset.ShrinkSorted(2)\n\trequire.Equal(t, set, threadsafe.New(\"a\"))\n\n\tset = threadsafe.New()\n\tset.ShrinkSorted(2)\n\trequire.Equal(t, set.Len(), 0)\n}\n\nfunc TestUnion(t *testing.T) {\n\trequire.Equal(t, threadsafe.Union(threadsafe.New(), strset.New()).Len(), 0)\n\trequire.Equal(t, threadsafe.Union(threadsafe.New(\"a\", \"b\"), strset.New()), threadsafe.New(\"a\", \"b\"))\n\trequire.Equal(t, threadsafe.Union(threadsafe.New(), strset.New(\"a\", \"b\")), threadsafe.New(\"a\", \"b\"))\n\trequire.Equal(t, threadsafe.Union(threadsafe.New(\"a\", \"a\"), strset.New(\"a\", \"b\")), threadsafe.New(\"a\", \"b\"))\n\trequire.Equal(t, threadsafe.Union(threadsafe.New(\"a\"), strset.New(\"b\")), threadsafe.New(\"a\", \"b\"))\n}\n\nfunc TestUnionThreadsafe(t *testing.T) {\n\trequire.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New(), threadsafe.New()).Len(), 0)\n\trequire.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New(\"a\", \"b\"), threadsafe.New()), threadsafe.New(\"a\", \"b\"))\n\trequire.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New(), threadsafe.New(\"a\", \"b\")), threadsafe.New(\"a\", \"b\"))\n\trequire.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New(\"a\", \"a\"), threadsafe.New(\"a\", \"b\")), threadsafe.New(\"a\", \"b\"))\n\trequire.Equal(t, threadsafe.UnionThreadsafe(threadsafe.New(\"a\"), threadsafe.New(\"b\")), threadsafe.New(\"a\", \"b\"))\n}\n\nfunc TestDifference(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset1Strset := strset.New(\"a\", \"b\", \"c\")\n\tset2 := threadsafe.New(\"z\", \"a\", \"b\")\n\tset2Strset := strset.New(\"z\", \"a\", \"b\")\n\n\td := threadsafe.Difference(set1, set2Strset)\n\trequire.Equal(t, d, threadsafe.New(\"c\"))\n\td = threadsafe.Difference(set2, set1Strset)\n\trequire.Equal(t, d, threadsafe.New(\"z\"))\n\td = threadsafe.Difference(set1, set1Strset)\n\trequire.Equal(t, d, threadsafe.New())\n}\n\nfunc TestDifferenceThreadsafe(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := threadsafe.New(\"z\", \"a\", \"b\")\n\n\td := threadsafe.DifferenceThreadsafe(set1, set2)\n\trequire.Equal(t, d, threadsafe.New(\"c\"))\n\td = threadsafe.DifferenceThreadsafe(set2, set1)\n\trequire.Equal(t, d, threadsafe.New(\"z\"))\n\td = threadsafe.DifferenceThreadsafe(set1, set1)\n\trequire.Equal(t, d, threadsafe.New())\n}\n\nfunc TestIntersection(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := threadsafe.New(\"a\", \"x\", \"y\")\n\tset2Strset := strset.New(\"a\", \"x\", \"y\")\n\tset3Strset := strset.New(\"z\", \"b\", \"c\")\n\tset4Strset := strset.New(\"z\", \"b\", \"w\")\n\n\td := threadsafe.Intersection(set1, set2Strset)\n\trequire.Equal(t, d, threadsafe.New(\"a\"))\n\td = threadsafe.Intersection(set1, set3Strset)\n\trequire.Equal(t, d, threadsafe.New(\"b\", \"c\"))\n\td = threadsafe.Intersection(set2, set3Strset)\n\trequire.Equal(t, d, threadsafe.New())\n\td = threadsafe.Intersection(set1, set3Strset, set4Strset)\n\trequire.Equal(t, d, threadsafe.New(\"b\"))\n}\n\nfunc TestIntersectionThreadsafe(t *testing.T) {\n\tset1 := threadsafe.New(\"a\", \"b\", \"c\")\n\tset2 := threadsafe.New(\"a\", \"x\", \"y\")\n\tset3 := threadsafe.New(\"z\", \"b\", \"c\")\n\tset4 := threadsafe.New(\"z\", \"b\", \"w\")\n\n\td := threadsafe.IntersectionThreadsafe(set1, set2)\n\trequire.Equal(t, d, threadsafe.New(\"a\"))\n\td = threadsafe.IntersectionThreadsafe(set1, set3)\n\trequire.Equal(t, d, threadsafe.New(\"b\", \"c\"))\n\td = threadsafe.IntersectionThreadsafe(set2, set3)\n\trequire.Equal(t, d, threadsafe.New())\n\td = threadsafe.IntersectionThreadsafe(set1, set3, set4)\n\trequire.Equal(t, d, threadsafe.New(\"b\"))\n}\n"
  },
  {
    "path": "pkg/lib/slices/bool.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nfunc HasTrue(list []bool) bool {\n\tfor _, elem := range list {\n\t\tif elem {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n"
  },
  {
    "path": "pkg/lib/slices/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrLenValuesWeightsMismatch = \"slices.len_values_weights_mismatch\"\n)\n\nfunc ErrorLenValuesWeightsMismatch() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrLenValuesWeightsMismatch,\n\t\tMessage: \"length of values is not equal to length of weights\",\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/slices/float32.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nfunc HasFloat32(list []float32, query float32) bool {\n\tfor _, elem := range list {\n\t\tif elem == query {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc CopyFloat32s(vals []float32) []float32 {\n\treturn append(vals[:0:0], vals...)\n}\n\nfunc Float32ToString(vals []float32) []string {\n\tstringSlice := []string{}\n\tfor _, elem := range vals {\n\t\tstringSlice = append(stringSlice, s.Float32(elem))\n\t}\n\treturn stringSlice\n}\n"
  },
  {
    "path": "pkg/lib/slices/float64.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nfunc HasFloat64(list []float64, query float64) bool {\n\tfor _, elem := range list {\n\t\tif elem == query {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc CopyFloat64s(vals []float64) []float64 {\n\treturn append(vals[:0:0], vals...)\n}\n\nfunc Float64ToString(vals []float64) []string {\n\tstringSlice := []string{}\n\tfor _, elem := range vals {\n\t\tstringSlice = append(stringSlice, s.Float64(elem))\n\t}\n\treturn stringSlice\n}\n"
  },
  {
    "path": "pkg/lib/slices/float64_ptr.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\n// For adding integers stored in floats\nfunc Float64PtrSumInt(floats ...*float64) int {\n\tsum := 0\n\tfor _, num := range floats {\n\t\tif num != nil {\n\t\t\tsum += int(*num)\n\t\t}\n\t}\n\treturn sum\n}\n\nfunc Float64PtrMin(floats ...*float64) *float64 {\n\tvar min *float64\n\n\tfor _, num := range floats {\n\t\tswitch {\n\t\tcase num != nil && min != nil && *num < *min:\n\t\t\tmin = num\n\t\tcase num != nil && min == nil:\n\t\t\tmin = num\n\t\t}\n\t}\n\treturn min\n}\n\nfunc Float64PtrMax(floats ...*float64) *float64 {\n\tvar max *float64\n\n\tfor _, num := range floats {\n\t\tswitch {\n\t\tcase num != nil && max != nil && *num > *max:\n\t\t\tmax = num\n\t\tcase num != nil && max == nil:\n\t\t\tmax = num\n\t\t}\n\t}\n\treturn max\n}\n\nfunc Float64PtrAvg(values []*float64, weights []*float64) (*float64, error) {\n\tif len(values) != len(weights) {\n\t\treturn nil, ErrorLenValuesWeightsMismatch()\n\t}\n\n\ttotalWeight := 0.0\n\tfor i, valPtr := range values {\n\t\tif valPtr != nil && weights[i] != nil && *weights[i] > 0 {\n\t\t\ttotalWeight += *weights[i]\n\t\t}\n\t}\n\n\tif totalWeight == 0.0 {\n\t\treturn nil, nil\n\t}\n\n\tavg := 0.0\n\tfor i, valPtr := range values {\n\t\tif valPtr != nil && weights[i] != nil && *weights[i] > 0 {\n\t\t\tavg += (*valPtr) * (*weights[i]) / float64(totalWeight)\n\t\t}\n\t}\n\n\treturn &avg, nil\n}\n"
  },
  {
    "path": "pkg/lib/slices/float64_ptr_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nvar _float64NilPtr = (*float64)(nil)\n\nfunc TestFloat64PtrSumInt(t *testing.T) {\n\trequire.Equal(t, 0, Float64PtrSumInt(nil))\n\trequire.Equal(t, 1, Float64PtrSumInt(pointer.Float64(1)))\n\trequire.Equal(t, 2, Float64PtrSumInt(pointer.Float64(1), pointer.Float64(1.5)))\n}\n\nfunc TestFloat64PtrMin(t *testing.T) {\n\trequire.Equal(t, _float64NilPtr, Float64PtrMin())\n\trequire.Equal(t, _float64NilPtr, Float64PtrMin(nil))\n\trequire.Equal(t, pointer.Float64(1), Float64PtrMin(pointer.Float64(1)))\n\trequire.Equal(t, pointer.Float64(-1), Float64PtrMin(_float64NilPtr, pointer.Float64(1), pointer.Float64(-1)))\n}\n\nfunc TestFloat64PtrMax(t *testing.T) {\n\trequire.Equal(t, _float64NilPtr, Float64PtrMax())\n\trequire.Equal(t, _float64NilPtr, Float64PtrMax(nil))\n\trequire.Equal(t, pointer.Float64(1), Float64PtrMax(pointer.Float64(1)))\n\trequire.Equal(t, pointer.Float64(1.5), Float64PtrMax(pointer.Float64(1), pointer.Float64(1.5), _float64NilPtr))\n}\n\nfunc TestFloat64PtrAvg(t *testing.T) {\n\tvar err error\n\tvar avg *float64\n\n\tavg, err = Float64PtrAvg([]*float64{pointer.Float64(1)}, []*float64{pointer.Float64(10)})\n\trequire.Equal(t, pointer.Float64(1), avg)\n\trequire.NoError(t, err)\n\n\tavg, err = Float64PtrAvg([]*float64{pointer.Float64(10)}, []*float64{pointer.Float64(1)})\n\trequire.Equal(t, pointer.Float64(10), avg)\n\trequire.NoError(t, err)\n\n\tavg, err = Float64PtrAvg([]*float64{pointer.Float64(1), pointer.Float64(4), _float64NilPtr}, []*float64{pointer.Float64(2), pointer.Float64(1), pointer.Float64(1)})\n\trequire.Equal(t, pointer.Float64(2), avg)\n\trequire.NoError(t, err)\n\n\tavg, err = Float64PtrAvg([]*float64{pointer.Float64(1), pointer.Float64(4), pointer.Float64(1)}, []*float64{pointer.Float64(2), pointer.Float64(1), _float64NilPtr})\n\trequire.Equal(t, pointer.Float64(2), avg)\n\trequire.NoError(t, err)\n\n\tavg, err = Float64PtrAvg([]*float64{pointer.Float64(1), pointer.Float64(4), _float64NilPtr}, []*float64{pointer.Float64(2), pointer.Float64(1), _float64NilPtr})\n\trequire.Equal(t, pointer.Float64(2), avg)\n\trequire.NoError(t, err)\n\n\tavg, err = Float64PtrAvg([]*float64{pointer.Float64(1)}, []*float64{pointer.Float64(2), pointer.Float64(1)})\n\trequire.Equal(t, _float64NilPtr, avg)\n\trequire.Error(t, err)\n\n\tavg, err = Float64PtrAvg([]*float64{pointer.Float64(2)}, []*float64{pointer.Float64(0)})\n\trequire.Equal(t, _float64NilPtr, avg)\n\trequire.NoError(t, err)\n\n\tavg, err = Float64PtrAvg([]*float64{pointer.Float64(0)}, []*float64{pointer.Float64(2)})\n\trequire.Equal(t, pointer.Float64(0), avg)\n\trequire.NoError(t, err)\n\n\tavg, err = Float64PtrAvg([]*float64{nil}, []*float64{pointer.Float64(2)})\n\trequire.Equal(t, _float64NilPtr, avg)\n\trequire.NoError(t, err)\n\n}\n"
  },
  {
    "path": "pkg/lib/slices/int.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nfunc HasInt(list []int, query int) bool {\n\tfor _, elem := range list {\n\t\tif elem == query {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc CopyInts(vals []int) []int {\n\treturn append(vals[:0:0], vals...)\n}\n\nfunc AreNGreaterThanZero(minCount int, val int, vals ...int) bool {\n\tcount := 0\n\tallVals := append(vals, val)\n\tfor _, val := range allVals {\n\t\tif val > 0 {\n\t\t\tcount++\n\t\t\tif count >= minCount {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc IntToString(vals []int) []string {\n\tstringSlice := []string{}\n\tfor _, elem := range vals {\n\t\tstringSlice = append(stringSlice, s.Int(elem))\n\t}\n\treturn stringSlice\n}\n"
  },
  {
    "path": "pkg/lib/slices/int32.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nfunc HasInt32(list []int32, query int32) bool {\n\tfor _, elem := range list {\n\t\tif elem == query {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc CopyInt32s(vals []int32) []int32 {\n\treturn append(vals[:0:0], vals...)\n}\n\nfunc Int32ToString(vals []int32) []string {\n\tstringSlice := []string{}\n\tfor _, elem := range vals {\n\t\tstringSlice = append(stringSlice, s.Int32(elem))\n\t}\n\treturn stringSlice\n}\n"
  },
  {
    "path": "pkg/lib/slices/int64.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nfunc HasInt64(list []int64, query int64) bool {\n\tfor _, elem := range list {\n\t\tif elem == query {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc CopyInt64s(vals []int64) []int64 {\n\treturn append(vals[:0:0], vals...)\n}\n\nfunc UniqueInt64(vals []int64) []int64 {\n\tkeys := make(map[int64]bool)\n\tlist := []int64{}\n\tfor _, entry := range vals {\n\t\tif _, value := keys[entry]; !value {\n\t\t\tkeys[entry] = true\n\t\t\tlist = append(list, entry)\n\t\t}\n\t}\n\treturn list\n}\n\nfunc Int64ToString(vals []int64) []string {\n\tstringSlice := []string{}\n\tfor _, elem := range vals {\n\t\tstringSlice = append(stringSlice, s.Int64(elem))\n\t}\n\treturn stringSlice\n}\n"
  },
  {
    "path": "pkg/lib/slices/sort.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\t\"math\"\n\t\"sort\"\n)\n\n// Inspiration: https://golang.org/src/sort/sort.go\n\ntype Int32Slice []int32\n\nfunc (p Int32Slice) Len() int           { return len(p) }\nfunc (p Int32Slice) Less(i, j int) bool { return p[i] < p[j] }\nfunc (p Int32Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }\nfunc SortInt32s(a []int32)              { sort.Sort(Int32Slice(a)) }\n\ntype Int64Slice []int64\n\nfunc (p Int64Slice) Len() int           { return len(p) }\nfunc (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] }\nfunc (p Int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }\nfunc SortInt64s(a []int64)              { sort.Sort(Int64Slice(a)) }\n\ntype Float32Slice []float32\n\nfunc (p Float32Slice) Len() int { return len(p) }\nfunc (p Float32Slice) Less(i, j int) bool {\n\treturn p[i] < p[j] || math.IsNaN(float64(p[i])) && !math.IsNaN(float64(p[j]))\n}\nfunc (p Float32Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }\nfunc SortFloat32s(a []float32)       { sort.Sort(Float32Slice(a)) }\n\n// Sort with copying\n\nfunc SortStrsCopy(a []string) []string {\n\taCopy := CopyStrings(a)\n\tsort.Strings(aCopy)\n\treturn aCopy\n}\n\nfunc SortIntsCopy(a []int) []int {\n\taCopy := CopyInts(a)\n\tsort.Ints(aCopy)\n\treturn aCopy\n}\n\nfunc SortInt32sCopy(a []int32) []int32 {\n\taCopy := CopyInt32s(a)\n\tSortInt32s(aCopy)\n\treturn aCopy\n}\n\nfunc SortInt64sCopy(a []int64) []int64 {\n\taCopy := CopyInt64s(a)\n\tSortInt64s(aCopy)\n\treturn aCopy\n}\n\nfunc SortFloat32sCopy(a []float32) []float32 {\n\taCopy := CopyFloat32s(a)\n\tSortFloat32s(aCopy)\n\treturn aCopy\n}\n\nfunc SortFloat64sCopy(a []float64) []float64 {\n\taCopy := CopyFloat64s(a)\n\tsort.Float64s(aCopy)\n\treturn aCopy\n}\n"
  },
  {
    "path": "pkg/lib/slices/string.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices\n\nimport (\n\t\"strconv\"\n\n\tlibmath \"github.com/cortexlabs/cortex/pkg/lib/math\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\n// HasString checks if a string slice contains a target string\nfunc HasString(list []string, query string) bool {\n\tfor _, elem := range list {\n\t\tif elem == query {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// HasAnyStrings checks if a string slice contains any string from the query string slice\nfunc HasAnyStrings(queries []string, list []string) bool {\n\tkeys := strset.New()\n\tfor _, elem := range queries {\n\t\tkeys.Add(elem)\n\t}\n\tfor _, elem := range list {\n\t\tif keys.Has(elem) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// HasAllStrings checks if a string slice contains all the strings from the query string slice\nfunc HasAllStrings(queries []string, list []string) bool {\n\tkeys := strset.New()\n\tfor _, elem := range list {\n\t\tkeys.Add(elem)\n\t}\n\tfor _, elem := range queries {\n\t\tif !keys.Has(elem) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc CopyStrings(vals []string) []string {\n\treturn append(vals[:0:0], vals...)\n}\n\nfunc UniqueStrings(strs []string) []string {\n\tkeys := strset.New()\n\tout := []string{}\n\tfor _, elem := range strs {\n\t\tif !keys.Has(elem) {\n\t\t\tkeys.Add(elem)\n\t\t\tout = append(out, elem)\n\t\t}\n\t}\n\treturn out\n}\n\nfunc RemoveEmpties(strs []string) []string {\n\tvar cleanStrs []string\n\tfor _, str := range strs {\n\t\tif str != \"\" {\n\t\t\tcleanStrs = append(cleanStrs, str)\n\t\t}\n\t}\n\treturn cleanStrs\n}\n\nfunc RemoveEmptiesAndUnique(strs []string) []string {\n\tkeys := strset.New()\n\tout := []string{}\n\tfor _, elem := range strs {\n\t\tif elem != \"\" {\n\t\t\tif !keys.Has(elem) {\n\t\t\t\tkeys.Add(elem)\n\t\t\t\tout = append(out, elem)\n\t\t\t}\n\t\t}\n\t}\n\treturn out\n}\n\n// RemoveString removes a target string from a string slice if it exists\nfunc RemoveString(strs []string, target string) []string {\n\tvar result []string\n\tfor _, item := range strs {\n\t\tif item == target {\n\t\t\tcontinue\n\t\t}\n\t\tresult = append(result, item)\n\t}\n\treturn result\n}\n\nfunc HasDuplicateStr(in []string) bool {\n\tkeys := strset.New()\n\tfor _, elem := range in {\n\t\tif keys.Has(elem) {\n\t\t\treturn true\n\t\t}\n\t\tkeys.Add(elem)\n\t}\n\treturn false\n}\n\nfunc FindDuplicateStrs(in []string) []string {\n\tdups := []string{}\n\tkeys := strset.New()\n\tfor _, elem := range in {\n\t\tif keys.Has(elem) {\n\t\t\tdups = append(dups, elem)\n\t\t}\n\t\tkeys.Add(elem)\n\t}\n\treturn dups\n}\n\nfunc SubtractStrSlice(slice1 []string, slice2 []string) []string {\n\tresult := []string{}\n\tfor _, elem := range slice1 {\n\t\tif !HasString(slice2, elem) {\n\t\t\tresult = append(result, elem)\n\t\t}\n\t}\n\treturn result\n}\n\nfunc StrSliceElementsMatch(strs1 []string, strs2 []string) bool {\n\tif len(strs1) == 0 && len(strs2) == 0 {\n\t\treturn true\n\t}\n\tif len(strs1) != len(strs2) {\n\t\treturn false\n\t}\n\treturn StrSlicesEqual(SortStrsCopy(strs1), SortStrsCopy(strs2))\n}\n\nfunc StrSlicesEqual(strs1 []string, strs2 []string) bool {\n\tif len(strs1) == 0 && len(strs2) == 0 {\n\t\treturn true\n\t}\n\tif len(strs1) != len(strs2) {\n\t\treturn false\n\t}\n\tfor i := range strs1 {\n\t\tif strs1[i] != strs2[i] {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc FilterStrs(strs []string, filterFn func(string) bool) []string {\n\tout := []string{}\n\tfor _, elem := range strs {\n\t\tif filterFn(elem) {\n\t\t\tout = append(out, elem)\n\t\t}\n\t}\n\treturn out\n}\n\nfunc MapStrs(strs []string, mapFn func(string) string) []string {\n\tout := make([]string, len(strs))\n\tfor i, elem := range strs {\n\t\tout[i] = mapFn(elem)\n\t}\n\treturn out\n}\n\nfunc MergeStrSlices(slices ...[]string) []string {\n\tif slices == nil || len(slices) == 0 {\n\t\treturn nil\n\t}\n\n\tvar totalLen int\n\tfor _, s := range slices {\n\t\ttotalLen += len(s)\n\t}\n\n\tresult := make([]string, totalLen)\n\tvar i int\n\tfor _, s := range slices {\n\t\ti += copy(result[i:], s)\n\t}\n\treturn result\n}\n\nfunc ZipStrsToMap(strs1 []string, strs2 []string) map[string]string {\n\tstrMap := map[string]string{}\n\tlength := libmath.MinInt(len(strs1), len(strs2))\n\tfor i := 0; i < length; i++ {\n\t\tstrMap[strs1[i]] = strs2[i]\n\t}\n\treturn strMap\n}\n\nfunc StringToInt(vals []string) ([]int, error) {\n\tintSlice := []int{}\n\tfor _, elem := range vals {\n\t\ti, err := strconv.Atoi(elem)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tintSlice = append(intSlice, i)\n\t}\n\treturn intSlice, nil\n}\n\nfunc StringToInt32(vals []string) ([]int32, error) {\n\tintSlice := []int32{}\n\tfor _, elem := range vals {\n\t\ti, err := strconv.ParseInt(elem, 10, 32)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tintSlice = append(intSlice, int32(i))\n\t}\n\treturn intSlice, nil\n}\n\nfunc StringToInt64(vals []string) ([]int64, error) {\n\tintSlice := []int64{}\n\tfor _, elem := range vals {\n\t\ti, err := strconv.ParseInt(elem, 10, 64)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tintSlice = append(intSlice, i)\n\t}\n\treturn intSlice, nil\n}\n"
  },
  {
    "path": "pkg/lib/slices/string_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage slices_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestStrSliceElementsMatch(t *testing.T) {\n\tvar strs1 []string\n\tvar strs2 []string\n\n\tstrs1 = []string{}\n\tstrs2 = []string{}\n\trequire.True(t, slices.StrSliceElementsMatch(strs1, strs2))\n\n\tstrs1 = []string{\"1\"}\n\tstrs2 = []string{\"1\"}\n\trequire.True(t, slices.StrSliceElementsMatch(strs1, strs2))\n\n\tstrs1 = []string{\"1\", \"2\", \"3\"}\n\tstrs2 = []string{\"1\", \"2\", \"3\"}\n\trequire.True(t, slices.StrSliceElementsMatch(strs1, strs2))\n\n\tstrs1 = []string{\"1\", \"2\", \"3\"}\n\tstrs2 = []string{\"1\", \"2\", \"3\", \"4\"}\n\trequire.False(t, slices.StrSliceElementsMatch(strs1, strs2))\n\n\tstrs1 = []string{\"1\", \"2\", \"3\"}\n\tstrs2 = []string{\"1\", \"4\", \"3\"}\n\trequire.False(t, slices.StrSliceElementsMatch(strs1, strs2))\n\n\tstrs1 = []string{\"1\", \"2\", \"3\"}\n\tstrs2 = []string{\"3\", \"2\", \"1\"}\n\trequire.True(t, slices.StrSliceElementsMatch(strs1, strs2))\n\n\tstrs1 = []string{\"2\", \"1\", \"2\", \"3\"}\n\tstrs2 = []string{\"3\", \"2\", \"1\", \"2\"}\n\trequire.True(t, slices.StrSliceElementsMatch(strs1, strs2))\n\trequire.Equal(t, []string{\"2\", \"1\", \"2\", \"3\"}, strs1) // ensure sort didn't get applied\n\trequire.Equal(t, []string{\"3\", \"2\", \"1\", \"2\"}, strs2) // ensure sort didn't get applied\n\n\tstrs1 = []string{\"2\", \"1\", \"2\", \"3\"}\n\tstrs2 = []string{\"3\", \"2\", \"1\"}\n\trequire.False(t, slices.StrSliceElementsMatch(strs1, strs2))\n}\n\nfunc TestHasString(t *testing.T) {\n\tcases := []struct {\n\t\tname     string\n\t\tslice    []string\n\t\ttarget   string\n\t\texpected bool\n\t}{\n\t\t{\n\t\t\tname:     \"exists\",\n\t\t\tslice:    []string{\"a\", \"b\", \"c\"},\n\t\t\ttarget:   \"a\",\n\t\t\texpected: true,\n\t\t},\n\t\t{\n\t\t\tname:     \"doesn't exist\",\n\t\t\tslice:    []string{\"a\", \"b\", \"c\"},\n\t\t\ttarget:   \"d\",\n\t\t\texpected: false,\n\t\t},\n\t\t{\n\t\t\tname:     \"nil\",\n\t\t\tslice:    nil,\n\t\t\ttarget:   \"a\",\n\t\t\texpected: false,\n\t\t},\n\t}\n\n\tfor _, tt := range cases {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tresult := slices.HasString(tt.slice, tt.target)\n\t\t\trequire.Equal(t, tt.expected, result)\n\t\t})\n\t}\n}\n\nfunc TestRemoveString(t *testing.T) {\n\tcases := []struct {\n\t\tname     string\n\t\tslice    []string\n\t\ttarget   string\n\t\texpected []string\n\t}{\n\t\t{\n\t\t\tname:     \"simple\",\n\t\t\tslice:    []string{\"a\", \"b\", \"c\"},\n\t\t\ttarget:   \"a\",\n\t\t\texpected: []string{\"b\", \"c\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"repeated\",\n\t\t\tslice:    []string{\"a\", \"b\", \"c\", \"c\"},\n\t\t\ttarget:   \"c\",\n\t\t\texpected: []string{\"a\", \"b\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"nil\",\n\t\t\tslice:    nil,\n\t\t\ttarget:   \"a\",\n\t\t\texpected: nil,\n\t\t},\n\t\t{\n\t\t\tname:     \"unchanged\",\n\t\t\tslice:    []string{\"a\", \"b\", \"c\", \"c\"},\n\t\t\ttarget:   \"d\",\n\t\t\texpected: []string{\"a\", \"b\", \"c\", \"c\"},\n\t\t},\n\t}\n\n\tfor _, tt := range cases {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tresult := slices.RemoveString(tt.slice, tt.target)\n\t\t\trequire.Equal(t, tt.expected, result)\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/strings/operations.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage strings\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\t\"unicode\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n)\n\nfunc ToTitle(str string) string {\n\treturn strings.Title(strings.ToLower(str))\n}\n\nfunc EnsureSingleOccurrenceCharPrefix(str string, character string) string {\n\treturn character + strings.TrimLeft(str, character)\n}\n\nfunc EnsureSingleOccurrenceCharSuffix(str string, character string) string {\n\treturn strings.TrimRight(str, character) + character\n}\n\nfunc EnsurePrefix(str string, prefix string) string {\n\tif prefix != \"\" && !strings.HasPrefix(str, prefix) {\n\t\treturn prefix + str\n\t}\n\treturn str\n}\n\nfunc EnsureSuffix(str string, suffix string) string {\n\tif suffix != \"\" && !strings.HasSuffix(str, suffix) {\n\t\treturn str + suffix\n\t}\n\treturn str\n}\n\nfunc EnsureBlankLineIfNotEmpty(str string) string {\n\tif str == \"\" {\n\t\treturn str\n\t}\n\tif strings.HasSuffix(str, \"\\n\\n\") {\n\t\treturn str\n\t}\n\tif strings.HasSuffix(str, \"\\n\") {\n\t\treturn str + \"\\n\"\n\t}\n\treturn str + \"\\n\\n\"\n}\n\nfunc TrimTrailingNewLines(str string) string {\n\treturn strings.TrimRight(str, \"\\n\")\n}\n\nfunc TrimTrailingWhitespace(str string) string {\n\treturn strings.TrimRightFunc(str, unicode.IsSpace)\n}\n\nfunc EnsureSingleTrailingNewLine(str string) string {\n\treturn strings.TrimRight(str, \"\\n\") + \"\\n\"\n}\n\nfunc HasPrefixAndSuffix(str string, substr string) bool {\n\treturn strings.HasPrefix(str, substr) && strings.HasSuffix(str, substr)\n}\n\nfunc TrimPrefixAndSuffix(str string, substr string) string {\n\treturn strings.TrimSuffix(strings.TrimPrefix(str, substr), substr)\n}\n\n// MaskString omits no more than half of the string\nfunc MaskString(str string, numPlain int) string {\n\tif numPlain > len(str)/2 {\n\t\tnumPlain = len(str) / 2\n\t}\n\treturn strings.Repeat(\"*\", len(str)-numPlain) + str[len(str)-numPlain:]\n}\n\n// Returns the portion str after the last occurrance of chars, or the entire str if chars are not found\nfunc LastSplit(str string, chars string) string {\n\tsplit := strings.Split(str, chars)\n\treturn split[len(split)-1]\n}\n\n// Returns the last n chars, or the entire string if the requested length is greater than the length of the string\nfunc LastNChars(str string, n int) string {\n\tif len(str) < n {\n\t\treturn str\n\t}\n\n\treturn str[len(str)-n:]\n}\n\nfunc LongestCommonPrefix(strs ...string) string {\n\tif len(strs) == 0 {\n\t\treturn \"\"\n\t}\n\n\tprefix := strs[0]\n\n\tif len(strs) == 1 {\n\t\treturn prefix\n\t}\n\n\tfor _, str := range strs[1:] {\n\t\tif len(prefix) == 0 || len(str) == 0 {\n\t\t\treturn \"\"\n\t\t}\n\n\t\tmaxLen := len(prefix)\n\t\tif len(str) < maxLen {\n\t\t\tmaxLen = len(str)\n\t\t}\n\t\tfor i := 0; i < maxLen; i++ {\n\t\t\tif prefix[i] != str[i] {\n\t\t\t\tprefix = prefix[:i]\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\treturn prefix\n}\n\nfunc MaxLen(strs ...string) int {\n\tif len(strs) == 0 {\n\t\treturn 0\n\t}\n\n\tmaxLen := len(strs[0])\n\tfor _, str := range strs {\n\t\tif len(str) > maxLen {\n\t\t\tmaxLen = len(str)\n\t\t}\n\t}\n\n\treturn maxLen\n}\n\nfunc TrimPrefixIfPresentInAll(strs []string, prefix string) ([]string, bool) {\n\tif prefix == \"\" {\n\t\treturn strs, false\n\t}\n\ttrimmedStrs := make([]string, len(strs))\n\tfor i, str := range strs {\n\t\tif !strings.HasPrefix(str, prefix) {\n\t\t\treturn strs, false\n\t\t}\n\t\ttrimmedStrs[i] = strings.TrimPrefix(str, prefix)\n\t}\n\treturn trimmedStrs, true\n}\n\nfunc StrsOr(strs []string) string {\n\treturn StrsSentence(strs, \"or\")\n}\n\nfunc StrsAnd(strs []string) string {\n\treturn StrsSentence(strs, \"and\")\n}\n\nfunc UserStrsOr(vals interface{}) string {\n\treturn StrsOr(UserStrs(vals))\n}\n\nfunc UserStrsAnd(vals interface{}) string {\n\treturn StrsAnd(UserStrs(vals))\n}\n\nfunc StrsSentence(strs []string, lastJoinWord string) string {\n\tswitch len(strs) {\n\tcase 0:\n\t\treturn \"\"\n\tcase 1:\n\t\treturn strs[0]\n\tcase 2:\n\t\treturn strings.Join(strs, \" \"+lastJoinWord+\" \")\n\tdefault:\n\t\tlastIndex := len(strs) - 1\n\t\treturn strings.Join(strs[:lastIndex], \", \") + \", \" + lastJoinWord + \" \" + strs[lastIndex]\n\t}\n}\n\nfunc SIfPlural(count interface{}) string {\n\treturn StrIfPlural(\"s\", count)\n}\n\nfunc EsIfPlural(count interface{}) string {\n\treturn StrIfPlural(\"es\", count)\n}\n\nfunc StrIfPlural(str string, count interface{}) string {\n\tcountInt, _ := cast.InterfaceToInt64(count)\n\tif countInt > 1 {\n\t\treturn str\n\t}\n\treturn \"\"\n}\n\nfunc PluralS(str string, count interface{}) string {\n\treturn PluralCustom(str, str+\"s\", count)\n}\n\nfunc PluralEs(str string, count interface{}) string {\n\treturn PluralCustom(str, str+\"es\", count)\n}\n\nfunc PluralIs(count interface{}) string {\n\treturn PluralCustom(\"is\", \"are\", count)\n}\n\nfunc PluralCustom(singular string, plural string, count interface{}) string {\n\tcountInt, _ := cast.InterfaceToInt64(count)\n\tif countInt == 1 {\n\t\treturn singular\n\t}\n\treturn plural\n}\n\n// RemoveDuplicates returns a filtered string slice without repeated entries.\n// The ignoreRegex parameter can optionally be used to ignore repeated patterns in each slice entry.\nfunc RemoveDuplicates(strs []string, ignoreRegex *regexp.Regexp) []string {\n\tvar result []string\n\tcounter := map[string]int64{}\n\n\tfor _, str := range strs {\n\t\tfilteredStr := str\n\t\tif ignoreRegex != nil {\n\t\t\tfilteredStr = ignoreRegex.ReplaceAllString(str, \"\")\n\t\t}\n\n\t\tcounter[filteredStr]++\n\t\tif counter[filteredStr] > 1 {\n\t\t\tcontinue\n\t\t}\n\t\tresult = append(result, str)\n\t}\n\n\treturn result\n}\n"
  },
  {
    "path": "pkg/lib/strings/operations_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage strings\n\nimport (\n\t\"regexp\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestLongestCommonPrefix(t *testing.T) {\n\tvar strs []string\n\tvar expected string\n\n\tstrs = []string{\n\t\t\"12345\",\n\t}\n\texpected = \"12345\"\n\trequire.Equal(t, expected, LongestCommonPrefix(strs...))\n\n\tstrs = []string{\n\t\t\"12345\",\n\t\t\"12345678\",\n\t}\n\texpected = \"12345\"\n\trequire.Equal(t, expected, LongestCommonPrefix(strs...))\n\n\tstrs = []string{\n\t\t\"12345\",\n\t\t\"12345678\",\n\t\t\"1239\",\n\t}\n\texpected = \"123\"\n\trequire.Equal(t, expected, LongestCommonPrefix(strs...))\n\n\tstrs = []string{\n\t\t\"123\",\n\t\t\"456\",\n\t}\n\texpected = \"\"\n\trequire.Equal(t, expected, LongestCommonPrefix(strs...))\n}\n\nfunc TestRemoveDuplicates(t *testing.T) {\n\tcases := []struct {\n\t\tname        string\n\t\tinput       []string\n\t\tprefixRegex *regexp.Regexp\n\t\texpected    []string\n\t}{\n\t\t{\n\t\t\tname:     \"abc\",\n\t\t\tinput:    []string{\"a\", \"b\", \"a\", \"b\", \"a\", \"a\", \"a\", \"c\"},\n\t\t\texpected: []string{\"a\", \"b\", \"c\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"nil\",\n\t\t\tinput:    nil,\n\t\t\texpected: nil,\n\t\t},\n\t\t{\n\t\t\tname:        \"eksctl\",\n\t\t\tprefixRegex: regexp.MustCompile(`^.*[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} \\[.+] {2}`),\n\t\t\tinput: []string{\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  eksctl version 0.40.0\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  using region us-east-1\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  subnets for us-east-1a - public:192.168.0.0/19 private:192.168.64.0/19\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  subnets for us-east-1b - public:192.168.32.0/19 private:192.168.96.0/19\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  nodegroup \\\"cx-operator\\\" will use \\\"ami-05edded4121b6bde8\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  nodegroup \\\"cx-ws-spot\\\" will use \\\"ami-05edded4121b6bde8\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  nodegroup \\\"cx-wd-cpu\\\" will use \\\"ami-05edded4121b6bde8\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  nodegroup \\\"cx-wd-gpu\\\" will use \\\"ami-00a430391abee258d\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  nodegroup \\\"cx-wd-inferentia\\\" will use \\\"ami-00a430391abee258d\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  using Kubernetes version 1.18\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  creating EKS cluster \\\"cortex\\\" in \\\"us-east-1\\\" region with un-managed nodes\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  5 nodegroups (cx-operator, cx-wd-cpu, cx-wd-gpu, cx-wd-inferentia, cx-ws-spot) were included (based on the include/exclude rules)\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  will create a CloudFormation stack for cluster itself and 5 nodegroup stack(s)\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  will create a CloudFormation stack for cluster itself and 0 managed nodegroup stack(s)\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=us-east-1 --cluster=cortex'\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  CloudWatch logging will not be enabled for cluster \\\"cortex\\\" in \\\"us-east-1\\\"\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  you can enable it with 'eksctl utils update-cluster-logging --enable-types={SPECIFY-YOUR-LOG-TYPES-HERE (e.g. all)} --region=us-east-1 --cluster=cortex'\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  Kubernetes API endpoint access will use default of {publicAccess=true, privateAccess=false} for cluster \\\"cortex\\\" in \\\"us-east-1\\\"\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  2 sequential tasks: { create cluster control plane \\\"cortex\\\", 3 sequential sub-tasks: { 2 sequential sub-tasks: { wait for control plane to become ready, tag cluster }, create addons, 5 parallel sub-tasks: { create nodegroup \\\"cx-operator\\\", create nodegroup \\\"cx-ws-spot\\\", create nodegroup \\\"cx-wd-cpu\\\", create nodegroup \\\"cx-wd-gpu\\\", create nodegroup \\\"cx-wd-inferentia\\\" } } }\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  building cluster stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:03:52 [ℹ]  deploying stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:03:52 [ℹ]  waiting for CloudFormation stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:04:09 [ℹ]  waiting for CloudFormation stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:04:28 [ℹ]  waiting for CloudFormation stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:04:47 [ℹ]  waiting for CloudFormation stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:05:07 [ℹ]  waiting for CloudFormation stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:05:25 [ℹ]  waiting for CloudFormation stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t},\n\t\t\texpected: []string{\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  eksctl version 0.40.0\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  using region us-east-1\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  subnets for us-east-1a - public:192.168.0.0/19 private:192.168.64.0/19\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  subnets for us-east-1b - public:192.168.32.0/19 private:192.168.96.0/19\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  nodegroup \\\"cx-operator\\\" will use \\\"ami-05edded4121b6bde8\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  nodegroup \\\"cx-ws-spot\\\" will use \\\"ami-05edded4121b6bde8\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:50 [ℹ]  nodegroup \\\"cx-wd-cpu\\\" will use \\\"ami-05edded4121b6bde8\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  nodegroup \\\"cx-wd-gpu\\\" will use \\\"ami-00a430391abee258d\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  nodegroup \\\"cx-wd-inferentia\\\" will use \\\"ami-00a430391abee258d\\\" [AmazonLinux2/1.18]\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  using Kubernetes version 1.18\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  creating EKS cluster \\\"cortex\\\" in \\\"us-east-1\\\" region with un-managed nodes\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  5 nodegroups (cx-operator, cx-wd-cpu, cx-wd-gpu, cx-wd-inferentia, cx-ws-spot) were included (based on the include/exclude rules)\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  will create a CloudFormation stack for cluster itself and 5 nodegroup stack(s)\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  will create a CloudFormation stack for cluster itself and 0 managed nodegroup stack(s)\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=us-east-1 --cluster=cortex'\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  CloudWatch logging will not be enabled for cluster \\\"cortex\\\" in \\\"us-east-1\\\"\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  you can enable it with 'eksctl utils update-cluster-logging --enable-types={SPECIFY-YOUR-LOG-TYPES-HERE (e.g. all)} --region=us-east-1 --cluster=cortex'\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  Kubernetes API endpoint access will use default of {publicAccess=true, privateAccess=false} for cluster \\\"cortex\\\" in \\\"us-east-1\\\"\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  2 sequential tasks: { create cluster control plane \\\"cortex\\\", 3 sequential sub-tasks: { 2 sequential sub-tasks: { wait for control plane to become ready, tag cluster }, create addons, 5 parallel sub-tasks: { create nodegroup \\\"cx-operator\\\", create nodegroup \\\"cx-ws-spot\\\", create nodegroup \\\"cx-wd-cpu\\\", create nodegroup \\\"cx-wd-gpu\\\", create nodegroup \\\"cx-wd-inferentia\\\" } } }\",\n\t\t\t\t\"2021-03-26 00:03:51 [ℹ]  building cluster stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:03:52 [ℹ]  deploying stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t\t\"2021-03-26 00:03:52 [ℹ]  waiting for CloudFormation stack \\\"eksctl-cortex-cluster\\\"\",\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range cases {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\toutput := RemoveDuplicates(tt.input, tt.prefixRegex)\n\t\t\trequire.Equal(t, tt.expected, output)\n\t\t})\n\t}\n}\n"
  },
  {
    "path": "pkg/lib/strings/parse.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage strings\n\nimport (\n\t\"strconv\"\n)\n\nfunc ParseBool(valStr string) (bool, bool) {\n\tcasted, err := strconv.ParseBool(valStr)\n\tif err != nil {\n\t\treturn false, false\n\t}\n\treturn casted, true\n}\n\nfunc ParseFloat32(valStr string) (float32, bool) {\n\tcasted, err := strconv.ParseFloat(valStr, 32)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\treturn float32(casted), true\n}\n\nfunc ParseFloat64(valStr string) (float64, bool) {\n\tcasted, err := strconv.ParseFloat(valStr, 64)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\treturn casted, true\n}\n\nfunc ParseInt(valStr string) (int, bool) {\n\tcasted, err := strconv.Atoi(valStr)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\treturn casted, true\n}\n\nfunc ParseInt64(valStr string) (int64, bool) {\n\tcasted, err := strconv.ParseInt(valStr, 10, 64)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\treturn casted, true\n}\n\nfunc ParseInt32(valStr string) (int32, bool) {\n\tcasted, err := strconv.ParseInt(valStr, 10, 32)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\treturn int32(casted), true\n}\n\nfunc ParseInt16(valStr string) (int16, bool) {\n\tcasted, err := strconv.ParseInt(valStr, 10, 16)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\treturn int16(casted), true\n}\n\nfunc ParseInt8(valStr string) (int8, bool) {\n\tcasted, err := strconv.ParseInt(valStr, 10, 8)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\treturn int8(casted), true\n}\n"
  },
  {
    "path": "pkg/lib/strings/stringify.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage strings\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/yaml\"\n)\n\nfunc Bool(val bool) string {\n\treturn strconv.FormatBool(val)\n}\n\nfunc Float32(val float32) string {\n\tstr := strconv.FormatFloat(float64(val), 'f', -1, 32)\n\tif !strings.Contains(str, \".\") {\n\t\tstr = str + \".0\"\n\t}\n\treturn str\n}\n\nfunc Float64(val float64) string {\n\tstr := strconv.FormatFloat(val, 'f', -1, 64)\n\tif !strings.Contains(str, \".\") {\n\t\tstr = str + \".0\"\n\t}\n\treturn str\n}\n\nfunc Int(val int) string {\n\treturn strconv.Itoa(val)\n}\n\nfunc Int64(val int64) string {\n\treturn strconv.FormatInt(val, 10)\n}\n\nfunc Int32(val int32) string {\n\treturn strconv.FormatInt(int64(val), 10)\n}\n\nfunc Int16(val int16) string {\n\treturn strconv.FormatInt(int64(val), 10)\n}\n\nfunc Int8(val int8) string {\n\treturn strconv.FormatInt(int64(val), 10)\n}\n\nfunc Uint(val uint) string {\n\treturn strconv.FormatUint(uint64(val), 10)\n}\n\nfunc Uint8(val uint8) string {\n\treturn strconv.FormatUint(uint64(val), 10)\n}\n\nfunc Uint16(val uint16) string {\n\treturn strconv.FormatUint(uint64(val), 10)\n}\n\nfunc Uint32(val uint32) string {\n\treturn strconv.FormatUint(uint64(val), 10)\n}\n\nfunc Uint64(val uint64) string {\n\treturn strconv.FormatUint(val, 10)\n}\n\nfunc Complex64(val complex64) string {\n\treturn fmt.Sprint(val)\n}\n\nfunc Complex128(val complex128) string {\n\treturn fmt.Sprint(val)\n}\n\nfunc Uintptr(val uintptr) string {\n\treturn fmt.Sprint(val)\n}\n\nfunc Round(val float64, decimalPlaces int, padToDecimalPlaces int) string {\n\trounded := math.Round(val*math.Pow10(decimalPlaces)) / math.Pow10(decimalPlaces)\n\tstr := strconv.FormatFloat(rounded, 'f', -1, 64)\n\tif padToDecimalPlaces == 0 {\n\t\treturn str\n\t}\n\tsplit := strings.Split(str, \".\")\n\tintVal := split[0]\n\tdecVal := \"\"\n\tif len(split) > 1 {\n\t\tdecVal = split[1]\n\t}\n\tif len(decVal) >= padToDecimalPlaces {\n\t\treturn str\n\t}\n\tnumZeros := padToDecimalPlaces - len(decVal)\n\treturn intVal + \".\" + decVal + strings.Repeat(\"0\", numZeros)\n}\n\n// copied from https://yourbasic.org/golang/formatting-byte-size-to-human-readable-format/\nfunc IntToBase2Byte(size int) string {\n\treturn Int64ToBase2Byte(int64(size))\n}\n\nfunc Int64ToBase2Byte(size int64) string {\n\tconst unit int64 = 1024\n\tif size < unit {\n\t\treturn fmt.Sprintf(\"%d B\", size)\n\t}\n\tdiv, exp := int64(unit), 0\n\tfor n := size / unit; n >= unit; n /= unit {\n\t\tdiv *= unit\n\t\texp++\n\t}\n\treturn fmt.Sprintf(\"%.1f %ciB\",\n\t\tfloat64(size)/float64(div), \"KMGTPE\"[exp])\n\n}\n\nfunc DollarsAndCents(val float64) string {\n\treturn \"$\" + Round(val, 2, 2)\n}\n\nfunc DollarsAndTenthsOfCents(val float64) string {\n\treturn \"$\" + Round(val, 3, 2)\n}\n\nfunc DollarsMaxPrecision(val float64) string {\n\treturn \"$\" + Round(val, 100, 2)\n}\n\n// This is similar to json.Marshal, but handles non-string keys (which we support). It should be valid YAML since we use it in templates\nfunc strIndent(val interface{}, indent string, currentIndent string, newlineChar string, quoteStr string) string {\n\tif val == nil {\n\t\treturn \"<null>\"\n\t}\n\n\tvalue := reflect.ValueOf(val)\n\tvalueType := value.Type()\n\n\tif value.Kind() == reflect.Invalid {\n\t\treturn \"<invalid>\"\n\t}\n\n\tif value.Kind() == reflect.Chan || value.Kind() == reflect.Func || value.Kind() == reflect.Interface || value.Kind() == reflect.Map || value.Kind() == reflect.Ptr || value.Kind() == reflect.Slice {\n\t\tif value.IsNil() {\n\t\t\treturn \"<null>\"\n\t\t}\n\t}\n\n\tstringSep := \",\" + newlineChar\n\tif len(newlineChar) == 0 {\n\t\tstringSep += \" \"\n\t}\n\n\t// Use a String() method if one exists\n\tfuncVal := value.MethodByName(\"String\")\n\tif funcVal.IsValid() {\n\t\tt := funcVal.Type()\n\t\tif t.NumIn() == 0 && t.NumOut() == 1 && t.Out(0).Kind() == reflect.String {\n\t\t\treturn strIndent(funcVal.Call(nil)[0].Interface().(string), indent, currentIndent, newlineChar, quoteStr)\n\t\t}\n\t}\n\tif _, ok := reflect.PtrTo(valueType).MethodByName(\"String\"); ok {\n\t\tptrValue := reflect.New(valueType)\n\t\tptrValue.Elem().Set(value)\n\t\tfuncVal := ptrValue.MethodByName(\"String\")\n\t\tif funcVal.IsValid() {\n\t\t\tt := funcVal.Type()\n\t\t\tif t.NumIn() == 0 && t.NumOut() == 1 && t.Out(0).Kind() == reflect.String {\n\t\t\t\treturn strIndent(funcVal.Call(nil)[0].Interface().(string), indent, currentIndent, newlineChar, quoteStr)\n\t\t\t}\n\t\t}\n\t}\n\n\tswitch value.Kind() {\n\n\tcase reflect.Bool:\n\t\tvar t bool\n\t\treturn Bool(value.Convert(reflect.TypeOf(t)).Interface().(bool))\n\tcase reflect.Float32:\n\t\tvar t float32\n\t\treturn Float32(value.Convert(reflect.TypeOf(t)).Interface().(float32))\n\tcase reflect.Float64:\n\t\tvar t float64\n\t\treturn Float64(value.Convert(reflect.TypeOf(t)).Interface().(float64))\n\tcase reflect.Int:\n\t\tvar t int\n\t\treturn Int(value.Convert(reflect.TypeOf(t)).Interface().(int))\n\tcase reflect.Int8:\n\t\tvar t int8\n\t\treturn Int8(value.Convert(reflect.TypeOf(t)).Interface().(int8))\n\tcase reflect.Int16:\n\t\tvar t int16\n\t\treturn Int16(value.Convert(reflect.TypeOf(t)).Interface().(int16))\n\tcase reflect.Int32:\n\t\tvar t int32\n\t\treturn Int32(value.Convert(reflect.TypeOf(t)).Interface().(int32))\n\tcase reflect.Int64:\n\t\tvar t int64\n\t\treturn Int64(value.Convert(reflect.TypeOf(t)).Interface().(int64))\n\tcase reflect.Uint:\n\t\tvar t uint\n\t\treturn Uint(value.Convert(reflect.TypeOf(t)).Interface().(uint))\n\tcase reflect.Uint8:\n\t\tvar t uint8\n\t\treturn Uint8(value.Convert(reflect.TypeOf(t)).Interface().(uint8))\n\tcase reflect.Uint16:\n\t\tvar t uint16\n\t\treturn Uint16(value.Convert(reflect.TypeOf(t)).Interface().(uint16))\n\tcase reflect.Uint32:\n\t\tvar t uint32\n\t\treturn Uint32(value.Convert(reflect.TypeOf(t)).Interface().(uint32))\n\tcase reflect.Uint64:\n\t\tvar t uint64\n\t\treturn Uint64(value.Convert(reflect.TypeOf(t)).Interface().(uint64))\n\tcase reflect.Complex64:\n\t\tvar t complex64\n\t\treturn Complex64(value.Convert(reflect.TypeOf(t)).Interface().(complex64))\n\tcase reflect.Complex128:\n\t\tvar t complex128\n\t\treturn Complex128(value.Convert(reflect.TypeOf(t)).Interface().(complex128))\n\tcase reflect.Uintptr:\n\t\tvar t uintptr\n\t\treturn Uintptr(value.Convert(reflect.TypeOf(t)).Interface().(uintptr))\n\n\tcase reflect.String:\n\t\tvar t string\n\t\tcasted := value.Convert(reflect.TypeOf(t)).Interface().(string)\n\n\t\tvar ok bool\n\t\tcasted, ok = yaml.UnescapeAtSymbolOk(casted)\n\t\tif ok {\n\t\t\treturn casted\n\t\t}\n\n\t\tswitch val.(type) {\n\t\tcase json.Number:\n\t\t\treturn casted\n\t\tdefault:\n\t\t\treturn quoteStr + casted + quoteStr\n\t\t}\n\n\tcase reflect.Slice:\n\t\tfallthrough\n\tcase reflect.Array:\n\t\tif value.Len() == 0 {\n\t\t\treturn \"[]\"\n\t\t}\n\t\tstrs := make([]string, value.Len())\n\t\tfor i := 0; i < value.Len(); i++ {\n\t\t\tstrs[i] = currentIndent + indent + strIndentValue(value.Index(i), indent, currentIndent+indent, newlineChar, quoteStr)\n\t\t}\n\n\t\treturn \"[\" + newlineChar + strings.Join(strs, stringSep) + newlineChar + currentIndent + \"]\"\n\n\tcase reflect.Map:\n\t\tif value.Len() == 0 {\n\t\t\treturn \"{}\"\n\t\t}\n\t\tstrs := make([]string, value.Len())\n\t\tfor i, keyValue := range value.MapKeys() {\n\t\t\tkeyStr := strIndentValue(keyValue, indent, currentIndent+indent, newlineChar, quoteStr)\n\t\t\tvalStr := strIndentValue(value.MapIndex(keyValue), indent, currentIndent+indent, newlineChar, quoteStr)\n\t\t\tstrs[i] = currentIndent + indent + keyStr + \": \" + valStr\n\t\t}\n\t\tsort.Strings(strs)\n\t\treturn \"{\" + newlineChar + strings.Join(strs, stringSep) + newlineChar + currentIndent + \"}\"\n\n\tcase reflect.Struct:\n\t\tif value.NumField() == 0 {\n\t\t\treturn \"{}\"\n\t\t}\n\n\t\tstrs := make([]string, value.NumField())\n\n\t\tfor i := 0; i < value.NumField(); i++ {\n\t\t\tstructField := valueType.Field(i)\n\t\t\tkeyStr := strIndent(structField.Name, indent, currentIndent+indent, newlineChar, quoteStr)\n\t\t\tif tag, ok := structField.Tag.Lookup(\"yaml\"); ok {\n\t\t\t\tkeyStr = strIndent(strings.Split(tag, \",\")[0], indent, currentIndent+indent, newlineChar, quoteStr)\n\t\t\t}\n\t\t\tif tag, ok := structField.Tag.Lookup(\"json\"); ok {\n\t\t\t\tkeyStr = strIndent(strings.Split(tag, \",\")[0], indent, currentIndent+indent, newlineChar, quoteStr)\n\t\t\t}\n\t\t\tvalStr := strIndentValue(value.Field(i), indent, currentIndent+indent, newlineChar, quoteStr)\n\t\t\tstrs[i] = currentIndent + indent + keyStr + \": \" + valStr\n\t\t}\n\t\treturn \"{\" + newlineChar + strings.Join(strs, stringSep) + newlineChar + currentIndent + \"}\"\n\n\tcase reflect.Ptr:\n\t\treturn strIndentValue(reflect.Indirect(value), indent, currentIndent, newlineChar, quoteStr)\n\n\tcase reflect.UnsafePointer:\n\t\treturn strIndentValue(reflect.Indirect(value), indent, currentIndent+indent, newlineChar, quoteStr)\n\n\tcase reflect.Interface:\n\t\treturn strIndentValue(value.Elem(), indent, currentIndent+indent, newlineChar, quoteStr)\n\n\tcase reflect.Func:\n\t\treturn \"<function>\"\n\n\tcase reflect.Chan:\n\t\treturn \"<channel>\"\n\n\tcase reflect.Invalid:\n\t\treturn \"<invalid>\"\n\n\tdefault:\n\t\treturn fmt.Sprint(val)\n\t}\n}\n\nfunc strIndentValue(val reflect.Value, indent string, currentIndent string, newlineChar string, quoteStr string) string {\n\tif val.IsValid() && val.CanInterface() {\n\t\treturn strIndent(val.Interface(), indent, currentIndent, newlineChar, quoteStr)\n\t}\n\treturn \"<hidden>\"\n}\n\nfunc YesNo(val bool) string {\n\tif val {\n\t\treturn \"yes\"\n\t}\n\treturn \"no\"\n}\n\nfunc Obj(val interface{}) string {\n\treturn strIndent(val, \"  \", \"\", \"\\n\", `\"`)\n}\n\n// Same as Obj(), but trim leading and trailing quotes if it's just a string\nfunc ObjStripped(val interface{}) string {\n\treturn TrimPrefixAndSuffix(Obj(val), `\"`)\n}\n\nfunc ObjFlat(val interface{}) string {\n\treturn strIndent(val, \"\", \"\", \"\", `\"`)\n}\n\nfunc ObjFlatNoQuotes(val interface{}) string {\n\treturn strIndent(val, \"\", \"\", \"\", \"\")\n}\n\nfunc UserStr(val interface{}) string {\n\treturn strIndent(val, \"\", \"\", \"\", `\"`)\n}\n\nfunc UserStrValue(val reflect.Value) string {\n\treturn strIndentValue(val, \"\", \"\", \"\", `\"`)\n}\n\nfunc UserStrStripped(val interface{}) string {\n\treturn TrimPrefixAndSuffix(UserStr(val), `\"`)\n}\n\nfunc UserStrs(val interface{}) []string {\n\tif val == nil {\n\t\treturn nil\n\t}\n\n\tif reflect.TypeOf(val).Kind() != reflect.Slice {\n\t\tval = []interface{}{val}\n\t}\n\n\tinVal := reflect.ValueOf(val)\n\tif inVal.IsNil() {\n\t\treturn nil\n\t}\n\n\t// Handle case where caller passed in a nested slice\n\tif inVal.Len() == 1 {\n\t\tif inVal.Index(0).Kind() == reflect.Slice {\n\t\t\tinVal = inVal.Index(0)\n\t\t} else if inVal.Index(0).Kind() == reflect.Interface { // Handle case where input is e.g. []interface{[]string{\"test\"}}\n\t\t\tfirstElementVal := reflect.ValueOf(inVal.Index(0).Interface())\n\t\t\tif firstElementVal.Kind() == reflect.Slice {\n\t\t\t\tinVal = firstElementVal\n\t\t\t}\n\t\t}\n\t}\n\n\tout := make([]string, inVal.Len())\n\tfor i := 0; i < inVal.Len(); i++ {\n\t\tout[i] = UserStrValue(inVal.Index(i))\n\t}\n\treturn out\n}\n\nfunc Index(index int) string {\n\treturn fmt.Sprintf(\"index %d\", index)\n}\n\nfunc Indent(str string, indent string) string {\n\tif str == \"\" {\n\t\treturn indent\n\t}\n\n\tif str[len(str)-1:] == \"\\n\" {\n\t\tout := \"\"\n\t\tfor _, line := range strings.Split(str[:len(str)-1], \"\\n\") {\n\t\t\tout += indent + line + \"\\n\"\n\t\t}\n\t\treturn out\n\t}\n\n\tout := \"\"\n\tfor _, line := range strings.Split(strings.TrimRight(str, \"\\n\"), \"\\n\") {\n\t\tout += indent + line + \"\\n\"\n\t}\n\treturn out[:len(out)-1]\n}\n\nfunc TruncateEllipses(str string, maxLength int) string {\n\tellipses := \" ...\"\n\tif len(str) > maxLength {\n\t\tstr = str[:maxLength-len(ellipses)]\n\t\tstr += ellipses\n\t}\n\treturn str\n}\n"
  },
  {
    "path": "pkg/lib/strings/stringify_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage strings\n\nimport (\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/stretchr/testify/require\"\n)\n\ntype MyFloat float64\ntype MyString string\ntype MyNested map[MyFloat][]MyString\n\ntype Test struct {\n\tStr      MyString\n\tFloat    float64 `json:\"float\"`\n\tTest2Ptr *Test2\n\tTest3    Test3\n}\n\ntype Test2 struct {\n\tBool     bool `json:\"bool\"`\n\tFloat    *float64\n\tStrs     *[]string\n\tTest3Ptr *Test3 `yaml:\"test3\"`\n}\n\ntype Test3 struct {\n\tStrs []string\n\tMap  map[interface{}]interface{} `json:\"map\"`\n}\n\ntype TestInterface interface {\n\tTest()\n}\n\nfunc (t *Test2) Test() {}\nfunc (t Test3) Test()  {}\n\nfunc TestObj(t *testing.T) {\n\tvar a interface{}\n\trequire.Equal(t, \"<null>\", Obj(a))\n\tvar b []string\n\trequire.Equal(t, \"<null>\", Obj(b))\n\tvar c *string\n\trequire.Equal(t, \"<null>\", Obj(c))\n\n\trequire.Equal(t, \"true\", Obj(true))\n\trequire.Equal(t, \"2.2\", Obj(float32(2.2)))\n\trequire.Equal(t, \"2.0\", Obj(float32(2)))\n\trequire.Equal(t, \"2.0\", Obj(float64(2)))\n\trequire.Equal(t, \"3\", Obj(int(3)))\n\trequire.Equal(t, \"3\", Obj(pointer.Int(3)))\n\trequire.Equal(t, \"-3\", Obj(int8(-3)))\n\trequire.Equal(t, \"3\", Obj(int16(3)))\n\trequire.Equal(t, \"-3\", Obj(int32(-3)))\n\trequire.Equal(t, \"3\", Obj(int64(3)))\n\trequire.Equal(t, \"4\", Obj(int(4)))\n\trequire.Equal(t, \"4\", Obj(int8(4)))\n\trequire.Equal(t, \"4\", Obj(int16(4)))\n\trequire.Equal(t, \"4\", Obj(int32(4)))\n\trequire.Equal(t, \"4\", Obj(int64(4)))\n\trequire.Equal(t, `\"\"`, Obj(\"\"))\n\trequire.Equal(t, `\"test\"`, Obj(\"test\"))\n\trequire.Equal(t, `\"test\"`, Obj(pointer.String(\"test\")))\n\n\tvar myFloat MyFloat = 2\n\trequire.Equal(t, \"2.0\", Obj(myFloat))\n\tvar myString MyString = \"test\"\n\trequire.Equal(t, `\"test\"`, Obj(myString))\n\n\tstrSlice := []string{\"a\", \"b\", \"c\"}\n\trequire.Equal(t, `[\"a\", \"b\", \"c\"]`, ObjFlat(strSlice))\n\trequire.Equal(t, `[\"a\", \"b\", \"c\"]`, ObjFlat(&strSlice))\n\tintSlice := []int8{1, 2, 3}\n\trequire.Equal(t, `[1, 2, 3]`, ObjFlat(intSlice))\n\tmixedSlice := []interface{}{int(1), float64(2), \"three\", \"\"}\n\trequire.Equal(t, `[1, 2.0, \"three\", \"\"]`, ObjFlat(mixedSlice))\n\tnestedSlice := []interface{}{int(1), \"three\", strSlice, []interface{}{\"a\", []float64{1, 2.2}}}\n\trequire.Equal(t, `[1, \"three\", [\"a\", \"b\", \"c\"], [\"a\", [1.0, 2.2]]]`, ObjFlat(nestedSlice))\n\n\tstrMap := map[string]string{\"b\": \"y\", \"a\": \"x\"}\n\trequire.Equal(t, `{\"a\": \"x\", \"b\": \"y\"}`, ObjFlat(strMap))\n\trequire.Equal(t, `{\"a\": \"x\", \"b\": \"y\"}`, ObjFlat(&strMap))\n\tmixedMap := map[interface{}]interface{}{\"1\": \"a\", true: strMap, int(3): strSlice}\n\trequire.Equal(t, `{\"1\": \"a\", 3: [\"a\", \"b\", \"c\"], true: {\"a\": \"x\", \"b\": \"y\"}}`, ObjFlat(mixedMap))\n\n\tmyNested := MyNested{myFloat: []MyString{myString, myString}}\n\trequire.Equal(t, `{2.0: [\"test\", \"test\"]}`, ObjFlat(myNested))\n\n\temptyMap := map[interface{}]interface{}{}\n\trequire.Equal(t, `{}`, Obj(emptyMap))\n\trequire.Equal(t, `{}`, ObjFlat(emptyMap))\n\n\temptyCollectionsInMap := map[interface{}]interface{}{\"empty_map\": map[interface{}]interface{}{}, \"a\": \"b\", \"empty_slice\": []interface{}{}}\n\trequire.Equal(t, `{\"a\": \"b\", \"empty_map\": {}, \"empty_slice\": []}`, ObjFlat(emptyCollectionsInMap))\n\trequire.Equal(t, `{\n  \"a\": \"b\",\n  \"empty_map\": {},\n  \"empty_slice\": []\n}`, Obj(emptyCollectionsInMap))\n\n\ttestStruct := Test{\n\t\tStr:   myString,\n\t\tFloat: 2,\n\t\tTest2Ptr: &Test2{\n\t\t\tBool:  false,\n\t\t\tFloat: pointer.Float64(1.7),\n\t\t\tStrs:  &strSlice,\n\t\t\tTest3Ptr: &Test3{\n\t\t\t\tStrs: []string{\"a\", \"b\", \"c\"},\n\t\t\t\tMap:  map[interface{}]interface{}{\"1\": \"a\", true: strMap, int(3): intSlice},\n\t\t\t},\n\t\t},\n\t\tTest3: Test3{\n\t\t\tStrs: nil,\n\t\t\tMap:  map[interface{}]interface{}{\"1\": nil, true: strMap, int(3): intSlice},\n\t\t},\n\t}\n\ttestStructStr := `{\"Str\": \"test\", \"float\": 2.0, \"Test2Ptr\": {\"bool\": false, \"Float\": 1.7, \"Strs\": [\"a\", \"b\", \"c\"], \"test3\": {\"Strs\": [\"a\", \"b\", \"c\"], \"map\": {\"1\": \"a\", 3: [1, 2, 3], true: {\"a\": \"x\", \"b\": \"y\"}}}}, \"Test3\": {\"Strs\": <null>, \"map\": {\"1\": <null>, 3: [1, 2, 3], true: {\"a\": \"x\", \"b\": \"y\"}}}}`\n\n\ttestStructStrMultiline := `{\n  \"Str\": \"test\",\n  \"float\": 2.0,\n  \"Test2Ptr\": {\n    \"bool\": false,\n    \"Float\": 1.7,\n    \"Strs\": [\n      \"a\",\n      \"b\",\n      \"c\"\n    ],\n    \"test3\": {\n      \"Strs\": [\n        \"a\",\n        \"b\",\n        \"c\"\n      ],\n      \"map\": {\n        \"1\": \"a\",\n        3: [\n          1,\n          2,\n          3\n        ],\n        true: {\n          \"a\": \"x\",\n          \"b\": \"y\"\n        }\n      }\n    }\n  },\n  \"Test3\": {\n    \"Strs\": <null>,\n    \"map\": {\n      \"1\": <null>,\n      3: [\n        1,\n        2,\n        3\n      ],\n      true: {\n        \"a\": \"x\",\n        \"b\": \"y\"\n      }\n    }\n  }\n}`\n\ttest2SubStrMultiline := `{\n  \"bool\": false,\n  \"Float\": 1.7,\n  \"Strs\": [\n    \"a\",\n    \"b\",\n    \"c\"\n  ],\n  \"test3\": {\n    \"Strs\": [\n      \"a\",\n      \"b\",\n      \"c\"\n    ],\n    \"map\": {\n      \"1\": \"a\",\n      3: [\n        1,\n        2,\n        3\n      ],\n      true: {\n        \"a\": \"x\",\n        \"b\": \"y\"\n      }\n    }\n  }\n}`\n\ttest2SubStr := `{\"bool\": false, \"Float\": 1.7, \"Strs\": [\"a\", \"b\", \"c\"], \"test3\": {\"Strs\": [\"a\", \"b\", \"c\"], \"map\": {\"1\": \"a\", 3: [1, 2, 3], true: {\"a\": \"x\", \"b\": \"y\"}}}}`\n\n\ttest3SubStrMultiline := `{\n  \"Strs\": <null>,\n  \"map\": {\n    \"1\": <null>,\n    3: [\n      1,\n      2,\n      3\n    ],\n    true: {\n      \"a\": \"x\",\n      \"b\": \"y\"\n    }\n  }\n}`\n\ttest3SubStr := `{\"Strs\": <null>, \"map\": {\"1\": <null>, 3: [1, 2, 3], true: {\"a\": \"x\", \"b\": \"y\"}}}`\n\n\trequire.Equal(t, testStructStr, ObjFlat(testStruct))\n\trequire.Equal(t, testStructStr, ObjFlat(&testStruct))\n\tptr := &testStruct\n\trequire.Equal(t, testStructStr, ObjFlat(&ptr))\n\n\tvar testInterface TestInterface\n\ttestInterface = testStruct.Test2Ptr\n\trequire.Equal(t, test2SubStr, ObjFlat(testInterface))\n\trequire.Equal(t, test2SubStr, ObjFlat(&testInterface))\n\ttestInterface = testStruct.Test3\n\trequire.Equal(t, test3SubStr, ObjFlat(testInterface))\n\trequire.Equal(t, test3SubStr, ObjFlat(&testInterface))\n\n\trequire.Equal(t, testStructStrMultiline, Obj(testStruct))\n\trequire.Equal(t, testStructStrMultiline, Obj(&testStruct))\n\tptr = &testStruct\n\trequire.Equal(t, testStructStrMultiline, Obj(&ptr))\n\n\ttestInterface = testStruct.Test2Ptr\n\trequire.Equal(t, test2SubStrMultiline, Obj(testInterface))\n\trequire.Equal(t, test2SubStrMultiline, Obj(&testInterface))\n\ttestInterface = testStruct.Test3\n\trequire.Equal(t, test3SubStrMultiline, Obj(testInterface))\n\trequire.Equal(t, test3SubStrMultiline, Obj(&testInterface))\n\n}\n\nfunc TestRound(t *testing.T) {\n\trequire.Equal(t, Round(1.111, 2, 0), \"1.11\")\n\trequire.Equal(t, Round(1.111, 3, 0), \"1.111\")\n\trequire.Equal(t, Round(1.111, 4, 0), \"1.111\")\n\trequire.Equal(t, Round(1.555, 2, 0), \"1.56\")\n\trequire.Equal(t, Round(1.555, 3, 0), \"1.555\")\n\trequire.Equal(t, Round(1.555, 4, 0), \"1.555\")\n\trequire.Equal(t, Round(1.100, 2, 0), \"1.1\")\n\n\trequire.Equal(t, Round(1.111, 2, 2), \"1.11\")\n\trequire.Equal(t, Round(1.111, 3, 3), \"1.111\")\n\trequire.Equal(t, Round(1.111, 4, 4), \"1.1110\")\n\trequire.Equal(t, Round(1.555, 2, 2), \"1.56\")\n\trequire.Equal(t, Round(1.555, 3, 3), \"1.555\")\n\trequire.Equal(t, Round(1.555, 4, 4), \"1.5550\")\n\trequire.Equal(t, Round(1.100, 2, 2), \"1.10\")\n\n\trequire.Equal(t, Round(30, 0, 0), \"30\")\n\trequire.Equal(t, Round(2, 1, 1), \"2.0\")\n\trequire.Equal(t, Round(1, 2, 2), \"1.00\")\n\trequire.Equal(t, Round(20, 3, 3), \"20.000\")\n\n\trequire.Equal(t, Round(1.5555, 3, 2), \"1.556\")\n\trequire.Equal(t, Round(1.5, 3, 2), \"1.50\")\n\trequire.Equal(t, Round(1, 3, 2), \"1.00\")\n\n\trequire.Equal(t, Round(1.5555, 3, 1), \"1.556\")\n\trequire.Equal(t, Round(1.5, 3, 1), \"1.5\")\n\trequire.Equal(t, Round(1, 3, 1), \"1.0\")\n\n\trequire.Equal(t, Round(1.5555, 3, 4), \"1.5560\")\n\trequire.Equal(t, Round(1.5, 3, 4), \"1.5000\")\n\trequire.Equal(t, Round(1, 3, 4), \"1.0000\")\n\n\trequire.Equal(t, Round(30, 0, 0), \"30\")\n\trequire.Equal(t, Round(2, 1, 0), \"2\")\n\trequire.Equal(t, Round(1, 2, 0), \"1\")\n\trequire.Equal(t, Round(20, 3, 0), \"20\")\n}\n"
  },
  {
    "path": "pkg/lib/structs/deepcopy.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage structs\n\nimport (\n\t\"bytes\"\n\t\"encoding/gob\"\n)\n\nfunc DeepCopy(dst, src interface{}) error {\n\tvar buf bytes.Buffer\n\tif err := gob.NewEncoder(&buf).Encode(src); err != nil {\n\t\treturn err\n\t}\n\treturn gob.NewDecoder(bytes.NewBuffer(buf.Bytes())).Decode(dst)\n}\n"
  },
  {
    "path": "pkg/lib/structs/deepcopy_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage structs\n\nimport (\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/stretchr/testify/require\"\n)\n\ntype SampleStruct struct {\n\tAField string\n\tBField *string\n\tCField *bool\n}\n\nfunc TestDeepCopy(t *testing.T) {\n\tt.Parallel()\n\n\tvar a SampleStruct\n\tb := SampleStruct{\n\t\tAField: \"fox\",\n\t\tBField: pointer.String(\"bull\"),\n\t}\n\n\terr := DeepCopy(&a, &b)\n\trequire.NoError(t, err)\n\n\trequire.EqualValues(t, b.AField, \"fox\")\n\trequire.EqualValues(t, b.AField, a.AField)\n\n\trequire.True(t, a.BField != nil)\n\trequire.True(t, b.BField != nil)\n\trequire.True(t, a.BField != b.BField)\n\trequire.True(t, a.CField == nil)\n\trequire.True(t, b.CField == nil)\n\n\trequire.EqualValues(t, *a.BField, \"bull\")\n\trequire.EqualValues(t, *a.BField, *b.BField)\n}\n"
  },
  {
    "path": "pkg/lib/table/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage table\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrAtLeastOneColumn                  = \"table.at_least_one_column\"\n\tErrHeaderWiderThanMaxWidth           = \"table.header_wider_than_max_width\"\n\tErrHeaderMinWidthGreaterThanMaxWidth = \"table.header_min_width_greater_than_max_width\"\n\tErrWrongNumberOfColumns              = \"table.wrong_number_of_columns\"\n)\n\nfunc ErrorAtLeastOneColumn() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAtLeastOneColumn,\n\t\tMessage: \"must have at least one column\",\n\t})\n}\n\nfunc ErrorHeaderWiderThanMaxWidth(headerTitle string, maxWidth int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrHeaderWiderThanMaxWidth,\n\t\tMessage: fmt.Sprintf(\"header %s is wider than max width (%d)\", headerTitle, maxWidth),\n\t})\n}\n\nfunc ErrorHeaderMinWidthGreaterThanMaxWidth(headerTitle string, minWidth int, maxWidth int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrHeaderMinWidthGreaterThanMaxWidth,\n\t\tMessage: fmt.Sprintf(\"header %s has min width > max width (%d > %d)\", headerTitle, minWidth, maxWidth),\n\t})\n}\n\nfunc ErrorWrongNumberOfColumns(rowNumber int, actualCols int, expectedCols int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrWrongNumberOfColumns,\n\t\tMessage: fmt.Sprintf(\"row %d does not have the expected number of columns (got %d, expected %d)\", rowNumber, actualCols, expectedCols),\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/table/key_value.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage table\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype KeyValuePairOpts struct {\n\tDelimiter     *string // default: \":\"\n\tNumSpaces     *int    // default: 1\n\tRightJustify  *bool   // default: false\n\tBoldFirstLine *bool   // default: false\n\tBoldKeys      *bool   // default: false\n}\n\ntype KeyValuePairs struct {\n\tkvs []kv\n}\n\ntype kv struct {\n\tk interface{}\n\tv interface{}\n}\n\nfunc (kvs *KeyValuePairs) Add(key interface{}, value interface{}) {\n\tkvs.kvs = append(kvs.kvs, kv{k: key, v: value})\n}\n\nfunc (kvs *KeyValuePairs) AddAll(kvs2 KeyValuePairs) {\n\tfor _, pair := range kvs2.kvs {\n\t\tkvs.Add(pair.k, pair.v)\n\t}\n}\n\nfunc (kvs KeyValuePairs) String(options ...*KeyValuePairOpts) string {\n\topts := mergeOptions(options...)\n\n\tvar maxLen int\n\tfor _, pair := range kvs.kvs {\n\t\tkeyLen := len(s.ObjFlatNoQuotes(pair.k))\n\t\tif keyLen > maxLen {\n\t\t\tmaxLen = keyLen\n\t\t}\n\t}\n\n\tvar b strings.Builder\n\tfor i, pair := range kvs.kvs {\n\t\tkeyStr := s.ObjFlatNoQuotes(pair.k)\n\t\tkeyLen := len(keyStr)\n\n\t\tif *opts.BoldKeys {\n\t\t\tkeyStr = console.Bold(keyStr)\n\t\t}\n\n\t\tvalStr := s.ObjFlatNoQuotes(pair.v)\n\t\tvar str string\n\t\tif *opts.RightJustify {\n\t\t\talignmentSpaces := strings.Repeat(\" \", maxLen-keyLen)\n\t\t\tdelimiterSpaces := strings.Repeat(\" \", *opts.NumSpaces)\n\t\t\tstr = alignmentSpaces + keyStr + *opts.Delimiter + delimiterSpaces + valStr + \"\\n\"\n\t\t} else {\n\t\t\tspaces := strings.Repeat(\" \", maxLen-keyLen+*opts.NumSpaces)\n\t\t\tstr = keyStr + *opts.Delimiter + spaces + valStr + \"\\n\"\n\t\t}\n\n\t\tif *opts.BoldFirstLine && i == 0 {\n\t\t\tstr = console.Bold(str)\n\t\t}\n\n\t\tb.WriteString(str)\n\t}\n\n\treturn b.String()\n}\n\nfunc (kvs KeyValuePairs) Print(options ...*KeyValuePairOpts) {\n\tfmt.Print(kvs.String(options...))\n}\n\nfunc mergeOptions(options ...*KeyValuePairOpts) KeyValuePairOpts {\n\tmergedOpts := KeyValuePairOpts{}\n\n\tfor _, opt := range options {\n\t\tif opt != nil && opt.Delimiter != nil {\n\t\t\tmergedOpts.Delimiter = opt.Delimiter\n\t\t}\n\t\tif opt != nil && opt.NumSpaces != nil {\n\t\t\tmergedOpts.NumSpaces = opt.NumSpaces\n\t\t}\n\t\tif opt != nil && opt.RightJustify != nil {\n\t\t\tmergedOpts.RightJustify = opt.RightJustify\n\t\t}\n\t\tif opt != nil && opt.BoldFirstLine != nil {\n\t\t\tmergedOpts.BoldFirstLine = opt.BoldFirstLine\n\t\t}\n\t\tif opt != nil && opt.BoldKeys != nil {\n\t\t\tmergedOpts.BoldKeys = opt.BoldKeys\n\t\t}\n\t}\n\n\tif mergedOpts.Delimiter == nil {\n\t\tmergedOpts.Delimiter = pointer.String(\":\")\n\t}\n\tif mergedOpts.NumSpaces == nil {\n\t\tmergedOpts.NumSpaces = pointer.Int(1)\n\t}\n\tif mergedOpts.RightJustify == nil {\n\t\tmergedOpts.RightJustify = pointer.Bool(false)\n\t}\n\tif mergedOpts.BoldFirstLine == nil {\n\t\tmergedOpts.BoldFirstLine = pointer.Bool(false)\n\t}\n\tif mergedOpts.BoldKeys == nil {\n\t\tmergedOpts.BoldKeys = pointer.Bool(false)\n\t}\n\n\treturn mergedOpts\n}\n"
  },
  {
    "path": "pkg/lib/table/table.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage table\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tlibmath \"github.com/cortexlabs/cortex/pkg/lib/math\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\ntype Table struct {\n\tHeaders []Header\n\tRows    [][]interface{}\n\tSpacing int // Spacing between rows. If 0 is provided, it defaults to 3.\n}\n\ntype Header struct {\n\tTitle    string\n\tMaxWidth int // Max width of the text (not including spacing). Items that are longer will be truncated to less than MaxWidth to fit the ellipses. If 0 is provided, it defaults to no max.\n\tMinWidth int // Min width of the text (not including spacing)\n\tHidden   bool\n}\n\nfunc (t *Table) FindHeaderByTitle(title string) *Header {\n\tfor i, header := range t.Headers {\n\t\tif header.Title == title {\n\t\t\treturn &t.Headers[i]\n\t\t}\n\t}\n\n\treturn nil\n}\n\ntype Opts struct {\n\tSort       *bool // default is true\n\tBoldHeader *bool // default is true\n}\n\nfunc mergeTableOptions(options ...*Opts) Opts {\n\tmergedOpts := Opts{}\n\n\tfor _, opt := range options {\n\t\tif opt != nil {\n\t\t\tif opt.Sort != nil {\n\t\t\t\tmergedOpts.Sort = opt.Sort\n\t\t\t}\n\n\t\t\tif opt.BoldHeader != nil {\n\t\t\t\tmergedOpts.BoldHeader = opt.BoldHeader\n\t\t\t}\n\t\t}\n\t}\n\n\tif mergedOpts.Sort == nil {\n\t\tmergedOpts.Sort = pointer.Bool(true)\n\t}\n\n\tif mergedOpts.BoldHeader == nil {\n\t\tmergedOpts.BoldHeader = pointer.Bool(true)\n\t}\n\n\treturn mergedOpts\n}\n\nfunc validate(t Table) error {\n\tnumCols := len(t.Headers)\n\n\tif numCols < 1 {\n\t\treturn ErrorAtLeastOneColumn()\n\t}\n\n\tfor _, header := range t.Headers {\n\t\tif header.MaxWidth != 0 && len(header.Title) > header.MaxWidth {\n\t\t\treturn ErrorHeaderWiderThanMaxWidth(header.Title, header.MaxWidth)\n\t\t}\n\n\t\tif header.MinWidth > header.MaxWidth {\n\t\t\treturn ErrorHeaderMinWidthGreaterThanMaxWidth(header.Title, header.MinWidth, header.MaxWidth)\n\t\t}\n\t}\n\n\tfor i, row := range t.Rows {\n\t\tif len(row) != numCols {\n\t\t\treturn ErrorWrongNumberOfColumns(i, len(row), numCols)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Prints the error message as a string (if there is an error)\nfunc (t *Table) MustPrint(opts ...*Opts) {\n\tfmt.Print(t.MustFormat(opts...))\n}\n\n// Return the error message as a string\nfunc (t *Table) MustFormat(opts ...*Opts) string {\n\tstr, err := t.Format(opts...)\n\tif err != nil {\n\t\treturn \"error: \" + errors.Message(err)\n\t}\n\treturn str\n}\n\nfunc (t *Table) Format(opts ...*Opts) (string, error) {\n\tmergedOpts := mergeTableOptions(opts...)\n\tif err := validate(*t); err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif t.Spacing <= 0 {\n\t\tt.Spacing = 3\n\t}\n\n\tcolWidths := make([]int, len(t.Headers))\n\tfor colNum, header := range t.Headers {\n\t\tcolWidths[colNum] = len(header.Title)\n\t}\n\n\trows := make([][]string, len(t.Rows))\n\tfor rowNum, row := range t.Rows {\n\t\trows[rowNum] = make([]string, len(row))\n\t\tfor colNum, val := range row {\n\t\t\tstrVal := s.ObjFlatNoQuotes(val)\n\t\t\trows[rowNum][colNum] = strVal\n\t\t\tif len(strVal) > colWidths[colNum] {\n\t\t\t\tcolWidths[colNum] = len(strVal)\n\t\t\t}\n\t\t}\n\t}\n\n\tmaxColWidths := make([]int, len(t.Headers))\n\tfor colNum, colWidth := range colWidths {\n\t\tif t.Headers[colNum].MaxWidth <= 0 {\n\t\t\tmaxColWidths[colNum] = colWidth\n\t\t} else {\n\t\t\tmaxColWidths[colNum] = libmath.MinInt(colWidth, t.Headers[colNum].MaxWidth)\n\t\t}\n\n\t\tif maxColWidths[colNum] < t.Headers[colNum].MinWidth {\n\t\t\tmaxColWidths[colNum] = t.Headers[colNum].MinWidth\n\t\t}\n\t}\n\n\tlastColIndex := len(t.Headers) - 1\n\n\tvar headerStr string\n\tfor colNum, header := range t.Headers {\n\t\tif header.Hidden {\n\t\t\tcontinue\n\t\t}\n\n\t\tif *mergedOpts.BoldHeader {\n\t\t\theaderStr += console.Bold(header.Title)\n\t\t} else {\n\t\t\theaderStr += header.Title\n\t\t}\n\t\tif colNum != lastColIndex {\n\t\t\theaderStr += strings.Repeat(\" \", maxColWidths[colNum]+t.Spacing-len(header.Title))\n\t\t}\n\t}\n\theaderStr = s.TrimTrailingWhitespace(headerStr)\n\n\tellipses := \"...\"\n\trowStrs := make([]string, len(rows))\n\tfor rowNum, row := range rows {\n\t\tvar rowStr string\n\t\tfor colNum, val := range row {\n\t\t\tif t.Headers[colNum].Hidden {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(val) > maxColWidths[colNum] {\n\t\t\t\tval = val[0:maxColWidths[colNum]]\n\t\t\t\t// Ensure at least one space after ellipses\n\t\t\t\tfor len(val)+len(ellipses) > maxColWidths[colNum]+t.Spacing-1 {\n\t\t\t\t\tval = val[0 : len(val)-1]\n\t\t\t\t}\n\t\t\t\tval += ellipses\n\t\t\t}\n\t\t\trowStr += val\n\t\t\tif colNum != lastColIndex {\n\t\t\t\trowStr += strings.Repeat(\" \", maxColWidths[colNum]+t.Spacing-len(val))\n\t\t\t}\n\t\t}\n\t\trowStrs[rowNum] = s.TrimTrailingWhitespace(rowStr)\n\t}\n\n\tif *mergedOpts.Sort {\n\t\tsort.Strings(rowStrs)\n\t}\n\n\treturn headerStr + \"\\n\" + strings.Join(rowStrs, \"\\n\") + \"\\n\", nil\n}\n"
  },
  {
    "path": "pkg/lib/telemetry/error_cache.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage telemetry\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\nconst (\n\t_initialCoolDownPeriod = 5 * time.Second  // The initial interval to wait before sending duplicate errors\n\t_maxCoolDownPeriod     = 24 * time.Hour   // The longest interval to wait before sending duplicate errors\n\t_coolDownFactor        = 2                // Factor by which to increase cool down period after each error report\n\t_cacheEvictionPeriod   = 24 * time.Hour   // Duration of not seeing an error after which seeing it again resets the cooldown period\n\t_cacheCleanupInterval  = 10 * time.Minute // Minimum time to wait before checking error cache for errors to evict\n)\n\nvar _errorCache = struct {\n\tm           map[string]*errorStatus\n\tlastCleanup time.Time\n\tsync.RWMutex\n}{m: make(map[string]*errorStatus)}\n\ntype errorStatus struct {\n\tLastReportTime time.Time\n\tLastSeenTime   time.Time\n\tCoolDownPeriod time.Duration\n}\n\nfunc shouldBlock(err error, backoffMode BackoffMode) bool {\n\tif backoffMode == NoBackoff {\n\t\treturn false\n\t}\n\n\terrMsg := errors.MessageFirstLine(err)\n\tnow := time.Now()\n\n\tif backoffMode == BackoffAnyMessages {\n\t\terrMsg = \"<msg>\"\n\t}\n\n\tdefer func() {\n\t\tgo cleanupCache()\n\t}()\n\n\t_errorCache.Lock()\n\tdefer _errorCache.Unlock()\n\n\terrStatus, ok := _errorCache.m[errMsg]\n\n\tif !ok || time.Since(errStatus.LastSeenTime) > _cacheEvictionPeriod {\n\t\t_errorCache.m[errMsg] = &errorStatus{\n\t\t\tLastReportTime: now,\n\t\t\tLastSeenTime:   now,\n\t\t\tCoolDownPeriod: _initialCoolDownPeriod,\n\t\t}\n\t\treturn false\n\t}\n\n\tif time.Since(errStatus.LastReportTime) > errStatus.CoolDownPeriod {\n\t\terrStatus.LastSeenTime = now\n\t\terrStatus.LastReportTime = now\n\t\terrStatus.CoolDownPeriod = time.Duration(float64(errStatus.CoolDownPeriod.Nanoseconds())*_coolDownFactor) * time.Nanosecond\n\t\tif errStatus.CoolDownPeriod > _maxCoolDownPeriod {\n\t\t\terrStatus.CoolDownPeriod = _maxCoolDownPeriod\n\t\t}\n\t\treturn false\n\t}\n\n\terrStatus.LastSeenTime = now\n\treturn true\n}\n\nfunc cleanupCache() {\n\tstaleErrorMessages := findStaleCachedErrors()\n\n\tif len(staleErrorMessages) == 0 {\n\t\treturn\n\t}\n\n\t_errorCache.Lock()\n\tdefer _errorCache.Unlock()\n\tfor errMsg := range staleErrorMessages {\n\t\tdelete(_errorCache.m, errMsg)\n\t}\n}\n\nfunc findStaleCachedErrors() strset.Set {\n\t_errorCache.RLock()\n\tdefer _errorCache.RUnlock()\n\n\tif time.Since(_errorCache.lastCleanup) < _cacheCleanupInterval {\n\t\treturn nil\n\t}\n\n\t_errorCache.lastCleanup = time.Now()\n\n\tstaleErrorMessages := strset.New()\n\tfor errMsg, errStatus := range _errorCache.m {\n\t\tif time.Since(errStatus.LastSeenTime) > _cacheEvictionPeriod {\n\t\t\tstaleErrorMessages.Add(errMsg)\n\t\t}\n\t}\n\n\treturn staleErrorMessages\n}\n"
  },
  {
    "path": "pkg/lib/telemetry/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage telemetry\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrUserIDNotSpecified         = \"telemetry.user_id_not_specified\"\n\tErrSentryFlushTimeoutExceeded = \"telemetry.sentry_flush_timeout_exceeded\"\n)\n\nfunc ErrorUserIDNotSpecified() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUserIDNotSpecified,\n\t\tMessage: \"user ID must be specified to enable telemetry\",\n\t})\n}\n\nfunc ErrorSentryFlushTimeoutExceeded() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSentryFlushTimeoutExceeded,\n\t\tMessage: \"sentry flush timout exceeded\",\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/telemetry/telemetry.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage telemetry\n\nimport (\n\t\"os\"\n\t\"reflect\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/maps\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/getsentry/sentry-go\"\n\t\"gopkg.in/segmentio/analytics-go.v3\"\n)\n\nvar _sentryDSN = \"https://5cea3d2d67194d028f7191fcc6ebca14@sentry.io/1825326\"\nvar _segmentWriteKey = \"BNhXifMk9EyhPICF2zAFpWYPCf4CRpV1\"\n\nvar _segment analytics.Client\nvar _config *Config\n\ntype Config struct {\n\tEnabled     bool\n\tUserID      string\n\tProperties  map[string]string\n\tEnvironment string\n\tLogErrors   bool\n\tBackoffMode BackoffMode\n}\n\ntype BackoffMode int\n\nconst (\n\tNoBackoff BackoffMode = iota\n\tBackoffDuplicateMessages\n\tBackoffAnyMessages\n)\n\ntype silentSegmentLogger struct{}\n\nfunc (logger silentSegmentLogger) Logf(_ string, _ ...interface{}) {\n\treturn\n}\n\nfunc (logger silentSegmentLogger) Errorf(_ string, _ ...interface{}) {\n\treturn\n}\n\ntype silentSentryLogger struct{}\n\nfunc (logger silentSentryLogger) Write(p []byte) (n int, err error) {\n\treturn len(p), nil\n}\n\nfunc getSentryDSN() string {\n\tif envVar := os.Getenv(\"CORTEX_TELEMETRY_SENTRY_DSN\"); envVar != \"\" {\n\t\treturn envVar\n\t}\n\treturn _sentryDSN\n}\n\nfunc Init(telemetryConfig Config) error {\n\tif !telemetryConfig.Enabled {\n\t\t_config = nil\n\t\treturn nil\n\t}\n\n\tif telemetryConfig.UserID == \"\" {\n\t\treturn ErrorUserIDNotSpecified()\n\t}\n\n\terr := sentry.Init(sentry.ClientOptions{\n\t\tDsn:         getSentryDSN(),\n\t\tRelease:     consts.CortexVersion,\n\t\tEnvironment: telemetryConfig.Environment,\n\t})\n\tif err != nil {\n\t\t_config = nil\n\t\treturn err\n\t}\n\n\tvar segmentLogger analytics.Logger\n\tif !telemetryConfig.LogErrors {\n\t\tsentry.Logger.SetOutput(silentSentryLogger{})\n\t\tsegmentLogger = silentSegmentLogger{}\n\t}\n\n\twriteKey := _segmentWriteKey\n\tif envVar := os.Getenv(\"CORTEX_TELEMETRY_SEGMENT_WRITE_KEY\"); envVar != \"\" {\n\t\twriteKey = envVar\n\t}\n\n\t_segment, err = analytics.NewWithConfig(writeKey, analytics.Config{\n\t\tBatchSize: 1,\n\t\tLogger:    segmentLogger,\n\t\tDefaultContext: &analytics.Context{\n\t\t\tApp: analytics.AppInfo{\n\t\t\t\tVersion: consts.CortexVersion,\n\t\t\t},\n\t\t\tDevice: analytics.DeviceInfo{\n\t\t\t\tType: telemetryConfig.Environment,\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\t_config = nil\n\t\treturn err\n\t}\n\n\t_config = &telemetryConfig\n\treturn nil\n}\n\nfunc Event(name string, properties ...map[string]interface{}) {\n\tintegrations := map[string]interface{}{\n\t\t\"All\":   true,\n\t\t\"Slack\": false,\n\t}\n\n\teventHelper(name, maps.MergeStrInterfaceMaps(properties...), integrations)\n}\n\nfunc EventNotify(name string, properties ...map[string]interface{}) {\n\tintegrations := map[string]interface{}{\n\t\t\"All\": true,\n\t}\n\n\teventHelper(name, maps.MergeStrInterfaceMaps(properties...), integrations)\n}\n\nfunc eventHelper(name string, properties map[string]interface{}, integrations map[string]interface{}) {\n\tif _config == nil || !_config.Enabled || strings.ToLower(os.Getenv(\"CORTEX_TELEMETRY_DISABLE\")) == \"true\" {\n\t\treturn\n\t}\n\n\tmergedProperties := maps.MergeStrInterfaceMaps(properties, cast.StrMapToStrInterfaceMap(_config.Properties))\n\n\terr := _segment.Enqueue(analytics.Track{\n\t\tEvent:        name,\n\t\tUserId:       _config.UserID,\n\t\tProperties:   mergedProperties,\n\t\tIntegrations: integrations,\n\t})\n\tif err != nil {\n\t\tError(err)\n\t}\n}\n\nfunc Error(err error, tags ...map[string]string) {\n\tif err == nil || _config == nil || errors.IsNoTelemetry(err) {\n\t\treturn\n\t}\n\n\tif shouldBlock(err, _config.BackoffMode) {\n\t\treturn\n\t}\n\n\tmergedTags := maps.MergeStrMapsString(tags...)\n\n\tsentry.WithScope(func(scope *sentry.Scope) {\n\t\te := EventFromException(err)\n\t\tscope.SetUser(sentry.User{ID: _config.UserID})\n\t\tscope.SetTags(maps.MergeStrMapsString(_config.Properties, mergedTags))\n\t\tscope.SetTags(map[string]string{\"error_type\": e.Exception[0].Type})\n\t\tsentry.CaptureEvent(e)\n\n\t\tgo sentry.Flush(10 * time.Second)\n\t})\n}\n\nfunc EventFromException(exception error) *sentry.Event {\n\tstacktrace := sentry.ExtractStacktrace(exception)\n\n\tif stacktrace == nil {\n\t\tstacktrace = sentry.NewStacktrace()\n\t}\n\n\terrTypeString := reflect.TypeOf(errors.CauseOrSelf(exception)).String()\n\n\terrKind := errors.GetKind(exception)\n\tif errKind != \"\" && errKind != errors.ErrNotCortexError {\n\t\terrTypeString = errKind\n\t}\n\n\tevent := sentry.NewEvent()\n\tevent.Level = sentry.LevelError\n\n\tvalue := errors.Message(exception)\n\tif metadata := errors.GetMetadata(exception); metadata != nil {\n\t\tvalue = value + \"\\n\\n########## metadata ##########\\n\" + s.ObjStripped(metadata)\n\t}\n\n\tevent.Exception = []sentry.Exception{{\n\t\tValue:      value,\n\t\tType:       errTypeString,\n\t\tStacktrace: stacktrace,\n\t}}\n\treturn event\n}\n\nfunc RecordOperatorID(clientID string, operatorID string) {\n\tif _config == nil || !_config.Enabled || strings.ToLower(os.Getenv(\"CORTEX_TELEMETRY_DISABLE\")) == \"true\" {\n\t\treturn\n\t}\n\n\t_ = _segment.Enqueue(analytics.Identify{\n\t\tUserId: clientID,\n\t\tTraits: analytics.NewTraits().\n\t\t\tSet(\"operator_id\", operatorID),\n\t})\n}\n\nfunc closeSentry() error {\n\tif !sentry.Flush(5 * time.Second) {\n\t\treturn ErrorSentryFlushTimeoutExceeded()\n\t}\n\treturn nil\n}\n\nfunc closeSegment() error {\n\tif _segment == nil {\n\t\treturn nil\n\t}\n\treturn _segment.Close()\n}\n\nfunc Close() {\n\tparallel.Run(closeSegment, closeSentry)\n\t_config = nil\n}\n"
  },
  {
    "path": "pkg/lib/time/time.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage time\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\nfunc MicrosecsStr(t time.Time) string {\n\tnanos := fmt.Sprintf(\"%09d\", t.Nanosecond())\n\treturn nanos[0:6]\n}\n\nfunc MillisecsStr(t time.Time) string {\n\tnanos := fmt.Sprintf(\"%09d\", t.Nanosecond())\n\treturn nanos[0:3]\n}\n\nfunc Timestamp(t time.Time) string {\n\tmicroseconds := MicrosecsStr(t)\n\treturn fmt.Sprintf(\"%d-%02d-%02d-%02d-%02d-%02d-%s\",\n\t\tt.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), microseconds)\n}\n\nfunc PtrsEqual(t1 *time.Time, t2 *time.Time) bool {\n\tif t1 == nil && t2 == nil {\n\t\treturn true\n\t}\n\tif t1 == nil || t2 == nil {\n\t\treturn false\n\t}\n\treturn t1.Equal(*t2)\n}\n\nfunc CopyPtr(t *time.Time) *time.Time {\n\tif t == nil {\n\t\treturn nil\n\t}\n\ttCopy := *t\n\treturn &tCopy\n}\n\nfunc DifferenceStr(t1 *time.Time, t2 *time.Time) string {\n\tvar duration time.Duration\n\tif t1 == nil && t2 == nil {\n\t\treturn \"-\"\n\t} else if t1 == nil {\n\t\treturn \"infinity\"\n\t} else if t2 == nil {\n\t\tduration = time.Since(*t1)\n\t} else {\n\t\tduration = (*t2).Sub(*t1)\n\t}\n\n\tdurationSecs := int(duration.Seconds())\n\tif durationSecs < 60 {\n\t\treturn strconv.Itoa(durationSecs) + \"s\"\n\t} else if durationSecs < 3600 {\n\t\treturn strconv.Itoa(durationSecs/60) + \"m\" + strconv.Itoa(durationSecs-durationSecs/60*60) + \"s\"\n\t} else if durationSecs < 48*3600 {\n\t\treturn strconv.Itoa(durationSecs/3600) + \"h\" + strconv.Itoa((durationSecs-durationSecs/3600*3600)/60) + \"m\"\n\t} else {\n\t\treturn strconv.Itoa(durationSecs/(24*3600)) + \"d\" + strconv.Itoa((durationSecs-durationSecs/(24*3600)*(24*3600))/3600) + \"h\"\n\t}\n}\n\nfunc SinceStr(t *time.Time) string {\n\tif t == nil {\n\t\treturn \"-\"\n\t}\n\tnow := time.Now()\n\treturn DifferenceStr(t, &now)\n}\n\nfunc LocalTimestamp(t *time.Time) string {\n\tif t == nil {\n\t\treturn \"-\"\n\t}\n\treturn (*t).Local().Format(\"2006-01-02 15:04:05 MST\")\n}\n\nfunc LocalTimestampHuman(t *time.Time) string {\n\tif t == nil {\n\t\treturn \"-\"\n\t}\n\treturn (*t).Local().Format(\"Monday, January 2, 2006 at 3:04pm MST\")\n}\n\nfunc LocalHourNow() string {\n\treturn time.Now().Local().Format(\"3:04:05pm MST\")\n}\n\nfunc MillisToTime(epochMillis int64) time.Time {\n\tseconds := epochMillis / 1000\n\tmillis := epochMillis % 1000\n\treturn time.Unix(seconds, millis*int64(time.Millisecond))\n}\n\nfunc ToMillis(t time.Time) int64 {\n\treturn t.UnixNano() / int64(time.Millisecond)\n}\n\ntype Timer struct {\n\tnames []string\n\tstart time.Time\n\tlast  time.Time\n}\n\nfunc StartTimer(names ...string) Timer {\n\treturn Timer{\n\t\tnames: names,\n\t\tstart: time.Now(),\n\t}\n}\n\nfunc (t *Timer) Print(messages ...string) {\n\tnow := time.Now()\n\n\tseparator := \"\"\n\tif len(t.names)+len(messages) > 0 {\n\t\tseparator = \": \"\n\t}\n\n\ttotalTime := fmt.Sprintf(\"%s total\", now.Sub(t.start))\n\n\tstepTime := \"\"\n\tif !t.last.IsZero() {\n\t\tstepTime = fmt.Sprintf(\"%s step, \", now.Sub(t.last))\n\t}\n\n\tfmt.Println(strings.Join(append(t.names, messages...), \": \") + separator + stepTime + totalTime)\n\n\tt.last = now\n}\n\nfunc MustParseDuration(str string) time.Duration {\n\td, err := time.ParseDuration(str)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn d\n}\n\nfunc MaxDuration(duration time.Duration, durations ...time.Duration) time.Duration {\n\tmax := duration\n\tfor _, d := range durations {\n\t\tif d > max {\n\t\t\tmax = d\n\t\t}\n\t}\n\n\treturn max\n}\n\nfunc GetCurrentUTCDate() time.Time {\n\ttimestamp := time.Now().UTC()\n\tyear, month, day := timestamp.Date()\n\treturn time.Date(year, month, day, 0, 0, 0, 0, time.UTC)\n}\n"
  },
  {
    "path": "pkg/lib/urls/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage urls\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrInvalidURL          = \"urls.invalid_url\"\n\tErrDNS1035             = \"urls.dns1035\"\n\tErrDNS1123             = \"urls.dns1123\"\n\tErrEndpoint            = \"urls.endpoint\"\n\tErrEndpointEmptyPath   = \"urls.endpoint_empty_path\"\n\tErrEndpointDoubleSlash = \"urls.endpoint_double_slash\"\n)\n\nfunc ErrorInvalidURL(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidURL,\n\t\tMessage: fmt.Sprintf(\"%s is not a valid URL\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorDNS1035(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDNS1035,\n\t\tMessage: fmt.Sprintf(\"%s must contain only lower case letters, numbers, and dashes, start with a letter, and cannot end with a dash\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorDNS1123(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDNS1123,\n\t\tMessage: fmt.Sprintf(\"%s must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorEndpoint(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEndpoint,\n\t\tMessage: fmt.Sprintf(\"%s must consist of lower case alphanumeric characters, '/', '-', '_', or '.'\", s.UserStr(provided)),\n\t})\n}\n\nfunc ErrorEndpointEmptyPath() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEndpointEmptyPath,\n\t\tMessage: fmt.Sprintf(\"%s is not allowed (a path must be specified)\", s.UserStr(\"/\")),\n\t})\n}\n\nfunc ErrorEndpointDoubleSlash(provided string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrEndpointDoubleSlash,\n\t\tMessage: fmt.Sprintf(\"%s cannot contain adjacent slashes\", s.UserStr(provided)),\n\t})\n}\n"
  },
  {
    "path": "pkg/lib/urls/urls.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage urls\n\nimport (\n\t\"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nvar (\n\t_dns1035Regex   = regexp.MustCompile(`^[a-z]([-a-z0-9]*[a-z0-9])?$`)\n\t_dns1123Regex   = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`)\n\t_endpointRegex  = regexp.MustCompile(`^[a-zA-Z0-9_\\-\\./]*$`)\n\t_urlQParamRegex = regexp.MustCompile(`(https?://.*)\\?[^:\\s]*`)\n)\n\nfunc Parse(rawurl string) (*url.URL, error) {\n\tu, err := url.Parse(rawurl)\n\tif err != nil {\n\t\treturn nil, ErrorInvalidURL(rawurl)\n\t}\n\treturn u, nil\n}\n\nfunc Join(str string, strs ...string) string {\n\tfullPath := str\n\tfor _, str := range strs {\n\t\tfullPath = s.EnsureSuffix(fullPath, \"/\")\n\t\tfullPath = fullPath + strings.TrimPrefix(str, \"/\")\n\t}\n\treturn fullPath\n}\n\nfunc CheckDNS1035(str string) error {\n\tif !_dns1035Regex.MatchString(str) {\n\t\treturn ErrorDNS1035(str)\n\t}\n\treturn nil\n}\n\nfunc CheckDNS1123(str string) error {\n\tif !_dns1123Regex.MatchString(str) {\n\t\treturn ErrorDNS1123(str)\n\t}\n\treturn nil\n}\n\nfunc ValidateEndpointAllowEmptyPath(str string) (string, error) {\n\tif !_endpointRegex.MatchString(str) {\n\t\treturn \"\", ErrorEndpoint(str)\n\t}\n\n\tif strings.Contains(str, \"//\") {\n\t\treturn \"\", ErrorEndpointDoubleSlash(str)\n\t}\n\n\treturn CanonicalizeEndpoint(str), nil\n}\n\nfunc ValidateEndpoint(str string) (string, error) {\n\tpath, err := ValidateEndpointAllowEmptyPath(str)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif path == \"/\" {\n\t\treturn \"\", ErrorEndpointEmptyPath()\n\t}\n\n\treturn path, nil\n}\n\nfunc CanonicalizeEndpoint(str string) string {\n\tif str == \"\" || str == \"/\" {\n\t\treturn \"/\"\n\t}\n\treturn strings.TrimSuffix(s.EnsurePrefix(str, \"/\"), \"/\")\n}\n\nfunc CanonicalizeEndpointWithTrailingSlash(str string) string {\n\tif str == \"\" || str == \"/\" {\n\t\treturn \"/\"\n\t}\n\treturn s.EnsureSuffix(s.EnsurePrefix(str, \"/\"), \"/\")\n}\n\nfunc TrimQueryParamsURL(u url.URL) string {\n\tu.RawQuery = \"\"\n\treturn u.String()\n}\n\nfunc TrimQueryParamsStr(str string) string {\n\treturn _urlQParamRegex.ReplaceAllString(str, \"$1\")\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/delete.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc Delete(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\tkeepCache := getOptionalBoolQParam(\"keepCache\", false, r)\n\n\tresponse, err := resources.DeleteAPI(apiName, keepCache)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\trespondJSON(w, r, response)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/deploy.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/files\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n)\n\nfunc Deploy(w http.ResponseWriter, r *http.Request) {\n\tforce := getOptionalBoolQParam(\"force\", false, r)\n\n\tconfigFileName, err := getRequiredQueryParam(\"configFileName\", r)\n\tif err != nil {\n\t\trespondError(w, r, errors.WithStack(err))\n\t\treturn\n\t}\n\n\tconfigBytes, err := files.ReadReqFile(r, \"config\")\n\tif err != nil {\n\t\trespondError(w, r, errors.WithStack(err))\n\t\treturn\n\t} else if len(configBytes) == 0 {\n\t\trespondError(w, r, ErrorFormFileMustBeProvided(\"config\"))\n\t\treturn\n\t}\n\n\tresponse, err := resources.Deploy(configFileName, configBytes, force)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, response)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/describe.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc DescribeAPI(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\n\tresponse, err := resources.DescribeAPI(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, response)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n)\n\nconst (\n\tErrAPIVersionMismatch     = \"endpoints.api_version_mismatch\"\n\tErrHeaderMissing          = \"endpoints.header_missing\"\n\tErrHeaderMalformed        = \"endpoints.header_malformed\"\n\tErrAuthAPIError           = \"endpoints.auth_api_error\"\n\tErrFormFileMustBeProvided = \"endpoints.form_file_must_be_provided\"\n\tErrAuthInvalid            = \"endpoints.auth_invalid\"\n\tErrAuthOtherAccount       = \"endpoints.auth_other_account\"\n\tErrQueryParamRequired     = \"endpoints.query_param_required\"\n\tErrPathParamRequired      = \"endpoints.path_param_required\"\n\tErrAnyQueryParamRequired  = \"endpoints.any_query_param_required\"\n\tErrAnyPathParamRequired   = \"endpoints.any_path_param_required\"\n\tErrLogsJobIDRequired      = \"endpoints.logs_job_id_required\"\n)\n\nfunc ErrorAPIVersionMismatch(operatorVersion string, clientVersion string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAPIVersionMismatch,\n\t\tMessage: fmt.Sprintf(\"your CLI version (%s) doesn't match your Cortex operator version (%s); please update your cluster by following the instructions at https://docs.cortexlabs.com, or update your CLI (pip install cortex==%s)\", clientVersion, operatorVersion, operatorVersion),\n\t})\n}\n\nfunc ErrorHeaderMissing(header string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrHeaderMissing,\n\t\tMessage: fmt.Sprintf(\"missing %s header\", header),\n\t})\n}\n\nfunc ErrorAuthHeaderMissing(header, host, url string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrHeaderMissing,\n\t\tMessage: fmt.Sprintf(\"missing %s header\", header),\n\t\tMetadata: map[string]string{\n\t\t\t\"host\": host,\n\t\t\t\"url\":  url,\n\t\t},\n\t})\n}\n\nfunc ErrorHeaderMalformed(header string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrHeaderMalformed,\n\t\tMessage: fmt.Sprintf(\"malformed %s header\", header),\n\t})\n}\n\nfunc ErrorAuthAPIError() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAuthAPIError,\n\t\tMessage: \"the operator is unable to verify user's credentials using AWS STS; run `aws sts get-caller-identity` to view the credentials being used by the cortex client\",\n\t})\n}\n\nfunc ErrorAuthInvalid() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAuthInvalid,\n\t\tMessage: \"invalid AWS credentials; run `aws sts get-caller-identity` to view the credentials being used by the cortex client\",\n\t})\n}\n\nfunc ErrorAuthOtherAccount() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAuthOtherAccount,\n\t\tMessage: \"the AWS account associated with your CLI's AWS credentials differs from the AWS account associated with your cluster's AWS credentials; run `aws sts get-caller-identity` to view the credentials being used by the cortex client\",\n\t})\n}\n\nfunc ErrorFormFileMustBeProvided(fileName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFormFileMustBeProvided,\n\t\tMessage: fmt.Sprintf(\"request form file %s must be provided\", s.UserStr(fileName)),\n\t})\n}\nfunc ErrorQueryParamRequired(param string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrQueryParamRequired,\n\t\tMessage: fmt.Sprintf(\"query param required: %s\", param),\n\t})\n}\n\nfunc ErrorPathParamRequired(param string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrPathParamRequired,\n\t\tMessage: fmt.Sprintf(\"path param required: %s\", param),\n\t})\n}\n\nfunc ErrorAnyQueryParamRequired(param string, params ...string) error {\n\tallParams := append([]string{param}, params...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAnyQueryParamRequired,\n\t\tMessage: fmt.Sprintf(\"query params required: %s\", s.UserStrsOr(allParams)),\n\t})\n}\n\nfunc ErrorAnyPathParamRequired(param string, params ...string) error {\n\tallParams := append([]string{param}, params...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAnyPathParamRequired,\n\t\tMessage: fmt.Sprintf(\"path params required: %s\", s.UserStrsOr(allParams)),\n\t})\n}\n\nfunc ErrorLogsJobIDRequired(resource operator.DeployedResource) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrLogsJobIDRequired,\n\t\tMessage: fmt.Sprintf(\"job id is required for %s; you can get a list of latest job ids with `cortex get %s` and use `cortex logs %s JOB_ID` to get the logs\", resource.UserString(), resource.Name, resource.Name),\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/get.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc GetAPIs(w http.ResponseWriter, r *http.Request) {\n\tresponse, err := resources.GetAPIs()\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, response)\n}\n\nfunc GetAPI(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\n\tresponse, err := resources.GetAPI(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, response)\n}\n\nfunc GetAPIByID(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\tapiID := mux.Vars(r)[\"apiID\"]\n\n\tresponse, err := resources.GetAPIByID(apiName, apiID)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, response)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/get_batch_job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc GetBatchJob(w http.ResponseWriter, r *http.Request) {\n\tvars := mux.Vars(r)\n\tapiName := vars[\"apiName\"]\n\tjobID, err := getRequiredQueryParam(\"jobID\", r)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tdeployedResource, err := resources.GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\tif deployedResource.Kind != userconfig.BatchAPIKind {\n\t\trespondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind))\n\t\treturn\n\t}\n\n\tjobKey := spec.JobKey{\n\t\tAPIName: apiName,\n\t\tID:      jobID,\n\t\tKind:    userconfig.BatchAPIKind,\n\t}\n\n\tjobResponse, err := batchapi.GetJob(jobKey)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, jobResponse)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/get_task_job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\t\"net/url\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc GetTaskJob(w http.ResponseWriter, r *http.Request) {\n\tvars := mux.Vars(r)\n\tapiName := vars[\"apiName\"]\n\tjobID, err := getRequiredQueryParam(\"jobID\", r)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tdeployedResource, err := resources.GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\tif deployedResource.Kind != userconfig.TaskAPIKind {\n\t\trespondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.TaskAPIKind))\n\t\treturn\n\t}\n\n\tjobKey := spec.JobKey{\n\t\tAPIName: apiName,\n\t\tID:      jobID,\n\t\tKind:    userconfig.TaskAPIKind,\n\t}\n\n\tjobStatus, err := taskapi.GetJobStatus(jobKey)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tapiSpec, err := operator.DownloadAPISpec(jobStatus.APIName, jobStatus.APIID)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tendpoint, err := operator.APIEndpoint(apiSpec)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tparsedURL, err := url.Parse(endpoint)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t}\n\tq := parsedURL.Query()\n\tq.Add(\"jobID\", jobKey.ID)\n\tparsedURL.RawQuery = q.Encode()\n\n\tresponse := schema.TaskJobResponse{\n\t\tJobStatus: *jobStatus,\n\t\tAPISpec:   *apiSpec,\n\t\tEndpoint:  parsedURL.String(),\n\t}\n\n\trespondJSON(w, r, response)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/info.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\t\"sort\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nfunc Info(w http.ResponseWriter, r *http.Request) {\n\tworkerNodeInfos, numPendingReplicas, err := getWorkerNodeInfos()\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\toperatorNodeInfos, err := getOperatorNodeInfos()\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tfullClusterConfig := clusterconfig.InternalConfig{\n\t\tConfig:           *config.ClusterConfig,\n\t\tOperatorMetadata: *config.OperatorMetadata,\n\t}\n\n\tresponse := schema.InfoResponse{\n\t\tClusterConfig:      fullClusterConfig,\n\t\tWorkerNodeInfos:    workerNodeInfos,\n\t\tOperatorNodeInfos:  operatorNodeInfos,\n\t\tNumPendingReplicas: numPendingReplicas,\n\t}\n\trespondJSON(w, r, response)\n}\n\nfunc getWorkerNodeInfos() ([]schema.WorkerNodeInfo, int, error) {\n\tpods, err := config.K8sAllNamspaces.ListPods(nil)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\n\tnodes, err := config.K8sAllNamspaces.ListNodesByLabel(\"workload\", \"true\")\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\n\tnodeInfoMap := make(map[string]*schema.WorkerNodeInfo, len(nodes)) // node name -> info\n\tspotPriceCache := make(map[string]float64)                         // instance type -> spot price\n\n\tfor i := range nodes {\n\t\tnode := nodes[i]\n\n\t\tinstanceType := node.Labels[\"node.kubernetes.io/instance-type\"]\n\t\tnodeGroupName := node.Labels[\"alpha.eksctl.io/nodegroup-name\"]\n\t\tisSpot := node.Labels[\"node-lifecycle\"] == \"spot\"\n\n\t\tprice := aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price\n\t\tif isSpot {\n\t\t\tif spotPrice, ok := spotPriceCache[instanceType]; ok {\n\t\t\t\tprice = spotPrice\n\t\t\t} else {\n\t\t\t\tspotPrice, err := config.AWS.SpotInstancePrice(instanceType)\n\t\t\t\tif err == nil && spotPrice != 0 {\n\t\t\t\t\tprice = spotPrice\n\t\t\t\t\tspotPriceCache[instanceType] = spotPrice\n\t\t\t\t} else {\n\t\t\t\t\tspotPriceCache[instanceType] = price // the request failed, so no need to try again in the future\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tnodeInfoMap[node.Name] = &schema.WorkerNodeInfo{\n\t\t\tNodeInfo: schema.NodeInfo{\n\t\t\t\tNodeGroupName: nodeGroupName,\n\t\t\t\tInstanceType:  instanceType,\n\t\t\t\tIsSpot:        isSpot,\n\t\t\t\tPrice:         price,\n\t\t\t},\n\t\t\tName:                 node.Name,\n\t\t\tNumReplicas:          0,                             // will be added to below\n\t\t\tComputeUserCapacity:  nodeComputeAllocatable(&node), // will be subtracted from below\n\t\t\tComputeAvailable:     nodeComputeAllocatable(&node), // will be subtracted from below\n\t\t\tComputeUserRequested: userconfig.ZeroCompute(),      // will be added to below\n\t\t}\n\t}\n\n\tvar numPendingReplicas int\n\n\tfor i := range pods {\n\t\tpod := pods[i]\n\n\t\tif pod.Status.Phase == kcore.PodSucceeded || pod.Status.Phase == kcore.PodFailed {\n\t\t\t// note: pending pods can be scheduled on nodes (image pull in progress)\n\t\t\tcontinue\n\t\t}\n\n\t\t_, isAPIPod := pod.Labels[\"apiName\"]\n\t\tbatchPodType, isBatchPod := pod.Labels[\"cortex.dev/batch\"]\n\n\t\tif pod.Spec.NodeName == \"\" && isAPIPod {\n\t\t\tnumPendingReplicas++\n\t\t\tcontinue\n\t\t}\n\n\t\tnode, ok := nodeInfoMap[pod.Spec.NodeName]\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\n\t\tif isAPIPod {\n\t\t\tif isBatchPod && batchPodType == \"enqueuer\" {\n\t\t\t\tnode.NumEnqueuerReplicas++\n\t\t\t} else {\n\t\t\t\tnode.NumReplicas++\n\t\t\t}\n\t\t}\n\n\t\tcpu, mem, gpu, inf := k8s.TotalPodCompute(&pod.Spec)\n\n\t\tnode.ComputeAvailable.CPU.SubQty(cpu)\n\t\tnode.ComputeAvailable.Mem.SubQty(mem)\n\t\tnode.ComputeAvailable.GPU -= gpu\n\t\tnode.ComputeAvailable.Inf -= inf\n\n\t\tif isAPIPod {\n\t\t\tnode.ComputeUserRequested.CPU.AddQty(cpu)\n\t\t\tnode.ComputeUserRequested.Mem.AddQty(mem)\n\t\t\tnode.ComputeUserRequested.GPU += gpu\n\t\t\tnode.ComputeUserRequested.Inf += inf\n\t\t} else {\n\t\t\tnode.ComputeUserCapacity.CPU.SubQty(cpu)\n\t\t\tnode.ComputeUserCapacity.Mem.SubQty(mem)\n\t\t\tnode.ComputeUserCapacity.GPU -= gpu\n\t\t\tnode.ComputeUserCapacity.Inf -= inf\n\t\t}\n\t}\n\n\tnodeNames := make([]string, 0, len(nodeInfoMap))\n\tfor nodeName := range nodeInfoMap {\n\t\tnodeNames = append(nodeNames, nodeName)\n\t}\n\n\tsort.Strings(nodeNames)\n\n\tnodeInfos := make([]schema.WorkerNodeInfo, len(nodeNames))\n\tfor i, nodeName := range nodeNames {\n\t\tnodeInfos[i] = *nodeInfoMap[nodeName]\n\t}\n\n\treturn nodeInfos, numPendingReplicas, nil\n}\n\nfunc nodeComputeAllocatable(node *kcore.Node) userconfig.Compute {\n\tgpuQty := node.Status.Allocatable[\"nvidia.com/gpu\"]\n\tinfQty := node.Status.Allocatable[\"aws.amazon.com/neuron\"]\n\n\treturn userconfig.Compute{\n\t\tCPU: k8s.WrapQuantity(*node.Status.Allocatable.Cpu()),\n\t\tMem: k8s.WrapQuantity(*node.Status.Allocatable.Memory()),\n\t\tGPU: gpuQty.Value(),\n\t\tInf: infQty.Value(),\n\t}\n}\n\nfunc getOperatorNodeInfos() ([]schema.NodeInfo, error) {\n\tnodes, err := config.K8sAllNamspaces.ListNodesByLabel(\"operator\", \"true\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tnodeInfoMap := make(map[string]*schema.NodeInfo, len(nodes)) // node name -> info\n\n\tfor i := range nodes {\n\t\tnode := nodes[i]\n\n\t\tinstanceType := node.Labels[\"node.kubernetes.io/instance-type\"]\n\t\tnodeGroupName := node.Labels[\"alpha.eksctl.io/nodegroup-name\"]\n\n\t\tprice := aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price\n\n\t\tnodeInfoMap[node.Name] = &schema.NodeInfo{\n\t\t\tNodeGroupName: nodeGroupName,\n\t\t\tInstanceType:  instanceType,\n\t\t\tPrice:         price,\n\t\t}\n\t}\n\n\tnodeNames := make([]string, 0, len(nodeInfoMap))\n\tfor nodeName := range nodeInfoMap {\n\t\tnodeNames = append(nodeNames, nodeName)\n\t}\n\n\tsort.Strings(nodeNames)\n\n\tnodeInfos := make([]schema.NodeInfo, len(nodeNames))\n\tfor i, nodeName := range nodeNames {\n\t\tnodeInfos[i] = *nodeInfoMap[nodeName]\n\t}\n\n\treturn nodeInfos, nil\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/logs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/asyncapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/realtimeapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n\t\"github.com/gorilla/websocket\"\n)\n\nfunc ReadLogs(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\tjobID := getOptionalQParam(\"jobID\", r)\n\n\tif jobID != \"\" {\n\t\tReadJobLogs(w, r)\n\t\treturn\n\t}\n\n\tdeployedResource, err := resources.GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tif deployedResource.Kind == userconfig.BatchAPIKind || deployedResource.Kind == userconfig.TaskAPIKind {\n\t\trespondError(w, r, ErrorLogsJobIDRequired(*deployedResource))\n\t\treturn\n\t} else if deployedResource.Kind != userconfig.RealtimeAPIKind && deployedResource.Kind != userconfig.AsyncAPIKind {\n\t\trespondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind))\n\t\treturn\n\t}\n\n\tdeploymentID := deployedResource.VirtualService.Labels[\"deploymentID\"]\n\tpodID := deployedResource.VirtualService.Labels[\"podID\"]\n\n\tupgrader := websocket.Upgrader{}\n\tsocket, err := upgrader.Upgrade(w, r, nil)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\tdefer socket.Close()\n\n\tlabels := map[string]string{\"apiName\": apiName, \"deploymentID\": deploymentID, \"podID\": podID}\n\n\toperator.StreamLogsFromRandomPod(labels, socket)\n}\n\nfunc GetLogURL(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\tjobID := getOptionalQParam(\"jobID\", r)\n\n\tif jobID != \"\" {\n\t\tGetJobLogURL(w, r)\n\t\treturn\n\t}\n\n\tdeployedResource, err := resources.GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tif deployedResource.Kind == userconfig.BatchAPIKind || deployedResource.Kind == userconfig.TaskAPIKind {\n\t\trespondError(w, r, ErrorLogsJobIDRequired(*deployedResource))\n\t\treturn\n\t}\n\n\tswitch deployedResource.Kind {\n\tcase userconfig.AsyncAPIKind:\n\t\tapiResponse, err := asyncapi.GetAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\t\tif apiResponse[0].Spec == nil {\n\t\t\trespondError(w, r, errors.ErrorUnexpected(\"unable to get api spec\", apiName))\n\t\t}\n\t\tlogURL, err := operator.APILogURL(*apiResponse[0].Spec)\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\t\trespondJSON(w, r, schema.LogResponse{\n\t\t\tLogURL: logURL,\n\t\t})\n\tcase userconfig.RealtimeAPIKind:\n\t\tapiResponse, err := realtimeapi.GetAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\t\tif apiResponse[0].Spec == nil {\n\t\t\trespondError(w, r, errors.ErrorUnexpected(\"unable to get api spec\", apiName))\n\t\t}\n\t\tlogURL, err := operator.APILogURL(*apiResponse[0].Spec)\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\t\trespondJSON(w, r, schema.LogResponse{\n\t\t\tLogURL: logURL,\n\t\t})\n\tdefault:\n\t\trespondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind))\n\t}\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/logs_job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n\t\"github.com/gorilla/websocket\"\n)\n\nfunc ReadJobLogs(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\tjobID, err := getRequiredQueryParam(\"jobID\", r)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tdeployedResource, err := resources.GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\tif deployedResource.Kind != userconfig.BatchAPIKind && deployedResource.Kind != userconfig.TaskAPIKind {\n\t\trespondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind, userconfig.TaskAPIKind))\n\t\treturn\n\t}\n\n\tupgrader := websocket.Upgrader{}\n\tsocket, err := upgrader.Upgrade(w, r, nil)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\tdefer socket.Close()\n\n\tlabels := map[string]string{\"apiName\": apiName, \"jobID\": jobID}\n\n\tif deployedResource.Kind == userconfig.BatchAPIKind {\n\t\tlabels[\"cortex.dev/batch\"] = \"worker\"\n\t}\n\n\toperator.StreamLogsFromRandomPod(labels, socket)\n}\n\nfunc GetJobLogURL(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\tjobID, err := getRequiredQueryParam(\"jobID\", r)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tdeployedResource, err := resources.GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tswitch deployedResource.Kind {\n\tcase userconfig.BatchAPIKind:\n\t\tjobResponse, err := batchapi.GetJob(spec.JobKey{\n\t\t\tID:      jobID,\n\t\t\tAPIName: apiName,\n\t\t\tKind:    userconfig.BatchAPIKind,\n\t\t})\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\t\tlogURL, err := operator.BatchJobLogURL(apiName, jobResponse.JobStatus)\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\t\trespondJSON(w, r, schema.LogResponse{\n\t\t\tLogURL: logURL,\n\t\t})\n\tcase userconfig.TaskAPIKind:\n\t\tjobStatus, err := taskapi.GetJobStatus(spec.JobKey{\n\t\t\tID:      jobID,\n\t\t\tAPIName: apiName,\n\t\t\tKind:    userconfig.TaskAPIKind,\n\t\t})\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\t\tlogURL, err := operator.TaskJobLogURL(apiName, *jobStatus)\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\t\trespondJSON(w, r, schema.LogResponse{\n\t\t\tLogURL: logURL,\n\t\t})\n\tdefault:\n\t\trespondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind, userconfig.TaskAPIKind))\n\t}\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/middleware.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"context\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n)\n\nvar _cachedClientIDs = strset.New()\n\ntype ctxKey int\n\nconst (\n\tctxKeyUnknown ctxKey = iota\n\tctxKeyClient\n)\n\nfunc PanicMiddleware(next http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tdefer recoverAndRespond(w, r)\n\t\tnext.ServeHTTP(w, r)\n\t})\n}\n\nfunc ClientIDMiddleware(next http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif clientID := r.URL.Query().Get(\"clientID\"); clientID != \"\" {\n\t\t\t// Add clientID to context\n\t\t\tctx := context.WithValue(r.Context(), ctxKeyClient, clientID)\n\t\t\tr = r.WithContext(ctx)\n\n\t\t\tif !_cachedClientIDs.Has(clientID) {\n\t\t\t\ttelemetry.RecordOperatorID(clientID, config.OperatorMetadata.OperatorID)\n\t\t\t\t_cachedClientIDs.Add(clientID)\n\t\t\t}\n\t\t}\n\t\tnext.ServeHTTP(w, r)\n\t})\n}\n\nfunc AWSAuthMiddleware(next http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tauthHeader := r.Header.Get(consts.AuthHeader)\n\n\t\tif authHeader == \"\" {\n\t\t\trespondError(w, r, ErrorAuthHeaderMissing(consts.AuthHeader, r.Host, r.RequestURI))\n\t\t\treturn\n\t\t}\n\n\t\taccountID, err := aws.ExecuteIdentityRequestFromHeader(authHeader)\n\t\tif err != nil {\n\t\t\trespondError(w, r, err)\n\t\t\treturn\n\t\t}\n\n\t\toperatorAccountID, _, err := config.AWS.GetCachedAccountID()\n\t\tif err != nil {\n\t\t\trespondError(w, r, ErrorAuthAPIError())\n\t\t\treturn\n\t\t}\n\n\t\tif accountID != operatorAccountID {\n\t\t\trespondErrorCode(w, r, http.StatusForbidden, ErrorAuthOtherAccount())\n\t\t\treturn\n\t\t}\n\n\t\tnext.ServeHTTP(w, r)\n\t})\n}\n\nfunc APIVersionCheckMiddleware(next http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif r.URL.Path == \"/info\" {\n\t\t\tnext.ServeHTTP(w, r)\n\t\t\treturn\n\t\t}\n\n\t\tclientVersion := r.Header.Get(\"CortexAPIVersion\")\n\t\tif clientVersion == \"\" {\n\t\t\trespondError(w, r, ErrorHeaderMissing(\"CortexAPIVersion\"))\n\t\t\treturn\n\t\t}\n\n\t\tif clientVersion != consts.CortexVersion {\n\t\t\trespondError(w, r, ErrorAPIVersionMismatch(consts.CortexVersion, clientVersion))\n\t\t\treturn\n\t\t}\n\t\tnext.ServeHTTP(w, r)\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/params.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc getRequiredPathParam(paramName string, r *http.Request) (string, error) {\n\tparam := mux.Vars(r)[paramName]\n\tif param == \"\" {\n\t\treturn \"\", ErrorPathParamRequired(paramName)\n\t}\n\treturn param, nil\n}\n\nfunc getRequiredQueryParam(paramName string, r *http.Request) (string, error) {\n\tparam := r.URL.Query().Get(paramName)\n\tif param == \"\" {\n\t\treturn \"\", ErrorQueryParamRequired(paramName)\n\t}\n\treturn param, nil\n}\n\nfunc getOptionalQParam(paramName string, r *http.Request) string {\n\treturn r.URL.Query().Get(paramName)\n}\n\nfunc getOptionalBoolQParam(paramName string, defaultVal bool, r *http.Request) bool {\n\tparam := r.URL.Query().Get(paramName)\n\tparamBool, ok := s.ParseBool(param)\n\tif ok {\n\t\treturn paramBool\n\t}\n\treturn defaultVal\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/refresh.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc Refresh(w http.ResponseWriter, r *http.Request) {\n\tapiName := mux.Vars(r)[\"apiName\"]\n\tforce := getOptionalBoolQParam(\"force\", false, r)\n\n\tmsg, err := resources.RefreshAPI(apiName, force)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tresponse := schema.RefreshResponse{\n\t\tMessage: msg,\n\t}\n\trespondJSON(w, r, response)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/respond.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"encoding/json\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nvar operatorLogger = logging.GetLogger()\n\nfunc respondJSON(w http.ResponseWriter, r *http.Request, response interface{}) {\n\tjsonBytes, err := libjson.Marshal(response)\n\tif err != nil {\n\t\trespondError(w, r, errors.Wrap(err, \"failed to encode response\"))\n\t\treturn\n\t}\n\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\tw.WriteHeader(http.StatusOK)\n\tw.Write(jsonBytes)\n}\n\nfunc respondError(w http.ResponseWriter, r *http.Request, err error, strs ...string) {\n\trespondErrorCode(w, r, http.StatusBadRequest, err, strs...)\n}\n\nfunc respondErrorCode(w http.ResponseWriter, r *http.Request, code int, err error, strs ...string) {\n\terr = errors.Wrap(err, strs...)\n\n\tif !errors.IsNoTelemetry(err) {\n\t\terrTags := map[string]string{}\n\t\tif clientID := r.Context().Value(ctxKeyClient); clientID != nil {\n\t\t\tif clientIDStr, ok := clientID.(string); ok {\n\t\t\t\terrTags[\"client_id\"] = clientIDStr\n\t\t\t}\n\t\t}\n\t\ttelemetry.Error(err, errTags)\n\t}\n\n\tif !errors.IsNoPrint(err) {\n\t\toperatorLogger.Error(err)\n\t}\n\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\tw.WriteHeader(code)\n\n\tresponse := schema.ErrorResponse{\n\t\tKind:    errors.GetKind(err),\n\t\tMessage: errors.Message(err),\n\t}\n\tjson.NewEncoder(w).Encode(response)\n}\n\nfunc recoverAndRespond(w http.ResponseWriter, r *http.Request, strs ...string) {\n\tif errInterface := recover(); errInterface != nil {\n\t\terr := errors.CastRecoverError(errInterface, strs...)\n\t\toperatorLogger.Error(err)\n\t\ttelemetry.Error(err)\n\t\trespondError(w, r, err)\n\t}\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/stop_batch_job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc StopBatchJob(w http.ResponseWriter, r *http.Request) {\n\tvars := mux.Vars(r)\n\tapiName := vars[\"apiName\"]\n\tjobID, err := getRequiredQueryParam(\"jobID\", r)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\terr = batchapi.StopJob(spec.JobKey{\n\t\tAPIName: apiName,\n\t\tID:      jobID,\n\t\tKind:    userconfig.BatchAPIKind,\n\t})\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, schema.DeleteResponse{\n\t\tMessage: fmt.Sprintf(\"stopped job %s\", jobID),\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/stop_task_job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc StopTaskJob(w http.ResponseWriter, r *http.Request) {\n\tvars := mux.Vars(r)\n\tapiName := vars[\"apiName\"]\n\tjobID, err := getRequiredQueryParam(\"jobID\", r)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\terr = taskapi.StopJob(spec.JobKey{\n\t\tAPIName: apiName,\n\t\tID:      jobID,\n\t\tKind:    userconfig.TaskAPIKind,\n\t})\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, schema.DeleteResponse{\n\t\tMessage: fmt.Sprintf(\"stopped job %s\", jobID),\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/submit_batch.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc SubmitBatchJob(w http.ResponseWriter, r *http.Request) {\n\tvars := mux.Vars(r)\n\tapiName := vars[\"apiName\"]\n\tdryRun := getOptionalBoolQParam(\"dryRun\", false, r)\n\n\tdeployedResource, err := resources.GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\tif deployedResource.Kind != userconfig.BatchAPIKind {\n\t\trespondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.BatchAPIKind))\n\t\treturn\n\t}\n\n\t// max payload size, same as API Gateway\n\trw := http.MaxBytesReader(w, r.Body, 10<<20)\n\n\tbodyBytes, err := ioutil.ReadAll(rw)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tsubmission := schema.BatchJobSubmission{}\n\n\terr = json.Unmarshal(bodyBytes, &submission)\n\tif err != nil {\n\t\trespondError(w, r, errors.Append(err, fmt.Sprintf(\"\\n\\njob submission schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor)))\n\t\treturn\n\t}\n\n\tif dryRun {\n\t\t// plain text response for dry run because it is typically consumed by people\n\t\tw.Header().Set(\"Content-type\", \"text/plain\")\n\n\t\tfileNames, err := batchapi.DryRun(&submission)\n\t\tif err != nil {\n\t\t\tw.WriteHeader(http.StatusBadRequest)\n\t\t\t_, _ = io.WriteString(w, \"\\n\"+err.Error()+\"\\n\")\n\t\t\treturn\n\t\t}\n\n\t\tfor _, fileName := range fileNames {\n\t\t\t_, err := io.WriteString(w, fileName+\"\\n\")\n\t\t\tif err != nil {\n\t\t\t\tw.WriteHeader(http.StatusBadRequest)\n\t\t\t\t_, _ = io.WriteString(w, \"\\n\"+err.Error()+\"\\n\")\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\t_, _ = io.WriteString(w, \"validations passed\")\n\t\treturn\n\t}\n\n\tjobSpec, err := batchapi.SubmitJob(apiName, &submission)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, jobSpec)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/submit_task.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/gorilla/mux\"\n)\n\nfunc SubmitTaskJob(w http.ResponseWriter, r *http.Request) {\n\tvars := mux.Vars(r)\n\tapiName := vars[\"apiName\"]\n\n\tdeployedResource, err := resources.GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\tif deployedResource.Kind != userconfig.TaskAPIKind {\n\t\trespondError(w, r, resources.ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.TaskAPIKind))\n\t\treturn\n\t}\n\n\t// max payload size, same as API Gateway\n\trw := http.MaxBytesReader(w, r.Body, 10<<20)\n\n\tbodyBytes, err := ioutil.ReadAll(rw)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\tsubmission := schema.TaskJobSubmission{\n\t\tRuntimeTaskJobConfig: spec.RuntimeTaskJobConfig{Workers: 1},\n\t}\n\n\terr = json.Unmarshal(bodyBytes, &submission)\n\tif err != nil {\n\t\trespondError(w, r, errors.Append(err,\n\t\t\tfmt.Sprintf(\"\\n\\ntask job submission schema can be found at https://docs.cortexlabs.com/v/%s/\",\n\t\t\t\tconsts.CortexVersionMinor)),\n\t\t)\n\t\treturn\n\t}\n\n\tjobSpec, err := taskapi.SubmitJob(apiName, &submission)\n\tif err != nil {\n\t\trespondError(w, r, err)\n\t\treturn\n\t}\n\n\trespondJSON(w, r, jobSpec)\n}\n"
  },
  {
    "path": "pkg/operator/endpoints/verify_cortex.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage endpoints\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nfunc VerifyCortex(w http.ResponseWriter, r *http.Request) {\n\trespondJSON(w, r, schema.VerifyCortexResponse{})\n}\n"
  },
  {
    "path": "pkg/operator/lib/exit/exit.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage exit\n\nimport (\n\t\"os\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n)\n\nvar operatorLogger = logging.GetLogger()\n\nfunc ErrorNoTelemetry(err error, wrapStrs ...string) {\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\tif err != nil && !errors.IsNoPrint(err) {\n\t\toperatorLogger.Error(err)\n\t}\n\n\ttelemetry.Close()\n\n\tos.Exit(1)\n}\n\nfunc Error(err error, wrapStrs ...string) {\n\tfor _, str := range wrapStrs {\n\t\terr = errors.Wrap(err, str)\n\t}\n\n\tif err != nil && !errors.IsNoTelemetry(err) {\n\t\ttelemetry.Error(err)\n\t}\n\n\tif err != nil && !errors.IsNoPrint(err) {\n\t\toperatorLogger.Error(err)\n\t}\n\n\ttelemetry.Close()\n\n\tos.Exit(1)\n}\n"
  },
  {
    "path": "pkg/operator/lib/routines/routines.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage routines\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n)\n\nvar operatorLogger = logging.GetLogger()\n\nfunc RunWithPanicHandler(f func()) {\n\tgo func() {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\terr := errors.CastRecoverError(r)\n\t\t\t\toperatorLogger.Error(err)\n\t\t\t\ttelemetry.Error(err)\n\t\t\t}\n\t\t}()\n\t\tf()\n\t}()\n}\n"
  },
  {
    "path": "pkg/operator/operator/cron.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage operator\n\nimport (\n\t\"context\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\tlibmath \"github.com/cortexlabs/cortex/pkg/lib/math\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promauto\"\n\tv1 \"k8s.io/api/core/v1\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\nvar operatorLogger = logging.GetLogger()\nvar previousListOfEvictedPods = strset.New()\n\nfunc DeleteEvictedPods() error {\n\tfailedPods, err := config.K8s.ListPods(&kmeta.ListOptions{\n\t\tFieldSelector: \"status.phase=Failed\",\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar errs []error\n\tcurrentEvictedPods := strset.New()\n\tfor _, pod := range failedPods {\n\t\tif pod.Status.Reason != k8s.ReasonEvicted {\n\t\t\tcontinue\n\t\t}\n\t\tif previousListOfEvictedPods.Has(pod.Name) {\n\t\t\t_, err := config.K8s.DeletePod(pod.Name)\n\t\t\tif err != nil {\n\t\t\t\terrs = append(errs, err)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tcurrentEvictedPods.Add(pod.Name)\n\t}\n\tpreviousListOfEvictedPods = currentEvictedPods\n\n\tif errors.HasError(errs) {\n\t\treturn errors.FirstError(errs...)\n\t}\n\treturn nil\n}\n\ntype instanceInfo struct {\n\tInstanceType  string  `json:\"instance_type\" yaml:\"instance_type\"`\n\tIsSpot        bool    `json:\"is_spot\" yaml:\"is_spot\"`\n\tPrice         float64 `json:\"price\" yaml:\"price\"`\n\tOnDemandPrice float64 `json:\"on_demand_price\" yaml:\"on_demand_price\"`\n\tCount         int32   `json:\"count\" yaml:\"count\"`\n\tMemory        int64   `json:\"memory\" yaml:\"memory\"`\n\tCPU           float64 `json:\"cpu\" yaml:\"cpu\"`\n\tGPU           int64   `json:\"gpu\" yaml:\"gpu\"`\n\tInf           int64   `json:\"inf\" yaml:\"inf\"`\n}\n\nfunc ClusterTelemetry() error {\n\tproperties, err := clusterTelemetryProperties()\n\tif err != nil {\n\t\treturn err\n\t}\n\ttelemetry.Event(\"operator.cron\", properties,\n\t\tconfig.ClusterConfig.CoreConfig.TelemetryEvent(),\n\t)\n\n\treturn nil\n}\n\nfunc clusterTelemetryProperties() (map[string]interface{}, error) {\n\tctx := context.Background()\n\tvar nodeList v1.NodeList\n\n\terr := config.K8s.List(ctx, &nodeList)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnodes := nodeList.Items\n\n\tinstanceInfos := make(map[string]*instanceInfo)\n\n\tvar numOperatorInstances int\n\tvar totalInstances int\n\tvar totalInstancePrice float64\n\tvar totalInstancePriceIfOnDemand float64\n\n\tspotPriceCache := make(map[string]float64) // instance type -> spot price\n\n\tfor _, node := range nodes {\n\t\tif node.Labels[\"workload\"] != \"true\" {\n\t\t\tif node.Labels[\"alpha.eksctl.io/nodegroup-name\"] == \"cx-operator\" {\n\t\t\t\tnumOperatorInstances++\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tinstanceType := node.Labels[\"node.kubernetes.io/instance-type\"]\n\t\tif instanceType == \"\" {\n\t\t\tinstanceType = \"unknown\"\n\t\t}\n\n\t\tisSpot := false\n\t\tif node.Labels[\"node-lifecycle\"] == \"spot\" {\n\t\t\tisSpot = true\n\t\t}\n\n\t\ttotalInstances++\n\n\t\tinstanceInfosKey := instanceType + \"_ondemand\"\n\t\tif isSpot {\n\t\t\tinstanceInfosKey = instanceType + \"_spot\"\n\t\t}\n\n\t\tif info, ok := instanceInfos[instanceInfosKey]; ok {\n\t\t\tinfo.Count++\n\t\t\tcontinue\n\t\t}\n\n\t\tonDemandPrice := aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price\n\t\tprice := onDemandPrice\n\t\tif isSpot {\n\t\t\tif spotPrice, ok := spotPriceCache[instanceType]; ok {\n\t\t\t\tprice = spotPrice\n\t\t\t} else {\n\t\t\t\tspotPrice, err := config.AWS.SpotInstancePrice(instanceType)\n\t\t\t\tif err == nil && spotPrice != 0 {\n\t\t\t\t\tprice = spotPrice\n\t\t\t\t\tspotPriceCache[instanceType] = spotPrice\n\t\t\t\t} else {\n\t\t\t\t\tspotPriceCache[instanceType] = price // the request failed, so no need to try again in the future\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tngName := node.Labels[\"alpha.eksctl.io/nodegroup-name\"]\n\t\tebsPricePerVolume := getEBSPriceForNodeGroupInstance(config.ClusterConfig.NodeGroups, ngName)\n\t\tonDemandPrice += ebsPricePerVolume\n\t\tprice += ebsPricePerVolume\n\n\t\tgpuQty := node.Status.Capacity[\"nvidia.com/gpu\"]\n\t\tinfQty := node.Status.Capacity[\"aws.amazon.com/neuron\"]\n\n\t\tinfo := instanceInfo{\n\t\t\tInstanceType:  instanceType,\n\t\t\tIsSpot:        isSpot,\n\t\t\tPrice:         price,\n\t\t\tOnDemandPrice: onDemandPrice,\n\t\t\tCount:         1,\n\t\t\tMemory:        node.Status.Capacity.Memory().Value(),\n\t\t\tCPU:           float64(node.Status.Capacity.Cpu().MilliValue()) / 1000,\n\t\t\tGPU:           gpuQty.Value(),\n\t\t\tInf:           infQty.Value(),\n\t\t}\n\n\t\tinstanceInfos[instanceInfosKey] = &info\n\t\ttotalInstancePrice += info.Price\n\t\ttotalInstancePriceIfOnDemand += info.OnDemandPrice\n\t}\n\n\tfixedPrice := cortexSystemPrice(numOperatorInstances, 1)\n\n\treturn map[string]interface{}{\n\t\t\"region\":                      config.ClusterConfig.Region,\n\t\t\"instance_count\":              totalInstances,\n\t\t\"instances\":                   instanceInfos,\n\t\t\"fixed_price\":                 fixedPrice,\n\t\t\"workload_price\":              totalInstancePrice,\n\t\t\"workload_price_if_on_demand\": totalInstancePriceIfOnDemand,\n\t\t\"total_price\":                 totalInstancePrice + fixedPrice,\n\t\t\"total_price_if_on_demand\":    totalInstancePriceIfOnDemand + fixedPrice,\n\t}, nil\n}\n\nfunc getEBSPriceForNodeGroupInstance(ngs []*clusterconfig.NodeGroup, ngName string) float64 {\n\tvar ebsPrice float64\n\tfor _, ng := range ngs {\n\t\tvar ngNamePrefix string\n\t\tif ng.Spot {\n\t\t\tngNamePrefix = \"cx-ws-\"\n\t\t} else {\n\t\t\tngNamePrefix = \"cx-wd-\"\n\t\t}\n\t\tif ng.Name == ngNamePrefix+ngName {\n\t\t\tebsPrice = aws.EBSMetadatas[config.ClusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24\n\t\t\tif ng.InstanceVolumeType == clusterconfig.IO1VolumeType && ng.InstanceVolumeIOPS != nil {\n\t\t\t\tebsPrice += aws.EBSMetadatas[config.ClusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24\n\t\t\t}\n\t\t\tif ng.InstanceVolumeType == clusterconfig.GP3VolumeType && ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput != nil {\n\t\t\t\tebsPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[config.ClusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS-3000)*float64(*ng.InstanceVolumeIOPS)/30/24)\n\t\t\t\tebsPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[config.ClusterConfig.Region][ng.InstanceVolumeType.String()].PriceThroughput-125)*float64(*ng.InstanceVolumeThroughput)/30/24)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif ebsPrice == 0 && (ngName == \"cx-operator\" || ngName == \"cx-prometheus\") {\n\t\treturn aws.EBSMetadatas[config.ClusterConfig.Region][\"gp3\"].PriceGB * 20 / 30 / 24\n\t}\n\n\treturn ebsPrice\n}\n\nfunc cortexSystemPrice(numOperatorInstances, numPrometheusInstances int) float64 {\n\teksPrice := aws.EKSPrices[config.ClusterConfig.Region]\n\tmetricsEBSPrice := aws.EBSMetadatas[config.ClusterConfig.Region][\"gp2\"].PriceGB * (40 + 2) / 30 / 24\n\tnlbPrice := aws.NLBMetadatas[config.ClusterConfig.Region].Price\n\tnatUnitPrice := aws.NATMetadatas[config.ClusterConfig.Region].Price\n\tvar natTotalPrice float64\n\n\tif config.ClusterConfig.NATGateway == clusterconfig.SingleNATGateway {\n\t\tnatTotalPrice = natUnitPrice\n\t} else if config.ClusterConfig.NATGateway == clusterconfig.HighlyAvailableNATGateway {\n\t\tnatTotalPrice = natUnitPrice * float64(len(config.ClusterConfig.AvailabilityZones))\n\t}\n\n\toperatorInstancePrice := aws.InstanceMetadatas[config.ClusterConfig.Region][\"t3.medium\"].Price\n\toperatorEBSPrice := aws.EBSMetadatas[config.ClusterConfig.Region][\"gp3\"].PriceGB * 20 / 30 / 24\n\n\tprometheusInstancePrice := aws.InstanceMetadatas[config.ClusterConfig.Region][config.ClusterConfig.PrometheusInstanceType].Price\n\tprometheusEBSPrice := aws.EBSMetadatas[config.ClusterConfig.Region][\"gp3\"].PriceGB * 20 / 30 / 24\n\n\tfixedCosts := eksPrice + metricsEBSPrice + 2*nlbPrice + natTotalPrice +\n\t\tfloat64(numOperatorInstances)*(operatorInstancePrice+operatorEBSPrice) +\n\t\tfloat64(numPrometheusInstances)*(prometheusInstancePrice+prometheusEBSPrice)\n\n\treturn fixedCosts\n}\n\nvar clusterGauge = promauto.NewGaugeVec(\n\tprometheus.GaugeOpts{\n\t\tName: \"cortex_cluster_cost\",\n\t\tHelp: \"The cost breakdown of the cortex cluster\",\n\t}, []string{\"api\", \"kind\", \"component\"},\n)\n\nfunc CostBreakdown() error {\n\tctx := context.Background()\n\n\tvar nodeList v1.NodeList\n\terr := config.K8s.List(ctx, &nodeList)\n\tif err != nil {\n\t\treturn err\n\t}\n\tnodes := nodeList.Items\n\n\tvar podList v1.PodList\n\terr = config.K8s.List(ctx, &podList,\n\t\tclient.InNamespace(consts.DefaultNamespace),\n\t\tclient.HasLabels{\"apiName\", \"apiKind\"},\n\t)\n\tif err != nil {\n\t\treturn err\n\t}\n\tpods := podList.Items\n\n\tspotPriceCache := make(map[string]float64) // instance type -> spot price\n\n\t// Total cluster costs = cortex system + cortex workloads\n\tvar totalClusterCosts float64 = cortexSystemPrice(0, 0)\n\t// Total cortex system costs = operator + prometheus node groups + workload daemonsets\n\tvar totalCortexSystemCosts float64 = cortexSystemPrice(0, 0)\n\t// Total workload compute costs by api name\n\ttotalWorkloadComputeCostsByAPIName := make(map[string]float64, 0)\n\t// Total workload compute costs by kind\n\ttotalWorkloadComputeCostsByAPIKind := make(map[string]float64, 0)\n\n\tfor _, node := range nodes {\n\t\tinstanceType := node.Labels[\"node.kubernetes.io/instance-type\"]\n\t\tif instanceType == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tworkloadNode := false\n\t\tif node.Labels[\"workload\"] == \"true\" {\n\t\t\tworkloadNode = true\n\t\t}\n\n\t\tisSpot := false\n\t\tif node.Labels[\"node-lifecycle\"] == \"spot\" {\n\t\t\tisSpot = true\n\t\t}\n\n\t\tvar instanceComputePrice float64\n\t\tif isSpot {\n\t\t\tif spotPrice, ok := spotPriceCache[instanceType]; ok {\n\t\t\t\tinstanceComputePrice = spotPrice\n\t\t\t} else {\n\t\t\t\tspotPrice, err := config.AWS.SpotInstancePrice(instanceType)\n\t\t\t\tif err == nil && spotPrice != 0 {\n\t\t\t\t\tinstanceComputePrice = spotPrice\n\t\t\t\t\tspotPriceCache[instanceType] = spotPrice\n\t\t\t\t} else {\n\t\t\t\t\tinstanceComputePrice = aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price // the request failed, so no need to try again in the future\n\t\t\t\t\tspotPriceCache[instanceType] = instanceComputePrice\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tinstanceComputePrice = aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType].Price\n\t\t}\n\n\t\tngName := node.Labels[\"alpha.eksctl.io/nodegroup-name\"]\n\t\tinstanceEBSPrice := getEBSPriceForNodeGroupInstance(config.ClusterConfig.NodeGroups, ngName)\n\n\t\tinstancePrice := instanceComputePrice + instanceEBSPrice\n\n\t\ttotalClusterCosts += instancePrice\n\t\tif !workloadNode {\n\t\t\ttotalCortexSystemCosts += instancePrice\n\t\t\tcontinue\n\t\t}\n\n\t\tfor _, pod := range pods {\n\t\t\tif pod.Spec.NodeName != node.Name {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmaxPods := k8s.HowManyPodsFitOnNode(pod.Spec, node, consts.CortexCPUPodReserved, consts.CortexMemPodReserved)\n\t\t\tcostPerPod := instancePrice / float64(maxPods)\n\n\t\t\tif apiName, ok := pod.Labels[\"apiName\"]; ok {\n\t\t\t\tif _, okMap := totalWorkloadComputeCostsByAPIName[apiName]; okMap {\n\t\t\t\t\ttotalWorkloadComputeCostsByAPIName[apiName] += costPerPod\n\t\t\t\t} else {\n\t\t\t\t\ttotalWorkloadComputeCostsByAPIName[apiName] = costPerPod\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif apiKind, ok := pod.Labels[\"apiKind\"]; ok {\n\t\t\t\tif _, okMap := totalWorkloadComputeCostsByAPIName[apiKind]; okMap {\n\t\t\t\t\ttotalWorkloadComputeCostsByAPIKind[apiKind] += costPerPod\n\t\t\t\t} else {\n\t\t\t\t\ttotalWorkloadComputeCostsByAPIKind[apiKind] = costPerPod\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\ttotalWorkloadComputeCosts := totalClusterCosts - totalCortexSystemCosts\n\tvar totalWorkloadComputeCostsFromAPIName float64\n\tvar totalWorkloadComputeCostsFromAPIKind float64\n\tapiNameRenormalizationRatio := float64(1)\n\tapiKindRenormalizationRatio := float64(1)\n\n\tfor apiName := range totalWorkloadComputeCostsByAPIName {\n\t\ttotalWorkloadComputeCostsFromAPIName += totalWorkloadComputeCostsByAPIName[apiName]\n\t}\n\tif totalWorkloadComputeCostsFromAPIName > 0 {\n\t\tapiNameRenormalizationRatio = totalWorkloadComputeCosts / totalWorkloadComputeCostsFromAPIName\n\t}\n\tfor apiKind := range totalWorkloadComputeCostsByAPIKind {\n\t\ttotalWorkloadComputeCostsFromAPIKind += totalWorkloadComputeCostsByAPIKind[apiKind]\n\t}\n\tif totalWorkloadComputeCostsFromAPIKind > 0 {\n\t\tapiKindRenormalizationRatio = totalWorkloadComputeCosts / totalWorkloadComputeCostsFromAPIKind\n\t}\n\n\tclusterGauge.Reset()\n\tclusterGauge.WithLabelValues(\"false\", \"false\", \"cluster-costs\").Set(totalClusterCosts)\n\tclusterGauge.WithLabelValues(\"false\", \"false\", \"cortex-system-costs\").Set(totalCortexSystemCosts)\n\tfor apiName := range totalWorkloadComputeCostsByAPIName {\n\t\tclusterGauge.WithLabelValues(\"true\", \"false\", apiName).Set(totalWorkloadComputeCostsByAPIName[apiName] * apiNameRenormalizationRatio)\n\t}\n\tfor apiKind := range totalWorkloadComputeCostsByAPIKind {\n\t\tclusterGauge.WithLabelValues(\"false\", \"true\", apiKind).Set(totalWorkloadComputeCostsByAPIKind[apiKind] * apiKindRenormalizationRatio)\n\t}\n\n\treturn nil\n}\n\nfunc ErrorHandler(cronName string) func(error) {\n\treturn func(err error) {\n\t\terr = errors.Wrap(err, cronName+\" cron failed\")\n\t\ttelemetry.Error(err)\n\t\toperatorLogger.Error(err)\n\t}\n}\n"
  },
  {
    "path": "pkg/operator/operator/deployed_resource.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage operator\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n)\n\ntype DeployedResource struct {\n\tuserconfig.Resource\n\tVirtualService *istioclientnetworking.VirtualService\n}\n\nfunc (deployedResourced *DeployedResource) ID() string {\n\treturn deployedResourced.VirtualService.Labels[\"apiID\"]\n}\n"
  },
  {
    "path": "pkg/operator/operator/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage operator\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrCortexInstallationBroken = \"operator.cortex_installation_broken\"\n\tErrLoadBalancerInitializing = \"operator.load_balancer_initializing\"\n\tErrInvalidOperatorLogLevel  = \"operator.invalid_operator_log_level\"\n)\n\nfunc ErrorCortexInstallationBroken() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCortexInstallationBroken,\n\t\tMessage: \"cortex is out of date or not installed properly; spin down your cluster with `cortex cluster down` and create a new one with `cortex cluster up`\",\n\t})\n}\n\nfunc ErrorLoadBalancerInitializing() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrLoadBalancerInitializing,\n\t\tMessage: \"load balancer is still initializing\",\n\t})\n}\n\nfunc ErrorInvalidOperatorLogLevel(provided string, loglevels []string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrLoadBalancerInitializing,\n\t\tMessage: fmt.Sprintf(\"invalid operator log level %s; must be one of %s\", provided, s.StrsOr(loglevels)),\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/operator/k8s.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage operator\n\nimport (\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\n// APILoadBalancerURL returns the http endpoint of the ingress load balancer for deployed APIs\nfunc APILoadBalancerURL() (string, error) {\n\treturn getLoadBalancerURL(\"ingressgateway-apis\")\n}\n\n// LoadBalancerURL returns the http endpoint of the ingress load balancer for the operator\nfunc LoadBalancerURL() (string, error) {\n\treturn getLoadBalancerURL(\"ingressgateway-operator\")\n}\n\nfunc getLoadBalancerURL(name string) (string, error) {\n\tservice, err := config.K8sIstio.GetService(name)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif service == nil {\n\t\treturn \"\", ErrorCortexInstallationBroken()\n\t}\n\tif len(service.Status.LoadBalancer.Ingress) == 0 {\n\t\treturn \"\", ErrorLoadBalancerInitializing()\n\t}\n\tif service.Status.LoadBalancer.Ingress[0].Hostname != \"\" {\n\t\treturn \"http://\" + service.Status.LoadBalancer.Ingress[0].Hostname, nil\n\t}\n\treturn \"http://\" + service.Status.LoadBalancer.Ingress[0].IP, nil\n}\n\nfunc APIEndpoint(api *spec.API) (string, error) {\n\tvar err error\n\tbaseAPIEndpoint := \"\"\n\n\tbaseAPIEndpoint, err = APILoadBalancerURL()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tbaseAPIEndpoint = strings.Replace(baseAPIEndpoint, \"https://\", \"http://\", 1)\n\n\treturn urls.Join(baseAPIEndpoint, *api.Networking.Endpoint), nil\n}\n\nfunc APIEndpointFromResource(deployedResource *DeployedResource) (string, error) {\n\tapiEndpoint, err := userconfig.EndpointFromAnnotation(deployedResource.VirtualService)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tbaseAPIEndpoint := \"\"\n\n\tbaseAPIEndpoint, err = APILoadBalancerURL()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tbaseAPIEndpoint = strings.Replace(baseAPIEndpoint, \"https://\", \"http://\", 1)\n\n\treturn urls.Join(baseAPIEndpoint, apiEndpoint), nil\n}\n"
  },
  {
    "path": "pkg/operator/operator/logging.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage operator\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"go.uber.org/zap\"\n)\n\nconst (\n\t_loggerTTL          = time.Hour * 1\n\t_evictionCronPeriod = time.Minute * 10\n)\n\ntype cachedLogger struct {\n\tvalue      *zap.SugaredLogger\n\tlastAccess time.Time\n}\n\ntype loggerCache struct {\n\tm map[string]*cachedLogger\n\tsync.Mutex\n}\n\nvar _loggerCache loggerCache\n\nfunc init() {\n\t_loggerCache = loggerCache{m: map[string]*cachedLogger{}}\n\n\tgo func() {\n\t\tfor range time.Tick(_evictionCronPeriod) {\n\t\t\t_loggerCache.Lock()\n\t\t\tfor k, v := range _loggerCache.m {\n\t\t\t\tif time.Since(v.lastAccess) > _loggerTTL {\n\t\t\t\t\tdelete(_loggerCache.m, k)\n\t\t\t\t}\n\t\t\t}\n\t\t\t_loggerCache.Unlock()\n\t\t}\n\t}()\n}\n\nfunc getFromCacheOrNil(key string) *zap.SugaredLogger {\n\t_loggerCache.Lock()\n\tdefer _loggerCache.Unlock()\n\n\titem, ok := _loggerCache.m[key]\n\tif ok {\n\t\titem.lastAccess = time.Now()\n\t\treturn item.value\n\t}\n\treturn nil\n}\n\nfunc initializeLogger(key string, level userconfig.LogLevel, fields map[string]interface{}) (*zap.SugaredLogger, error) {\n\tloggerConfig := logging.DefaultZapConfig(level, fields)\n\n\tdisableJSONLogging := strings.ToLower(os.Getenv(\"CORTEX_DISABLE_JSON_LOGGING\"))\n\tif disableJSONLogging == \"true\" {\n\t\tloggerConfig.Encoding = \"console\"\n\t}\n\n\tlogger, err := loggerConfig.Build()\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tsugarLogger := logger.Sugar()\n\n\t_loggerCache.Lock()\n\tdefer _loggerCache.Unlock()\n\n\t_loggerCache.m[key] = &cachedLogger{\n\t\tlastAccess: time.Now(),\n\t\tvalue:      sugarLogger,\n\t}\n\n\treturn sugarLogger, nil\n}\n\nfunc GetRealtimeAPILogger(apiName string, apiID string) (*zap.SugaredLogger, error) {\n\tloggerCacheKey := fmt.Sprintf(\"apiName=%s,apiID=%s\", apiName, apiID)\n\tlogger := getFromCacheOrNil(loggerCacheKey)\n\n\tif logger != nil {\n\t\treturn logger, nil\n\t}\n\n\tapiSpec, err := DownloadAPISpec(apiName, apiID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{\n\t\t\"apiName\": apiSpec.Name,\n\t\t\"apiKind\": apiSpec.Kind.String(),\n\t\t\"apiID\":   apiSpec.ID,\n\t})\n}\n\nfunc GetRealtimeAPILoggerFromSpec(apiSpec *spec.API) (*zap.SugaredLogger, error) {\n\tloggerCacheKey := fmt.Sprintf(\"apiName=%s,apiID=%s\", apiSpec.Name, apiSpec.ID)\n\tlogger := getFromCacheOrNil(loggerCacheKey)\n\tif logger != nil {\n\t\treturn logger, nil\n\t}\n\n\treturn initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{\n\t\t\"apiName\": apiSpec.Name,\n\t\t\"apiKind\": apiSpec.Kind.String(),\n\t\t\"apiID\":   apiSpec.ID,\n\t})\n}\n\nfunc GetJobLogger(jobKey spec.JobKey) (*zap.SugaredLogger, error) {\n\tloggerCacheKey := fmt.Sprintf(\"apiName=%s,jobID=%s\", jobKey.APIName, jobKey.ID)\n\tlogger := getFromCacheOrNil(loggerCacheKey)\n\tif logger != nil {\n\t\treturn logger, nil\n\t}\n\n\treturn initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{\n\t\t\"apiName\": jobKey.APIName,\n\t\t\"apiKind\": jobKey.Kind.String(),\n\t\t\"jobID\":   jobKey.ID,\n\t})\n}\n\nfunc GetJobLoggerFromSpec(apiSpec *spec.API, jobKey spec.JobKey) (*zap.SugaredLogger, error) {\n\tloggerCacheKey := fmt.Sprintf(\"apiName=%s,jobID=%s\", jobKey.APIName, jobKey.ID)\n\tlogger := getFromCacheOrNil(loggerCacheKey)\n\tif logger != nil {\n\t\treturn logger, nil\n\t}\n\n\treturn initializeLogger(loggerCacheKey, userconfig.InfoLogLevel, map[string]interface{}{\n\t\t\"apiName\": jobKey.APIName,\n\t\t\"apiKind\": jobKey.Kind.String(),\n\t\t\"jobID\":   jobKey.ID,\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/operator/memory_capacity.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage operator\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nconst _memConfigMapName = \"cortex-instance-memory\"\nconst _configKeyPrefix = \"memory-capacity-\"\n\nfunc getMemoryCapacityFromNodes(primaryInstances []string) (map[string]*kresource.Quantity, error) {\n\topts := kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(map[string]string{\n\t\t\t\"workload\": \"true\",\n\t\t}).String(),\n\t}\n\tnodes, err := config.K8s.ListNodes(&opts)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tminMemMap := map[string]*kresource.Quantity{}\n\tfor _, primaryInstance := range primaryInstances {\n\t\tminMemMap[primaryInstance] = nil\n\t}\n\n\tfor _, node := range nodes {\n\t\tisPrimaryInstance := false\n\t\tvar primaryInstanceType string\n\t\tfor k, v := range node.Labels {\n\t\t\tif k == \"node.kubernetes.io/instance-type\" && slices.HasString(primaryInstances, v) {\n\t\t\t\tisPrimaryInstance = true\n\t\t\t\tprimaryInstanceType = v\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !isPrimaryInstance {\n\t\t\tcontinue\n\t\t}\n\n\t\tcurMem := node.Status.Capacity.Memory()\n\n\t\tif curMem == nil || curMem.IsZero() {\n\t\t\tcontinue\n\t\t}\n\n\t\tif minMemMap[primaryInstanceType] == nil || minMemMap[primaryInstanceType].Cmp(*curMem) > 0 {\n\t\t\tminMemMap[primaryInstanceType] = curMem\n\t\t}\n\t}\n\n\treturn minMemMap, nil\n}\n\nfunc getMemoryCapacityFromConfigMap() (map[string]*kresource.Quantity, error) {\n\tconfigMapData, _, err := config.K8s.GetConfigMapData(_memConfigMapName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(configMapData) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tmemoryCapacitiesMap := map[string]*kresource.Quantity{}\n\tfor k := range configMapData {\n\t\tmemoryUserStr := configMapData[k]\n\t\tmem, err := kresource.ParseQuantity(memoryUserStr)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tinstanceType := k[len(_configKeyPrefix):]\n\t\tif mem.IsZero() {\n\t\t\tmemoryCapacitiesMap[instanceType] = nil\n\t\t} else {\n\t\t\tmemoryCapacitiesMap[instanceType] = &mem\n\t\t}\n\t}\n\n\treturn memoryCapacitiesMap, nil\n}\n\nfunc UpdateMemoryCapacityConfigMap() (map[string]kresource.Quantity, error) {\n\tprimaryInstances := []string{}\n\n\tminMemMap := map[string]kresource.Quantity{}\n\tfor _, ng := range config.ClusterConfig.NodeGroups {\n\t\tinstanceMetadata := aws.InstanceMetadatas[config.ClusterConfig.Region][ng.InstanceType]\n\t\tminMemMap[ng.InstanceType] = instanceMetadata.Memory\n\t\tprimaryInstances = append(primaryInstances, ng.InstanceType)\n\t}\n\n\tnodeMemCapacityMap, err := getMemoryCapacityFromNodes(primaryInstances)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tpreviousMinMemMap, err := getMemoryCapacityFromConfigMap()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconfigMapData := map[string]string{}\n\tfor _, primaryInstance := range primaryInstances {\n\t\tminMem := minMemMap[primaryInstance]\n\n\t\tif nodeMemCapacityMap[primaryInstance] != nil && minMem.Cmp(*nodeMemCapacityMap[primaryInstance]) > 0 {\n\t\t\tminMem = *nodeMemCapacityMap[primaryInstance]\n\t\t}\n\n\t\tif previousMinMemMap[primaryInstance] != nil && minMem.Cmp(*previousMinMemMap[primaryInstance]) > 0 {\n\t\t\tminMem = *previousMinMemMap[primaryInstance]\n\t\t}\n\n\t\tif previousMinMemMap[primaryInstance] == nil || minMem.Cmp(*previousMinMemMap[primaryInstance]) < 0 {\n\t\t\tconfigMapData[_configKeyPrefix+primaryInstance] = minMem.String()\n\t\t} else {\n\t\t\tconfigMapData[_configKeyPrefix+primaryInstance] = previousMinMemMap[primaryInstance].String()\n\t\t}\n\n\t\tminMemMap[primaryInstance] = minMem\n\t}\n\n\tconfigMap := k8s.ConfigMap(&k8s.ConfigMapSpec{\n\t\tName: _memConfigMapName,\n\t\tData: configMapData,\n\t})\n\n\t_, err = config.K8s.ApplyConfigMap(configMap)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn minMemMap, nil\n}\n"
  },
  {
    "path": "pkg/operator/operator/storage.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage operator\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n)\n\nfunc DownloadAPISpec(apiName string, apiID string) (*spec.API, error) {\n\tbucketKey := spec.Key(apiName, apiID, config.ClusterConfig.ClusterUID)\n\tvar api spec.API\n\tif err := config.AWS.ReadJSONFromS3(&api, config.ClusterConfig.Bucket, bucketKey); err != nil {\n\t\treturn nil, err\n\t}\n\treturn &api, nil\n}\n\nfunc DownloadAPISpecs(apiNames []string, apiIDs []string) ([]spec.API, error) {\n\tapis := make([]spec.API, len(apiNames))\n\tfns := make([]func() error, len(apiNames))\n\n\tfor i := range apiNames {\n\t\tlocalIdx := i\n\t\tfns[i] = func() error {\n\t\t\tapi, err := DownloadAPISpec(apiNames[localIdx], apiIDs[localIdx])\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tapis[localIdx] = *api\n\t\t\treturn nil\n\t\t}\n\t}\n\n\tif len(fns) > 0 {\n\t\terr := parallel.RunFirstErr(fns[0], fns[1:]...)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn apis, nil\n}\n\nfunc DownloadBatchJobSpec(jobKey spec.JobKey) (*spec.BatchJob, error) {\n\tjobSpec := spec.BatchJob{}\n\tif err := config.AWS.ReadJSONFromS3(&jobSpec, config.ClusterConfig.Bucket, jobKey.SpecFilePath(config.ClusterConfig.ClusterUID)); err != nil {\n\t\treturn nil, errors.Wrap(err, \"unable to download job specification\", jobKey.UserString())\n\t}\n\treturn &jobSpec, nil\n}\n\nfunc DownloadTaskJobSpec(jobKey spec.JobKey) (*spec.TaskJob, error) {\n\tjobSpec := spec.TaskJob{}\n\tif err := config.AWS.ReadJSONFromS3(&jobSpec, config.ClusterConfig.Bucket, jobKey.SpecFilePath(config.ClusterConfig.ClusterUID)); err != nil {\n\t\treturn nil, errors.Wrap(err, \"unable to download job specification\", jobKey.UserString())\n\t}\n\treturn &jobSpec, nil\n}\n"
  },
  {
    "path": "pkg/operator/operator/workload_logging.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage operator\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"text/template\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/gorilla/websocket\"\n)\n\nconst (\n\t_socketWriteDeadlineWait = 10 * time.Second\n\t_socketCloseGracePeriod  = 10 * time.Second\n\t_socketMaxMessageSize    = 8192\n\t_readBufferSize          = 4096\n\n\t_pendingPodCheckInterval = 1 * time.Second\n\t_pollPeriod              = 250 * time.Millisecond\n)\n\nfunc timeString(t time.Time) string {\n\treturn fmt.Sprintf(\"%sT%02d*3a%02d*3a%02d\", t.Format(\"2006-01-02\"), t.Hour(), t.Minute(), t.Second())\n}\n\nvar _apiLogURLTemplate *template.Template = template.Must(template.New(\"api_log_url_template\").Parse(strings.TrimSpace(`\nhttps://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-3600$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529\n`)))\n\nvar _completedJobLogURLTemplate *template.Template = template.Must(template.New(\"completed_job_log_url_template\").Parse(strings.TrimSpace(`\nhttps://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E$2527{{.EndTime}}$257Estart$257E$2527{{.StartTime}}$257EtimeType$257E$2527ABSOLUTE$257Etz$257E$2527Local$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*20and*20cortex.labels.jobID*3d*22{{.JobID}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529\n`)))\n\nvar _inProgressJobLogsURLTemplate *template.Template = template.Must(template.New(\"in_progress_job_log_url_template\").Parse(strings.TrimSpace(`\nhttps://console.{{.Partition}}.com/cloudwatch/home?region={{.Region}}#logsV2:logs-insights$3FqueryDetail$3D$257E$2528end$257E0$257Estart$257E-3600$257EtimeType$257E$2527RELATIVE$257Eunit$257E$2527seconds$257EeditorString$257E$2527fields*20*40timestamp*2c*20message*0a*7c*20filter*20cortex.labels.apiName*3d*22{{.APIName}}*22*20and*20cortex.labels.jobID*3d*22{{.JobID}}*22*0a*7c*20sort*20*40timestamp*20asc*0a$257Esource$257E$2528$257E$2527{{.LogGroup}}$2529$2529\n`)))\n\ntype apiLogURLTemplateArgs struct {\n\tPartition string\n\tRegion    string\n\tLogGroup  string\n\tAPIName   string\n}\n\ntype completedJobLogURLTemplateArgs struct {\n\tPartition string\n\tRegion    string\n\tStartTime string\n\tEndTime   string\n\tLogGroup  string\n\tAPIName   string\n\tJobID     string\n}\n\ntype inProgressJobLogURLTemplateArgs struct {\n\tPartition string\n\tRegion    string\n\tLogGroup  string\n\tAPIName   string\n\tJobID     string\n}\n\nfunc completedBatchJobLogsURL(args completedJobLogURLTemplateArgs) (string, error) {\n\tbuf := &bytes.Buffer{}\n\terr := _completedJobLogURLTemplate.Execute(buf, args)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn strings.TrimSpace(buf.String()), nil\n}\n\nfunc inProgressBatchJobLogsURL(args inProgressJobLogURLTemplateArgs) (string, error) {\n\tbuf := &bytes.Buffer{}\n\terr := _inProgressJobLogsURLTemplate.Execute(buf, args)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn strings.TrimSpace(buf.String()), nil\n}\n\nfunc APILogURL(api spec.API) (string, error) {\n\tpartition := \"aws.amazon\"\n\tregion := config.ClusterConfig.Region\n\tif awslib.PartitionFromRegion(region) == \"aws-us-gov\" {\n\t\tpartition = \"amazonaws-us-gov\"\n\t}\n\tlogGroup := config.ClusterConfig.ClusterName\n\n\targs := apiLogURLTemplateArgs{\n\t\tPartition: partition,\n\t\tRegion:    region,\n\t\tLogGroup:  logGroup,\n\t\tAPIName:   api.Name,\n\t}\n\n\tbuf := &bytes.Buffer{}\n\terr := _apiLogURLTemplate.Execute(buf, args)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn strings.TrimSpace(buf.String()), nil\n}\n\nfunc BatchJobLogURL(apiName string, jobStatus status.BatchJobStatus) (string, error) {\n\tpartition := \"aws.amazon\"\n\tregion := config.ClusterConfig.Region\n\tif awslib.PartitionFromRegion(region) == \"aws-us-gov\" {\n\t\tpartition = \"amazonaws-us-gov\"\n\t}\n\tlogGroup := config.ClusterConfig.ClusterName\n\n\tif jobStatus.EndTime != nil {\n\t\tendTime := *jobStatus.EndTime\n\t\tendTime = endTime.Add(60 * time.Second)\n\t\treturn completedBatchJobLogsURL(completedJobLogURLTemplateArgs{\n\t\t\tPartition: partition,\n\t\t\tRegion:    region,\n\t\t\tStartTime: timeString(jobStatus.StartTime),\n\t\t\tEndTime:   timeString(endTime),\n\t\t\tLogGroup:  logGroup,\n\t\t\tAPIName:   apiName,\n\t\t\tJobID:     jobStatus.ID,\n\t\t})\n\t}\n\treturn inProgressBatchJobLogsURL(inProgressJobLogURLTemplateArgs{\n\t\tPartition: partition,\n\t\tRegion:    region,\n\t\tLogGroup:  logGroup,\n\t\tAPIName:   apiName,\n\t\tJobID:     jobStatus.ID,\n\t})\n}\n\nfunc TaskJobLogURL(apiName string, jobStatus status.TaskJobStatus) (string, error) {\n\tpartition := \"aws.amazon\"\n\tregion := config.ClusterConfig.Region\n\tif awslib.PartitionFromRegion(region) == \"aws-us-gov\" {\n\t\tpartition = \"amazonaws-us-gov\"\n\t}\n\tlogGroup := config.ClusterConfig.ClusterName\n\tif jobStatus.EndTime != nil {\n\t\tendTime := *jobStatus.EndTime\n\t\tendTime = endTime.Add(60 * time.Second)\n\t\treturn completedBatchJobLogsURL(completedJobLogURLTemplateArgs{\n\t\t\tPartition: partition,\n\t\t\tRegion:    region,\n\t\t\tStartTime: timeString(jobStatus.StartTime),\n\t\t\tEndTime:   timeString(endTime),\n\t\t\tLogGroup:  logGroup,\n\t\t\tAPIName:   apiName,\n\t\t\tJobID:     jobStatus.ID,\n\t\t})\n\t}\n\treturn inProgressBatchJobLogsURL(inProgressJobLogURLTemplateArgs{\n\t\tPartition: partition,\n\t\tRegion:    region,\n\t\tLogGroup:  logGroup,\n\t\tAPIName:   apiName,\n\t\tJobID:     jobStatus.ID,\n\t})\n}\n\nfunc waitForPodToBeNotPending(podName string, cancelListener chan struct{}, socket *websocket.Conn) bool {\n\twrotePending := false\n\ttimer := time.NewTimer(0)\n\n\tfor true {\n\t\tselect {\n\t\tcase <-cancelListener:\n\t\t\treturn false\n\t\tcase <-timer.C:\n\t\t\tpod, err := config.K8s.GetPod(podName)\n\t\t\tif err != nil {\n\t\t\t\twriteAndCloseSocket(socket, fmt.Sprintf(\"error encountered while attempting to stream logs from pod %s\\n%s\", podName, err.Error()))\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif pod == nil {\n\t\t\t\twriteAndCloseSocket(socket, \"unable to find pod\")\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tpodStatus := k8s.GetPodStatus(pod)\n\t\t\tif podStatus == k8s.PodStatusPending {\n\t\t\t\tif !wrotePending {\n\t\t\t\t\twriteString(socket, \"waiting for pod to initialize ...\\n\")\n\t\t\t\t}\n\t\t\t\twrotePending = true\n\t\t\t\ttimer.Reset(_pendingPodCheckInterval)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\ntype jsonMessage struct {\n\tMessage string `json:\"message\"`\n\tExcInfo string `json:\"exc_info\"`\n}\n\nfunc startKubectlProcess(podName string, cancelListener chan struct{}, socket *websocket.Conn) {\n\tshouldContinue := waitForPodToBeNotPending(podName, cancelListener, socket)\n\tif !shouldContinue {\n\t\treturn\n\t}\n\n\tcmd := exec.Command(\"/usr/local/bin/kubectl\", \"-n=\"+config.K8s.Namespace, \"logs\", \"--all-containers\", podName, \"--follow\")\n\n\tcleanup := func() {\n\t\t// trigger a wait on the child process and while the process is being waited on,\n\t\t// send the kill signal to allow cleanup to happen correctly and prevent zombie processes\n\t\ttime.AfterFunc(1*time.Second, func() {\n\t\t\tcmd.Process.Kill()\n\t\t})\n\n\t\tcmd.Process.Wait()\n\t}\n\tdefer cleanup()\n\n\tlogStream, err := cmd.StdoutPipe()\n\tif err != nil {\n\t\ttelemetry.Error(errors.ErrorUnexpected(err.Error()))\n\t\toperatorLogger.Error(err)\n\t}\n\n\tcmd.Start()\n\n\troutines.RunWithPanicHandler(func() {\n\t\tpumpStdout(socket, logStream)\n\t})\n\n\t<-cancelListener\n}\n\nfunc pumpStdout(socket *websocket.Conn, reader io.Reader) {\n\t// it seems like if the buffer is maxed out with no ending token, the scanner just exits.\n\t// increase the buffer used by the scanner to accommodate larger log lines (a common issue when printing progress)\n\tp := make([]byte, 1024*1024)\n\tscanner := bufio.NewScanner(reader)\n\tscanner.Buffer(p, 1024*1024)\n\tfor scanner.Scan() {\n\t\tlogBytes := scanner.Bytes()\n\t\tvar message jsonMessage\n\t\terr := json.Unmarshal(logBytes, &message)\n\t\tif err != nil {\n\t\t\twriteString(socket, string(logBytes)+\"\\n\")\n\t\t} else {\n\t\t\twriteString(socket, message.Message+\"\\n\")\n\t\t\tif message.ExcInfo != \"\" {\n\t\t\t\twriteString(socket, message.ExcInfo+\"\\n\")\n\t\t\t}\n\t\t}\n\t}\n\n\tcloseSocket(socket)\n}\n\nfunc StreamLogsFromRandomPod(podSearchLabels map[string]string, socket *websocket.Conn) {\n\tpods, err := config.K8s.ListPodsByLabels(podSearchLabels)\n\tif err != nil {\n\t\twriteAndCloseSocket(socket, err.Error())\n\t\treturn\n\t}\n\tif len(pods) == 0 {\n\t\twriteAndCloseSocket(socket, \"there are currently no pods running for this workload; please visit your logging dashboard for historical logs\\n\")\n\t\treturn\n\t}\n\n\tcancelListener := make(chan struct{})\n\tdefer close(cancelListener)\n\troutines.RunWithPanicHandler(func() {\n\t\tstartKubectlProcess(pods[0].Name, cancelListener, socket)\n\t})\n\tpumpStdin(socket)\n\tcancelListener <- struct{}{}\n}\n\nfunc pumpStdin(socket *websocket.Conn) {\n\tsocket.SetReadLimit(_socketMaxMessageSize)\n\tfor {\n\t\t_, _, err := socket.ReadMessage()\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc writeString(socket *websocket.Conn, message string) {\n\tsocket.WriteMessage(websocket.TextMessage, []byte(message))\n}\n\nfunc writeAndCloseSocket(socket *websocket.Conn, message string) {\n\twriteString(socket, message)\n\tcloseSocket(socket)\n}\n\nfunc closeSocket(socket *websocket.Conn) {\n\tsocket.SetWriteDeadline(time.Now().Add(_socketWriteDeadlineWait))\n\tsocket.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, \"\"))\n\ttime.Sleep(_socketCloseGracePeriod)\n}\n"
  },
  {
    "path": "pkg/operator/resources/asyncapi/api.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage asyncapi\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/cron\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nconst (\n\t_tickPeriodMetrics = 10 * time.Second\n\t_asyncDashboardUID = \"asyncapi\"\n)\n\nvar (\n\t_metricsCrons = make(map[string]cron.Cron)\n)\n\ntype resources struct {\n\tapiDeployment     *kapps.Deployment\n\tapiConfigMap      *kcore.ConfigMap\n\tapiVirtualService *istioclientnetworking.VirtualService\n}\n\nfunc generateDeploymentID() string {\n\treturn k8s.RandomName()[:10]\n}\n\nfunc UpdateAPI(apiConfig userconfig.API, force bool) (*spec.API, string, error) {\n\tprevK8sResources, err := getK8sResources(apiConfig.Name)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tinitialDeploymentTime := time.Now().UnixNano()\n\tdeploymentID := generateDeploymentID()\n\tif prevK8sResources.apiVirtualService != nil && prevK8sResources.apiVirtualService.Labels[\"initialDeploymentTime\"] != \"\" {\n\t\tvar err error\n\t\tinitialDeploymentTime, err = k8s.ParseInt64Label(prevK8sResources.apiVirtualService, \"initialDeploymentTime\")\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\t\tdeploymentID = prevK8sResources.apiVirtualService.Labels[\"deploymentID\"]\n\t}\n\n\tapi := spec.GetAPISpec(&apiConfig, initialDeploymentTime, deploymentID, config.ClusterConfig.ClusterUID)\n\n\t// resource creation\n\tif prevK8sResources.apiVirtualService == nil {\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"upload api spec\")\n\t\t}\n\n\t\ttags := map[string]string{\n\t\t\t\"apiName\": apiConfig.Name,\n\t\t}\n\n\t\tqueueURL, err := createFIFOQueue(apiConfig.Name, initialDeploymentTime, tags)\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\tif err = applyK8sResources(*api, prevK8sResources, queueURL); err != nil {\n\t\t\troutines.RunWithPanicHandler(func() {\n\t\t\t\t_ = parallel.RunFirstErr(\n\t\t\t\t\tfunc() error {\n\t\t\t\t\t\treturn deleteQueueByURL(queueURL)\n\t\t\t\t\t},\n\t\t\t\t\tfunc() error {\n\t\t\t\t\t\treturn deleteK8sResources(api.Name)\n\t\t\t\t\t},\n\t\t\t\t)\n\t\t\t})\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"creating %s\", api.Resource.UserString()), nil\n\t}\n\n\t// resource update\n\tif prevK8sResources.apiVirtualService.Labels[\"specID\"] != api.SpecID {\n\t\tisUpdating, err := isAPIUpdating(prevK8sResources.apiDeployment)\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\t\tif isUpdating && !force {\n\t\t\treturn nil, \"\", ErrorAPIUpdating(api.Name)\n\t\t}\n\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"upload api spec\")\n\t\t}\n\n\t\tinitialDeploymentTime, err := k8s.ParseInt64Label(prevK8sResources.apiVirtualService, \"initialDeploymentTime\")\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\tqueueURL, err := getQueueURL(api.Name, initialDeploymentTime)\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\tif err = applyK8sResources(*api, prevK8sResources, queueURL); err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"updating %s\", api.Resource.UserString()), nil\n\t}\n\n\t// nothing changed\n\tisUpdating, err := isAPIUpdating(prevK8sResources.apiDeployment)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tif isUpdating {\n\t\treturn api, fmt.Sprintf(\"%s is already updating\", api.Resource.UserString()), nil\n\t}\n\treturn api, fmt.Sprintf(\"%s is up to date\", api.Resource.UserString()), nil\n}\n\nfunc RefreshAPI(apiName string, force bool) (string, error) {\n\tprevK8sResources, err := getK8sResources(apiName)\n\tif err != nil {\n\t\treturn \"\", err\n\t} else if prevK8sResources.apiVirtualService == nil || prevK8sResources.apiDeployment == nil {\n\t\treturn \"\", errors.ErrorUnexpected(\"unable to find deployment\", apiName)\n\t}\n\n\tisUpdating, err := isAPIUpdating(prevK8sResources.apiDeployment)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif isUpdating && !force {\n\t\treturn \"\", ErrorAPIUpdating(apiName)\n\t}\n\n\tapiID, err := k8s.GetLabel(prevK8sResources.apiVirtualService, \"apiID\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tapi, err := operator.DownloadAPISpec(apiName, apiID)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tinitialDeploymentTime, err := k8s.ParseInt64Label(prevK8sResources.apiVirtualService, \"initialDeploymentTime\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tapi = spec.GetAPISpec(api.API, initialDeploymentTime, generateDeploymentID(), config.ClusterConfig.ClusterUID)\n\n\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\treturn \"\", errors.Wrap(err, \"upload api spec\")\n\t}\n\n\tqueueURL, err := getQueueURL(api.Name, initialDeploymentTime)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif err = applyK8sResources(*api, prevK8sResources, queueURL); err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn fmt.Sprintf(\"updating %s\", api.Resource.UserString()), nil\n}\n\nfunc DeleteAPI(apiName string, keepCache bool) error {\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tvs, err := config.K8s.GetVirtualService(workloads.K8sName(apiName))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif vs != nil {\n\t\t\t\tinitialDeploymentTime, err := k8s.ParseInt64Label(vs, \"initialDeploymentTime\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tqueueURL, err := getQueueURL(apiName, initialDeploymentTime)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\t// best effort deletion\n\t\t\t\t_ = deleteQueueByURL(queueURL)\n\t\t\t}\n\t\t\treturn nil\n\t\t},\n\t\tfunc() error {\n\t\t\treturn deleteK8sResources(apiName)\n\t\t},\n\t\tfunc() error {\n\t\t\tif keepCache {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\t// best effort deletion, swallow errors because there could be weird error messages\n\t\t\t_ = deleteBucketResources(apiName)\n\t\t\treturn nil\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc GetAllAPIs(deployments []kapps.Deployment) ([]schema.APIResponse, error) {\n\tasyncAPIs := make([]schema.APIResponse, 0)\n\tmappedAsyncAPIs := make(map[string]schema.APIResponse, 0)\n\tapiNames := make([]string, 0)\n\n\tfor i := range deployments {\n\t\tapiName := deployments[i].Labels[\"apiName\"]\n\t\tapiNames = append(apiNames, apiName)\n\n\t\tmetadata, err := spec.MetadataFromDeployment(&deployments[i])\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"api %s\", apiName))\n\t\t}\n\t\tmappedAsyncAPIs[apiName] = schema.APIResponse{\n\t\t\tStatus:   status.FromDeployment(&deployments[i]),\n\t\t\tMetadata: metadata,\n\t\t}\n\t}\n\n\tsort.Strings(apiNames)\n\tfor _, apiName := range apiNames {\n\t\tasyncAPIs = append(asyncAPIs, mappedAsyncAPIs[apiName])\n\t}\n\n\treturn asyncAPIs, nil\n}\n\nfunc GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) {\n\tapiDeployment, err := config.K8s.GetDeployment(workloads.K8sName(deployedResource.Name))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif apiDeployment == nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to find api deployment\", deployedResource.Name)\n\t}\n\n\tapiStatus := status.FromDeployment(apiDeployment)\n\tapiMetadata, err := spec.MetadataFromDeployment(apiDeployment)\n\tif err != nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to obtain metadata\", deployedResource.Name)\n\t}\n\n\tapi, err := operator.DownloadAPISpec(apiMetadata.Name, apiMetadata.APIID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tapiEndpoint, err := operator.APIEndpoint(api)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdashboardURL := pointer.String(getDashboardURL(api.Name))\n\n\treturn []schema.APIResponse{\n\t\t{\n\t\t\tSpec:         api,\n\t\t\tMetadata:     apiMetadata,\n\t\t\tStatus:       apiStatus,\n\t\t\tEndpoint:     &apiEndpoint,\n\t\t\tDashboardURL: dashboardURL,\n\t\t},\n\t}, nil\n}\n\nfunc DescribeAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) {\n\tvar apiDeployment *kapps.Deployment\n\n\tapiDeployment, err := config.K8s.GetDeployment(workloads.K8sName(deployedResource.Name))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif apiDeployment == nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to find api deployment\", deployedResource.Name)\n\t}\n\n\tapiStatus := status.FromDeployment(apiDeployment)\n\tapiMetadata, err := spec.MetadataFromDeployment(apiDeployment)\n\tif err != nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to obtain metadata\", deployedResource.Name)\n\t}\n\n\tapiPods, err := config.K8s.ListPodsByLabels(map[string]string{\n\t\t\"apiName\": apiDeployment.Labels[\"apiName\"],\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tapiStatus.ReplicaCounts = GetReplicaCounts(apiDeployment, apiPods)\n\n\tapiEndpoint, err := operator.APIEndpointFromResource(deployedResource)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdashboardURL := pointer.String(getDashboardURL(deployedResource.Name))\n\n\treturn []schema.APIResponse{\n\t\t{\n\t\t\tMetadata:     apiMetadata,\n\t\t\tStatus:       apiStatus,\n\t\t\tEndpoint:     &apiEndpoint,\n\t\t\tDashboardURL: dashboardURL,\n\t\t},\n\t}, nil\n}\n\nfunc UpdateAPIMetricsCron(apiDeployment *kapps.Deployment) error {\n\tapiName := apiDeployment.Labels[\"apiName\"]\n\n\tif prevMetricsCron, ok := _metricsCrons[apiName]; ok {\n\t\tprevMetricsCron.Cancel()\n\t}\n\n\tinitialDeploymentTime, err := k8s.ParseInt64Label(apiDeployment, \"initialDeploymentTime\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tqueueURL, err := getQueueURL(apiName, initialDeploymentTime)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tmetricsCron := updateQueueLengthMetricsFn(apiName, queueURL)\n\n\t_metricsCrons[apiName] = cron.Run(metricsCron, operator.ErrorHandler(apiName+\" metrics\"), _tickPeriodMetrics)\n\n\treturn nil\n}\n\nfunc getK8sResources(apiName string) (resources, error) {\n\tvar deployment *kapps.Deployment\n\tvar apiConfigMap *kcore.ConfigMap\n\tvar apiVirtualService *istioclientnetworking.VirtualService\n\n\tapiK8sName := workloads.K8sName(apiName)\n\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tdeployment, err = config.K8s.GetDeployment(apiK8sName)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tapiConfigMap, err = config.K8s.GetConfigMap(apiK8sName)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tapiVirtualService, err = config.K8s.GetVirtualService(apiK8sName)\n\t\t\treturn err\n\t\t},\n\t)\n\n\treturn resources{\n\t\tapiDeployment:     deployment,\n\t\tapiConfigMap:      apiConfigMap,\n\t\tapiVirtualService: apiVirtualService,\n\t}, err\n}\n\nfunc applyK8sResources(api spec.API, prevK8sResources resources, queueURL string) error {\n\tapiDeployment := deploymentSpec(api, prevK8sResources.apiDeployment, queueURL)\n\tapiConfigMap, err := configMapSpec(api)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tapiVirtualService := apiVirtualServiceSpec(api, queueURL)\n\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tif err := applyK8sConfigMap(prevK8sResources.apiConfigMap, &apiConfigMap); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tif err := applyK8sDeployment(prevK8sResources.apiDeployment, &apiDeployment); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tif err := UpdateAPIMetricsCron(&apiDeployment); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\treturn nil\n\t\t},\n\t\tfunc() error {\n\t\t\treturn applyK8sVirtualService(prevK8sResources.apiVirtualService, &apiVirtualService)\n\t\t},\n\t)\n}\n\nfunc applyK8sConfigMap(prevConfigMap *kcore.ConfigMap, newConfigMap *kcore.ConfigMap) error {\n\tif prevConfigMap == nil {\n\t\t_, err := config.K8s.CreateConfigMap(newConfigMap)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\t_, err := config.K8s.UpdateConfigMap(newConfigMap)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc applyK8sDeployment(prevDeployment *kapps.Deployment, newDeployment *kapps.Deployment) error {\n\tif prevDeployment == nil {\n\t\t_, err := config.K8s.CreateDeployment(newDeployment)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if prevDeployment.Status.ReadyReplicas == 0 {\n\t\t// Delete deployment if it never became ready\n\t\t_, _ = config.K8s.DeleteDeployment(prevDeployment.Name)\n\t\t_, err := config.K8s.CreateDeployment(newDeployment)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\t_, err := config.K8s.UpdateDeployment(newDeployment)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc applyK8sVirtualService(prevVirtualService *istioclientnetworking.VirtualService, newVirtualService *istioclientnetworking.VirtualService) error {\n\tif prevVirtualService == nil {\n\t\t_, err := config.K8s.CreateVirtualService(newVirtualService)\n\t\treturn err\n\t}\n\n\t_, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService)\n\treturn err\n}\n\nfunc deleteBucketResources(apiName string) error {\n\tprefix := filepath.Join(config.ClusterConfig.ClusterUID, \"apis\", apiName)\n\treturn config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true)\n}\n\nfunc deleteK8sResources(apiName string) error {\n\tapiK8sName := workloads.K8sName(apiName)\n\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tif metricsCron, ok := _metricsCrons[apiName]; ok {\n\t\t\t\tmetricsCron.Cancel()\n\t\t\t\tdelete(_metricsCrons, apiName)\n\t\t\t}\n\n\t\t\t_, err := config.K8s.DeleteDeployment(apiK8sName)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\t_, err := config.K8s.DeleteConfigMap(apiK8sName)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\t_, err := config.K8s.DeleteVirtualService(apiK8sName)\n\t\t\treturn err\n\t\t},\n\t)\n\n\treturn err\n}\n\n// returns true if min_replicas are not ready and no updated replicas have errored\nfunc isAPIUpdating(deployment *kapps.Deployment) (bool, error) {\n\tpods, err := config.K8s.ListPodsByLabel(\"apiName\", deployment.Labels[\"apiName\"])\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treplicaCounts := GetReplicaCounts(deployment, pods)\n\n\tautoscalingSpec, err := userconfig.AutoscalingFromAnnotations(deployment)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif replicaCounts.Ready < autoscalingSpec.MinReplicas && replicaCounts.TotalFailed() == 0 {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc isPodSpecLatest(deployment *kapps.Deployment, pod *kcore.Pod) bool {\n\treturn deployment.Spec.Template.Labels[\"podID\"] == pod.Labels[\"podID\"] &&\n\t\tdeployment.Spec.Template.Labels[\"deploymentID\"] == pod.Labels[\"deploymentID\"]\n}\n\nfunc getDashboardURL(apiName string) string {\n\tloadBalancerURL, err := operator.LoadBalancerURL()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\n\tdashboardURL := fmt.Sprintf(\n\t\t\"%s/dashboard/d/%s/asyncapi?orgId=1&refresh=30s&var-api_name=%s\",\n\t\tloadBalancerURL, _asyncDashboardUID, apiName,\n\t)\n\n\treturn dashboardURL\n}\n"
  },
  {
    "path": "pkg/operator/resources/asyncapi/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage asyncapi\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrAPIUpdating = \"asyncapi.api_updating\"\n)\n\nfunc ErrorAPIUpdating(apiName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAPIUpdating,\n\t\tMessage: fmt.Sprintf(\"%s is updating (override with --force)\", apiName),\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/resources/asyncapi/k8s_specs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage asyncapi\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistionetworking \"istio.io/api/networking/v1beta1\"\n\t\"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nvar _terminationGracePeriodSeconds int64 = 60 // seconds\n\nfunc apiVirtualServiceSpec(api spec.API, queueURL string) v1beta1.VirtualService {\n\treturn *k8s.VirtualService(&k8s.VirtualServiceSpec{\n\t\tName:     workloads.K8sName(api.Name),\n\t\tGateways: []string{\"apis-gateway\"},\n\t\tDestinations: []k8s.Destination{\n\t\t\t{\n\t\t\t\tServiceName: \"async-gateway\",\n\t\t\t\tWeight:      100,\n\t\t\t\tPort:        uint32(consts.ProxyPortInt32),\n\t\t\t\tHeaders: &istionetworking.Headers{\n\t\t\t\t\tRequest: &istionetworking.Headers_HeaderOperations{\n\t\t\t\t\t\tSet: map[string]string{\n\t\t\t\t\t\t\tconsts.CortexAPINameHeader:  api.Name,\n\t\t\t\t\t\t\tconsts.CortexQueueURLHeader: queueURL,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tPrefixPath:  api.Networking.Endpoint,\n\t\tRewrite:     pointer.String(\"/\"),\n\t\tAnnotations: api.ToK8sAnnotations(),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":               api.Name,\n\t\t\t\"apiKind\":               api.Kind.String(),\n\t\t\t\"apiID\":                 api.ID,\n\t\t\t\"specID\":                api.SpecID,\n\t\t\t\"initialDeploymentTime\": s.Int64(api.InitialDeploymentTime),\n\t\t\t\"deploymentID\":          api.DeploymentID,\n\t\t\t\"podID\":                 api.PodID,\n\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t},\n\t})\n}\n\nfunc configMapSpec(api spec.API) (kcore.ConfigMap, error) {\n\tconfigMapConfig := workloads.ConfigMapConfig{\n\t\tProbes: workloads.GetReadinessProbesFromContainers(api.Pod.Containers),\n\t}\n\n\tconfigMapData, err := configMapConfig.GenerateConfigMapData()\n\tif err != nil {\n\t\treturn kcore.ConfigMap{}, err\n\t}\n\n\treturn *k8s.ConfigMap(&k8s.ConfigMapSpec{\n\t\tName: workloads.K8sName(api.Name),\n\t\tData: configMapData,\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":        api.Name,\n\t\t\t\"apiKind\":        api.Kind.String(),\n\t\t\t\"cortex.dev/api\": \"true\",\n\t\t},\n\t}), nil\n}\n\nfunc deploymentSpec(api spec.API, prevDeployment *kapps.Deployment, queueURL string) kapps.Deployment {\n\tvar (\n\t\tcontainers []kcore.Container\n\t\tvolumes    []kcore.Volume\n\t)\n\n\tcontainers, volumes = workloads.AsyncContainers(api, queueURL)\n\n\treturn *k8s.Deployment(&k8s.DeploymentSpec{\n\t\tName:           workloads.K8sName(api.Name),\n\t\tReplicas:       getRequestedReplicasFromDeployment(api, prevDeployment),\n\t\tMaxSurge:       pointer.String(api.UpdateStrategy.MaxSurge),\n\t\tMaxUnavailable: pointer.String(api.UpdateStrategy.MaxUnavailable),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":               api.Name,\n\t\t\t\"apiKind\":               api.Kind.String(),\n\t\t\t\"apiID\":                 api.ID,\n\t\t\t\"specID\":                api.SpecID,\n\t\t\t\"initialDeploymentTime\": s.Int64(api.InitialDeploymentTime),\n\t\t\t\"deploymentID\":          api.DeploymentID,\n\t\t\t\"podID\":                 api.PodID,\n\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t},\n\t\tAnnotations: api.ToK8sAnnotations(),\n\t\tSelector: map[string]string{\n\t\t\t\"apiName\": api.Name,\n\t\t\t\"apiKind\": api.Kind.String(),\n\t\t},\n\t\tPodSpec: k8s.PodSpec{\n\t\t\tLabels: map[string]string{\n\t\t\t\t\"apiName\":               api.Name,\n\t\t\t\t\"apiKind\":               api.Kind.String(),\n\t\t\t\t\"apiID\":                 api.ID,\n\t\t\t\t\"initialDeploymentTime\": s.Int64(api.InitialDeploymentTime),\n\t\t\t\t\"deploymentID\":          api.DeploymentID,\n\t\t\t\t\"podID\":                 api.PodID,\n\t\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t\t},\n\t\t\tK8sPodSpec: kcore.PodSpec{\n\t\t\t\tRestartPolicy:                 \"Always\",\n\t\t\t\tTerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds),\n\t\t\t\tContainers:                    containers,\n\t\t\t\tNodeSelector:                  workloads.NodeSelectors(),\n\t\t\t\tTolerations:                   workloads.GenerateResourceTolerations(),\n\t\t\t\tAffinity:                      workloads.GenerateNodeAffinities(api.NodeGroups),\n\t\t\t\tVolumes:                       volumes,\n\t\t\t\tServiceAccountName:            workloads.ServiceAccountName,\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc getRequestedReplicasFromDeployment(api spec.API, deployment *kapps.Deployment) int32 {\n\trequestedReplicas := api.Autoscaling.InitReplicas\n\n\tif deployment != nil && deployment.Spec.Replicas != nil {\n\t\trequestedReplicas = *deployment.Spec.Replicas\n\t}\n\n\tif requestedReplicas < api.Autoscaling.MinReplicas {\n\t\trequestedReplicas = api.Autoscaling.MinReplicas\n\t}\n\n\tif requestedReplicas > api.Autoscaling.MaxReplicas {\n\t\trequestedReplicas = api.Autoscaling.MaxReplicas\n\t}\n\n\treturn requestedReplicas\n}\n"
  },
  {
    "path": "pkg/operator/resources/asyncapi/queue.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage asyncapi\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n)\n\nfunc createFIFOQueue(apiName string, initialDeploymentTime int64, tags map[string]string) (string, error) {\n\tfor key, value := range config.ClusterConfig.Tags {\n\t\ttags[key] = value\n\t}\n\n\tqueueName := apiQueueName(apiName, initialDeploymentTime)\n\n\tattributes := map[string]string{\n\t\tsqs.QueueAttributeNameFifoQueue:         \"true\",\n\t\tsqs.QueueAttributeNameVisibilityTimeout: \"60\",\n\t}\n\n\toutput, err := config.AWS.SQS().CreateQueue(\n\t\t&sqs.CreateQueueInput{\n\t\t\tAttributes: aws.StringMap(attributes),\n\t\t\tQueueName:  aws.String(queueName),\n\t\t\tTags:       aws.StringMap(tags),\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to create sqs queue\", queueName)\n\t}\n\n\treturn *output.QueueUrl, nil\n}\n\nfunc apiQueueName(apiName string, initialDeploymentTime int64) string {\n\t// initialDeploymentTime is incorporated so that the queue name changes when doing a deploy after a delete\n\t// (if the queue name doesn't change, the user would have to wait 60 seconds before recreating the queue)\n\tinitialDeploymentTimeStr := s.Int64(initialDeploymentTime)\n\tinitialDeploymentTimeID := initialDeploymentTimeStr[len(initialDeploymentTimeStr)-10:]\n\treturn config.ClusterConfig.SQSNamePrefix() + apiName + clusterconfig.SQSQueueDelimiter + initialDeploymentTimeID + \".fifo\"\n}\n\nfunc deleteQueueByURL(queueURL string) error {\n\t_, err := config.AWS.SQS().DeleteQueue(&sqs.DeleteQueueInput{\n\t\tQueueUrl: aws.String(queueURL),\n\t})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to delete queue\", queueURL)\n\t}\n\n\treturn err\n}\n\nfunc getQueueURL(apiName string, initialDeploymentTime int64) (string, error) {\n\toperatorAccountID, _, err := config.AWS.GetCachedAccountID()\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to construct queue url\", \"unable to get account id\")\n\t}\n\n\treturn fmt.Sprintf(\n\t\t\"https://sqs.%s.amazonaws.com/%s/%s\",\n\t\tconfig.AWS.Region, operatorAccountID, apiQueueName(apiName, initialDeploymentTime),\n\t), nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/asyncapi/queue_metrics.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage asyncapi\n\nimport (\n\t\"context\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/sqs\"\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promauto\"\n)\n\nconst (\n\t_sqsQueryTimeoutSeconds = 10\n)\n\nvar activeGauge = promauto.NewGaugeVec(\n\tprometheus.GaugeOpts{\n\t\tName:        \"cortex_async_active\",\n\t\tHelp:        \"The number of messages that are actively being processed by an AsyncAPI\",\n\t\tConstLabels: map[string]string{\"api_kind\": userconfig.AsyncAPIKind.String()},\n\t}, []string{\"api_name\"},\n)\n\nvar queuedGauge = promauto.NewGaugeVec(\n\tprometheus.GaugeOpts{\n\t\tName:        \"cortex_async_queued\",\n\t\tHelp:        \"The number queued messages for an AsyncAPI\",\n\t\tConstLabels: map[string]string{\"api_kind\": userconfig.AsyncAPIKind.String()},\n\t}, []string{\"api_name\"},\n)\n\nvar inFlightGauge = promauto.NewGaugeVec(\n\tprometheus.GaugeOpts{\n\t\tName:        \"cortex_async_in_flight\",\n\t\tHelp:        \"The number of in-flight messages for an AsyncAPI (including active and queued)\",\n\t\tConstLabels: map[string]string{\"api_kind\": userconfig.AsyncAPIKind.String()},\n\t}, []string{\"api_name\"},\n)\n\nfunc updateQueueLengthMetricsFn(apiName, queueURL string) func() error {\n\treturn func() error {\n\t\tsqsClient := config.AWS.SQS()\n\n\t\tctx, cancel := context.WithTimeout(context.Background(), _sqsQueryTimeoutSeconds*time.Second)\n\t\tdefer cancel()\n\n\t\tinput := &sqs.GetQueueAttributesInput{\n\t\t\tAttributeNames: []*string{\n\t\t\t\taws.String(\"ApproximateNumberOfMessages\"),\n\t\t\t\taws.String(\"ApproximateNumberOfMessagesNotVisible\"),\n\t\t\t},\n\t\t\tQueueUrl: aws.String(queueURL),\n\t\t}\n\n\t\toutput, err := sqsClient.GetQueueAttributesWithContext(ctx, input)\n\t\tif err != nil {\n\t\t\treturn errors.WithStack(err)\n\t\t}\n\n\t\tvisibleMessagesStr := output.Attributes[\"ApproximateNumberOfMessages\"]\n\t\tinvisibleMessagesStr := output.Attributes[\"ApproximateNumberOfMessagesNotVisible\"]\n\n\t\tvisibleMessages, err := strconv.ParseFloat(*visibleMessagesStr, 64)\n\t\tif err != nil {\n\t\t\treturn errors.WithStack(err)\n\t\t}\n\n\t\tinvisibleMessages, err := strconv.ParseFloat(*invisibleMessagesStr, 64)\n\t\tif err != nil {\n\t\t\treturn errors.WithStack(err)\n\t\t}\n\n\t\tactiveGauge.WithLabelValues(apiName).Set(invisibleMessages)\n\t\tqueuedGauge.WithLabelValues(apiName).Set(visibleMessages)\n\t\tinFlightGauge.WithLabelValues(apiName).Set(invisibleMessages + visibleMessages)\n\n\t\treturn nil\n\t}\n}\n"
  },
  {
    "path": "pkg/operator/resources/asyncapi/status.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage asyncapi\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nfunc GetReplicaCounts(deployment *kapps.Deployment, pods []kcore.Pod) *status.ReplicaCounts {\n\tcounts := status.ReplicaCounts{}\n\tcounts.Requested = *deployment.Spec.Replicas\n\n\tfor i := range pods {\n\t\tpod := pods[i]\n\n\t\tif pod.Labels[\"apiName\"] != deployment.Labels[\"apiName\"] {\n\t\t\tcontinue\n\t\t}\n\t\taddPodToReplicaCounts(&pod, deployment, &counts)\n\t}\n\n\treturn &counts\n}\n\nfunc addPodToReplicaCounts(pod *kcore.Pod, deployment *kapps.Deployment, counts *status.ReplicaCounts) {\n\tlatest := false\n\tif isPodSpecLatest(deployment, pod) {\n\t\tlatest = true\n\t}\n\n\tisPodReady := k8s.IsPodReady(pod)\n\tif latest && isPodReady {\n\t\tcounts.Ready++\n\t\treturn\n\t} else if !latest && isPodReady {\n\t\tcounts.ReadyOutOfDate++\n\t\treturn\n\t}\n\n\tpodStatus := k8s.GetPodStatus(pod)\n\n\tif podStatus == k8s.PodStatusTerminating {\n\t\tcounts.Terminating++\n\t\treturn\n\t}\n\n\tif !latest {\n\t\treturn\n\t}\n\n\tswitch podStatus {\n\tcase k8s.PodStatusPending:\n\t\tcounts.Pending++\n\tcase k8s.PodStatusStalled:\n\t\tcounts.Stalled++\n\tcase k8s.PodStatusCreating:\n\t\tcounts.Creating++\n\tcase k8s.PodStatusReady:\n\t\tcounts.Ready++\n\tcase k8s.PodStatusNotReady:\n\t\tcounts.NotReady++\n\tcase k8s.PodStatusErrImagePull:\n\t\tcounts.ErrImagePull++\n\tcase k8s.PodStatusFailed:\n\t\tcounts.Failed++\n\tcase k8s.PodStatusKilled:\n\t\tcounts.Killed++\n\tcase k8s.PodStatusKilledOOM:\n\t\tcounts.KilledOOM++\n\tcase k8s.PodStatusUnknown:\n\t\tcounts.Unknown++\n\t}\n}\n"
  },
  {
    "path": "pkg/operator/resources/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage resources\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/console\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n)\n\nconst (\n\tErrOperationIsOnlySupportedForKind  = \"resources.operation_is_only_supported_for_kind\"\n\tErrAPINotDeployed                   = \"resources.api_not_deployed\"\n\tErrAPIIDNotFound                    = \"resources.api_id_not_found\"\n\tErrCannotChangeTypeOfDeployedAPI    = \"resources.cannot_change_kind_of_deployed_api\"\n\tErrNoAvailableNodeComputeLimit      = \"resources.no_available_node_compute_limit\"\n\tErrJobIDRequired                    = \"resources.job_id_required\"\n\tErrRealtimeAPIUsedByTrafficSplitter = \"resources.realtime_api_used_by_traffic_splitter\"\n\tErrAPIsNotDeployed                  = \"resources.apis_not_deployed\"\n\tErrInvalidNodeGroupSelector         = \"resources.invalid_node_group_selector\"\n\tErrNoNodeGroups                     = \"resources.no_node_groups\"\n)\n\nfunc ErrorOperationIsOnlySupportedForKind(resource operator.DeployedResource, supportedKind userconfig.Kind, supportedKinds ...userconfig.Kind) error {\n\tsupportedKindsSlice := append(make([]string, 0, 1+len(supportedKinds)), supportedKind.String())\n\tfor _, kind := range supportedKinds {\n\t\tsupportedKindsSlice = append(supportedKindsSlice, kind.String())\n\t}\n\n\tmsg := fmt.Sprintf(\"%s %s\", s.StrsOr(supportedKindsSlice), s.PluralS(userconfig.KindKey, len(supportedKindsSlice)))\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrOperationIsOnlySupportedForKind,\n\t\tMessage: fmt.Sprintf(\"this operation is only allowed for %s and is not supported for %s of kind %s\", msg, resource.Name, resource.Kind),\n\t})\n}\n\nfunc ErrorAPINotDeployed(apiName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAPINotDeployed,\n\t\tMessage: fmt.Sprintf(\"%s is not deployed\", apiName),\n\t})\n}\n\nfunc ErrorAPIIDNotFound(apiName string, apiID string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAPIIDNotFound,\n\t\tMessage: fmt.Sprintf(\"%s with id %s has never been deployed\", apiName, apiID),\n\t})\n}\n\nfunc ErrorCannotChangeKindOfDeployedAPI(name string, newKind, prevKind userconfig.Kind) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCannotChangeTypeOfDeployedAPI,\n\t\tMessage: fmt.Sprintf(\"cannot change the kind of %s to %s because it has already been deployed with kind %s; please delete it with `cortex delete %s` and redeploy after updating the api configuration appropriately\", name, newKind.String(), prevKind.String(), name),\n\t})\n}\n\nfunc ErrorNoAvailableNodeComputeLimit(api *userconfig.API, compute userconfig.Compute, maxMemMap map[string]kresource.Quantity) error {\n\tmsg := \"no instance types in your cluster are large enough to satisfy the requested resources for your pod\\n\\n\"\n\tmsg += console.Bold(\"requested pod resources\\n\")\n\tmsg += podResourceRequestsTable(api, compute)\n\tmsg += \"\\n\" + s.TrimTrailingNewLines(nodeGroupResourcesTable(api, compute, maxMemMap))\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoAvailableNodeComputeLimit,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorAPIUsedByTrafficSplitter(trafficSplitters []string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrRealtimeAPIUsedByTrafficSplitter,\n\t\tMessage: fmt.Sprintf(\"cannot delete api because it is used by the following %s: %s\", s.PluralS(\"TrafficSplitter\", len(trafficSplitters)), s.StrsSentence(trafficSplitters, \"\")),\n\t})\n}\n\nfunc ErrorAPIsNotDeployed(notDeployedAPIs []string) error {\n\tmessage := fmt.Sprintf(\"apis %s were either not found or are not RealtimeAPIs\", s.StrsAnd(notDeployedAPIs))\n\tif len(notDeployedAPIs) == 1 {\n\t\tmessage = fmt.Sprintf(\"api %s was either not found or is not a RealtimeAPI\", notDeployedAPIs[0])\n\t}\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAPIsNotDeployed,\n\t\tMessage: message,\n\t})\n}\n\nfunc ErrorInvalidNodeGroupSelector(selected string, availableNodeGroups []string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidNodeGroupSelector,\n\t\tMessage: fmt.Sprintf(\"node group \\\"%s\\\" doesn't exist; remove the node group selector to let Cortex determine automatically where to place the API, or specify a valid node group name (%s)\", selected, s.StrsOr(availableNodeGroups)),\n\t})\n}\n\nfunc ErrorNoNodeGroups() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoNodeGroups,\n\t\tMessage: fmt.Sprintf(\"your api cannot be deployed because your cluster doesn't have any node groups; create a node group with `cortex cluster configure CLUSTER_CONFIG_FILE`\"),\n\t})\n}\n\nfunc podResourceRequestsTable(api *userconfig.API, compute userconfig.Compute) string {\n\tsidecarCPUNote := \"\"\n\tsidecarMemNote := \"\"\n\tif api.Kind == userconfig.RealtimeAPIKind {\n\t\tsidecarCPUNote = fmt.Sprintf(\" (including %s for the %s sidecar container)\", consts.CortexProxyCPU.String(), workloads.ProxyContainerName)\n\t\tsidecarMemNote = fmt.Sprintf(\" (including %s for the %s sidecar container)\", k8s.ToMiCeilStr(consts.CortexProxyMem), workloads.ProxyContainerName)\n\t} else if api.Kind == userconfig.AsyncAPIKind || api.Kind == userconfig.BatchAPIKind {\n\t\tsidecarCPUNote = fmt.Sprintf(\" (including %s for the %s sidecar container)\", consts.CortexDequeuerCPU.String(), workloads.DequeuerContainerName)\n\t\tsidecarMemNote = fmt.Sprintf(\" (including %s for the %s sidecar container)\", k8s.ToMiCeilStr(consts.CortexDequeuerMem), workloads.DequeuerContainerName)\n\t}\n\n\tvar items table.KeyValuePairs\n\tif compute.CPU != nil {\n\t\titems.Add(\"CPU\", compute.CPU.String()+sidecarCPUNote)\n\t}\n\tif compute.Mem != nil {\n\t\titems.Add(\"memory\", compute.Mem.ToMiCeilStr()+sidecarMemNote)\n\t}\n\tif compute.GPU > 0 {\n\t\titems.Add(\"GPU\", compute.GPU)\n\t}\n\tif compute.Inf > 0 {\n\t\titems.Add(\"Inf\", compute.Inf)\n\t}\n\n\treturn items.String()\n}\n\nfunc nodeGroupResourcesTable(api *userconfig.API, compute userconfig.Compute, maxMemMap map[string]kresource.Quantity) string {\n\tvar skippedNodeGroups []string\n\tvar nodeGroupResourceRows [][]interface{}\n\n\tshowGPU := false\n\tshowInf := false\n\tif compute.GPU > 0 {\n\t\tshowGPU = true\n\t}\n\tif compute.Inf > 0 {\n\t\tshowInf = true\n\t}\n\n\tfor _, ng := range config.ClusterConfig.NodeGroups {\n\t\tnodeCPU, nodeMem, nodeGPU, nodeInf := getNodeCapacity(ng.InstanceType, maxMemMap)\n\t\tif nodeGPU > 0 {\n\t\t\tshowGPU = true\n\t\t}\n\t\tif nodeInf > 0 {\n\t\t\tshowInf = true\n\t\t}\n\n\t\tif api.NodeGroups != nil && !slices.HasString(api.NodeGroups, ng.Name) {\n\t\t\tskippedNodeGroups = append(skippedNodeGroups, ng.Name)\n\t\t} else {\n\t\t\tnodeGroupResourceRows = append(nodeGroupResourceRows, []interface{}{ng.Name, ng.InstanceType, nodeCPU, k8s.ToMiFloorStr(nodeMem), nodeGPU, nodeInf})\n\t\t}\n\t}\n\n\tnodeGroupResourceRowsTable := table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{Title: \"node group\"},\n\t\t\t{Title: \"instance type\"},\n\t\t\t{Title: \"CPU\"},\n\t\t\t{Title: \"memory\"},\n\t\t\t{Title: \"GPU\", Hidden: !showGPU},\n\t\t\t{Title: \"Inf\", Hidden: !showInf},\n\t\t},\n\t\tRows: nodeGroupResourceRows,\n\t}\n\n\tout := nodeGroupResourceRowsTable.MustFormat()\n\tif len(skippedNodeGroups) > 0 {\n\t\tout += fmt.Sprintf(\"\\nthe following %s skipped (based on the api configuration's %s field): %s\", s.PluralCustom(\"node group was\", \"node groups were\", len(skippedNodeGroups)), userconfig.NodeGroupsKey, strings.Join(skippedNodeGroups, \", \"))\n\t}\n\n\treturn out\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/batchapi/api.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchapi\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\nconst _batchDashboardUID = \"batchapi\"\n\nfunc UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) {\n\tprevVirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiConfig.Name))\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tinitialDeploymentTime := time.Now().UnixNano()\n\tif prevVirtualService != nil && prevVirtualService.Labels[\"initialDeploymentTime\"] != \"\" {\n\t\tvar err error\n\t\tinitialDeploymentTime, err = k8s.ParseInt64Label(prevVirtualService, \"initialDeploymentTime\")\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\t}\n\n\tapi := spec.GetAPISpec(apiConfig, initialDeploymentTime, \"\", config.ClusterConfig.ClusterUID) // Deployment ID not needed for BatchAPI spec\n\n\tif prevVirtualService == nil {\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"upload api spec\")\n\t\t}\n\n\t\terr = applyK8sResources(api, prevVirtualService)\n\t\tif err != nil {\n\t\t\troutines.RunWithPanicHandler(func() {\n\t\t\t\t_ = deleteK8sResources(api.Name)\n\t\t\t})\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"created %s\", api.Resource.UserString()), nil\n\t}\n\n\tif prevVirtualService.Labels[\"specID\"] != api.SpecID {\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"upload api spec\")\n\t\t}\n\n\t\terr = applyK8sResources(api, prevVirtualService)\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"updated %s\", api.Resource.UserString()), nil\n\t}\n\n\treturn api, fmt.Sprintf(\"%s is up to date\", api.Resource.UserString()), nil\n}\n\nfunc DeleteAPI(apiName string, keepCache bool) error {\n\t// best effort deletion, so don't handle error yet\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\treturn deleteK8sResources(apiName)\n\t\t},\n\t\tfunc() error {\n\t\t\tif keepCache {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn deleteS3Resources(apiName)\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc deleteS3Resources(apiName string) error {\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tprefix := filepath.Join(config.ClusterConfig.ClusterUID, \"apis\", apiName)\n\t\t\treturn config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true)\n\t\t},\n\t\tfunc() error {\n\t\t\tprefix := spec.JobAPIPrefix(config.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, apiName)\n\t\t\troutines.RunWithPanicHandler(func() {\n\t\t\t\t_ = config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) // deleting job files may take a while\n\t\t\t})\n\t\t\treturn nil\n\t\t},\n\t)\n}\n\n// GetAllAPIs returns all batch apis, for each API returning the most recently submitted job and all running jobs\nfunc GetAllAPIs(virtualServices []istioclientnetworking.VirtualService, batchJobList []batch.BatchJob) ([]schema.APIResponse, error) {\n\tbatchAPIsMap := map[string]*schema.APIResponse{}\n\n\tjobIDToBatchJobMap := map[string]*batch.BatchJob{}\n\tapiNameToBatchJobsMap := map[string][]*batch.BatchJob{}\n\tfor i, batchJob := range batchJobList {\n\t\tjobIDToBatchJobMap[batchJob.Name] = &batchJobList[i]\n\t\tapiNameToBatchJobsMap[batchJob.Spec.APIName] = append(apiNameToBatchJobsMap[batchJob.Spec.APIName], &batchJobList[i])\n\t}\n\n\tfor i := range virtualServices {\n\t\tapiName := virtualServices[i].Labels[\"apiName\"]\n\t\tmetadata, err := spec.MetadataFromVirtualService(&virtualServices[i])\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"api %s\", apiName))\n\t\t}\n\n\t\tvar jobStatuses []status.BatchJobStatus\n\t\tbatchJobs := apiNameToBatchJobsMap[metadata.Name]\n\n\t\tif len(batchJobs) == 0 {\n\t\t\tjobStates, err := job.GetMostRecentlySubmittedJobStates(metadata.Name, 1, userconfig.BatchAPIKind)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tif len(jobStates) > 0 {\n\t\t\t\tjobStatus, err := getJobStatusFromJobState(jobStates[0])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\tjobStatuses = append(jobStatuses, *jobStatus)\n\t\t\t}\n\t\t} else {\n\t\t\tfor i := range batchJobs {\n\t\t\t\tbatchJob := batchJobs[i]\n\t\t\t\tjobStatus, err := getJobStatusFromBatchJob(*batchJob)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\n\t\t\t\tjobStatuses = append(jobStatuses, *jobStatus)\n\t\t\t}\n\t\t}\n\n\t\tbatchAPIsMap[metadata.Name] = &schema.APIResponse{\n\t\t\tMetadata:         metadata,\n\t\t\tBatchJobStatuses: jobStatuses,\n\t\t}\n\t}\n\n\tbatchAPIList := make([]schema.APIResponse, 0, len(batchAPIsMap))\n\n\tfor _, batchAPI := range batchAPIsMap {\n\t\tbatchAPIList = append(batchAPIList, *batchAPI)\n\t}\n\n\treturn batchAPIList, nil\n}\n\nfunc GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) {\n\tmetadata, err := spec.MetadataFromVirtualService(deployedResource.VirtualService)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tapi, err := operator.DownloadAPISpec(deployedResource.Name, metadata.APIID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tctx := context.Background()\n\tbatchJobList := batch.BatchJobList{}\n\tif err = config.K8s.List(\n\t\tctx, &batchJobList,\n\t\tclient.InNamespace(config.K8s.Namespace),\n\t\tclient.MatchingLabels{\"apiName\": deployedResource.Name},\n\t); err != nil {\n\t\treturn nil, err\n\t}\n\n\tendpoint, err := operator.APIEndpoint(api)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar jobStatuses []status.BatchJobStatus\n\tjobIDSet := strset.New()\n\tfor _, batchJob := range batchJobList.Items {\n\t\tjobStatus, err := getJobStatusFromBatchJob(batchJob)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tjobStatuses = append(jobStatuses, *jobStatus)\n\t\tjobIDSet.Add(batchJob.Name)\n\t}\n\n\tif len(jobStatuses) < 10 {\n\t\tjobStates, err := job.GetMostRecentlySubmittedJobStates(deployedResource.Name, 10+len(jobStatuses), userconfig.BatchAPIKind)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfor _, jobState := range jobStates {\n\t\t\tif jobIDSet.Has(jobState.ID) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tjobIDSet.Add(jobState.ID)\n\n\t\t\tjobStatus, err := getJobStatusFromJobState(jobState)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tif jobStatus != nil {\n\t\t\t\tjobStatuses = append(jobStatuses, *jobStatus)\n\t\t\t\tif len(jobStatuses) == 10 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tdashboardURL := pointer.String(getDashboardURL(api.Name))\n\n\treturn []schema.APIResponse{\n\t\t{\n\t\t\tSpec:             api,\n\t\t\tMetadata:         metadata,\n\t\t\tBatchJobStatuses: jobStatuses,\n\t\t\tEndpoint:         &endpoint,\n\t\t\tDashboardURL:     dashboardURL,\n\t\t},\n\t}, nil\n}\n\nfunc getDashboardURL(apiName string) string {\n\tloadBalancerURL, err := operator.LoadBalancerURL()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\n\tdashboardURL := fmt.Sprintf(\n\t\t\"%s/dashboard/d/%s/batchapi?orgId=1&refresh=30s&var-api_name=%s\",\n\t\tloadBalancerURL, _batchDashboardUID, apiName,\n\t)\n\n\treturn dashboardURL\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/batchapi/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchapi\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrNoS3FilesFound            = \"batchapi.no_s3_files_found\"\n\tErrBatchItemSizeExceedsLimit = \"batchapi.item_size_exceeds_limit\"\n)\n\nfunc ErrorNoS3FilesFound() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoS3FilesFound,\n\t\tMessage: \"no s3 files match search criteria\",\n\t})\n}\n\nfunc ErrorItemSizeExceedsLimit(index int, size int, limit int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrBatchItemSizeExceedsLimit,\n\t\tMessage: fmt.Sprintf(\"item %d has size %d bytes which exceeds the limit (%d bytes)\", index, size, limit),\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/batchapi/job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchapi\n\nimport (\n\t\"context\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\t\"github.com/cortexlabs/yaml\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\nconst _batchJobTTL = 40 * time.Second // Double the duration of the statsd pod monitor\n\nfunc DryRun(submission *schema.BatchJobSubmission) ([]string, error) {\n\terr := validateJobSubmission(submission)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif submission.FilePathLister != nil {\n\t\ts3Files, err := listFilesDryRun(&submission.FilePathLister.S3Lister)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, schema.FilePathListerKey)\n\t\t}\n\n\t\treturn s3Files, nil\n\t}\n\n\tif submission.DelimitedFiles != nil {\n\t\ts3Files, err := listFilesDryRun(&submission.DelimitedFiles.S3Lister)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, schema.DelimitedFilesKey)\n\t\t}\n\n\t\treturn s3Files, nil\n\t}\n\n\treturn nil, nil\n}\n\nfunc SubmitJob(apiName string, submission *schema.BatchJobSubmission) (*spec.BatchJob, error) {\n\terr := validateJobSubmission(submission)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiName))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tapiID := virtualService.Labels[\"apiID\"]\n\tjobID := spec.MonotonicallyDecreasingID()\n\n\tapiSpec, err := operator.DownloadAPISpec(apiName, apiID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobSpec := spec.BatchJob{\n\t\tRuntimeBatchJobConfig: submission.RuntimeBatchJobConfig,\n\t\tJobKey: spec.JobKey{\n\t\t\tAPIName: apiName,\n\t\t\tID:      jobID,\n\t\t\tKind:    userconfig.BatchAPIKind,\n\t\t},\n\t\tAPIID:     apiSpec.ID,\n\t\tStartTime: time.Now(),\n\t}\n\n\terr = uploadJobSpec(&jobSpec)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// upload job payload for enqueuer\n\tpayloadKey := spec.JobPayloadKey(config.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, apiName, jobID)\n\tif err = config.AWS.UploadJSONToS3(submission, config.ClusterConfig.Bucket, payloadKey); err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar jobConfig *string\n\tif submission.Config != nil {\n\t\tjobConfigBytes, err := yaml.Marshal(submission.Config)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tjobConfig = pointer.String(string(jobConfigBytes))\n\t}\n\n\tvar timeout *kmeta.Duration\n\tif submission.Timeout != nil {\n\t\ttimeout = &kmeta.Duration{Duration: time.Duration(*submission.Timeout) * time.Second}\n\t}\n\n\tvar deadLetterQueue *batch.DeadLetterQueueSpec\n\tif submission.SQSDeadLetterQueue != nil {\n\t\tdeadLetterQueue = &batch.DeadLetterQueueSpec{\n\t\t\tARN:             submission.SQSDeadLetterQueue.ARN,\n\t\t\tMaxReceiveCount: int32(submission.SQSDeadLetterQueue.MaxReceiveCount),\n\t\t}\n\t}\n\n\tbatchJob := batch.BatchJob{\n\t\tObjectMeta: kmeta.ObjectMeta{\n\t\t\tName:      jobID,\n\t\t\tNamespace: config.K8s.Namespace,\n\t\t\tLabels: map[string]string{\n\t\t\t\t\"apiName\":        apiName,\n\t\t\t\t\"apiID\":          apiID,\n\t\t\t\t\"specID\":         virtualService.Labels[\"specID\"],\n\t\t\t\t\"apiKind\":        userconfig.BatchAPIKind.String(),\n\t\t\t\t\"cortex.dev/api\": \"true\",\n\t\t\t},\n\t\t},\n\t\tSpec: batch.BatchJobSpec{\n\t\t\tAPIName:         apiName,\n\t\t\tAPIID:           apiID,\n\t\t\tWorkers:         int32(submission.Workers),\n\t\t\tConfig:          jobConfig,\n\t\t\tTimeout:         timeout,\n\t\t\tDeadLetterQueue: deadLetterQueue,\n\t\t\tTTL:             &kmeta.Duration{Duration: _batchJobTTL},\n\t\t\tNodeGroups:      apiSpec.NodeGroups,\n\t\t\tProbes:          workloads.GetReadinessProbesFromContainers(apiSpec.Pod.Containers),\n\t\t},\n\t}\n\n\tctx := context.Background()\n\tif err = config.K8s.Create(ctx, &batchJob); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &jobSpec, nil\n}\n\nfunc StopJob(jobKey spec.JobKey) error {\n\treturn config.K8s.Delete(context.Background(), &batch.BatchJob{\n\t\tObjectMeta: kmeta.ObjectMeta{Name: jobKey.ID, Namespace: config.K8s.Namespace},\n\t})\n}\n\nfunc uploadJobSpec(jobSpec *spec.BatchJob) error {\n\terr := config.AWS.UploadJSONToS3(jobSpec, config.ClusterConfig.Bucket, jobSpec.SpecFilePath(config.ClusterConfig.ClusterUID))\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/batchapi/job_status.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n  http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchapi\n\nimport (\n\t\"context\"\n\t\"net/url\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/metrics\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/yaml\"\n\tkerrors \"k8s.io/apimachinery/pkg/api/errors\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\nfunc GetJob(jobKey spec.JobKey) (*schema.BatchJobResponse, error) {\n\tctx := context.Background()\n\tvar batchJob batch.BatchJob\n\n\terr := config.K8s.Get(ctx, client.ObjectKey{Name: jobKey.ID, Namespace: config.K8s.Namespace}, &batchJob)\n\tif err != nil && !kerrors.IsNotFound(err) {\n\t\treturn nil, err\n\t}\n\n\tif kerrors.IsNotFound(err) {\n\t\treturn getJobFromS3(jobKey)\n\t}\n\n\treturn getJobFromCluster(batchJob)\n}\n\nfunc getJobFromS3(jobKey spec.JobKey) (*schema.BatchJobResponse, error) {\n\tjobState, err := job.GetJobState(jobKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobStatus, err := getJobStatusFromJobState(jobState)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar jobMetrics *metrics.BatchMetrics\n\tif _, ok := jobState.LastUpdatedMap[spec.MetricsFileKey]; ok && jobState.Status.IsCompleted() {\n\t\tjobMetrics, err = readMetricsFromS3(jobState.JobKey)\n\t\tif err != nil {\n\t\t\ttelemetry.Error(err)\n\t\t}\n\t}\n\n\tif jobMetrics == nil {\n\t\t// try to get metrics from prometheus if they aren't available in S3 because there might be a delay\n\t\tjobMetrics, err = batch.GetMetrics(config.Prometheus, jobStatus.JobKey, time.Now())\n\t\tif err != nil {\n\t\t\ttelemetry.Error(err)\n\t\t}\n\t}\n\n\tapiSpec, err := operator.DownloadAPISpec(jobStatus.APIName, jobStatus.APIID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tendpoint, err := getJobEndpoint(apiSpec, jobKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &schema.BatchJobResponse{\n\t\tAPISpec:   *apiSpec,\n\t\tJobStatus: *jobStatus,\n\t\tMetrics:   jobMetrics,\n\t\tEndpoint:  endpoint,\n\t}, nil\n}\n\nfunc getJobFromCluster(batchJob batch.BatchJob) (*schema.BatchJobResponse, error) {\n\tjobStatus, err := getJobStatusFromBatchJob(batchJob)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobMetrics, err := batch.GetMetrics(config.Prometheus, jobStatus.JobKey, time.Now())\n\tif err != nil {\n\t\ttelemetry.Error(err)\n\t}\n\n\tapiSpec, err := operator.DownloadAPISpec(jobStatus.APIName, jobStatus.APIID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tendpoint, err := getJobEndpoint(apiSpec, jobStatus.JobKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &schema.BatchJobResponse{\n\t\tAPISpec:   *apiSpec,\n\t\tJobStatus: *jobStatus,\n\t\tMetrics:   jobMetrics,\n\t\tEndpoint:  endpoint,\n\t}, nil\n}\n\nfunc getJobStatusFromBatchJob(batchJob batch.BatchJob) (*status.BatchJobStatus, error) {\n\tjobKey := spec.JobKey{\n\t\tID:      batchJob.Name,\n\t\tAPIName: batchJob.Spec.APIName,\n\t\tKind:    userconfig.BatchAPIKind,\n\t}\n\n\tvar deadLetterQueue *spec.SQSDeadLetterQueue\n\tif batchJob.Spec.DeadLetterQueue != nil {\n\t\tdeadLetterQueue = &spec.SQSDeadLetterQueue{\n\t\t\tARN:             batchJob.Spec.DeadLetterQueue.ARN,\n\t\t\tMaxReceiveCount: int(batchJob.Spec.DeadLetterQueue.MaxReceiveCount),\n\t\t}\n\t}\n\n\tvar jobConfig map[string]interface{}\n\tif batchJob.Spec.Config != nil {\n\t\tif err := yaml.Unmarshal([]byte(*batchJob.Spec.Config), &jobConfig); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tvar timeout *int\n\tif batchJob.Spec.Timeout != nil {\n\t\ttimeout = pointer.Int(int(batchJob.Spec.Timeout.Seconds()))\n\t}\n\n\tjobStatus := status.BatchJobStatus{\n\t\tBatchJob: spec.BatchJob{\n\t\t\tJobKey: jobKey,\n\t\t\tRuntimeBatchJobConfig: spec.RuntimeBatchJobConfig{\n\t\t\t\tWorkers:            int(batchJob.Spec.Workers),\n\t\t\t\tSQSDeadLetterQueue: deadLetterQueue,\n\t\t\t\tConfig:             jobConfig,\n\t\t\t\tTimeout:            timeout,\n\t\t\t},\n\t\t\tAPIID:           batchJob.Spec.APIID,\n\t\t\tStartTime:       batchJob.CreationTimestamp.Time,\n\t\t\tSQSUrl:          batchJob.Status.QueueURL,\n\t\t\tTotalBatchCount: batchJob.Status.TotalBatchCount,\n\t\t},\n\t\tWorkerCounts: batchJob.Status.WorkerCounts,\n\t\tStatus:       batchJob.Status.Status,\n\t}\n\n\tif batchJob.Status.EndTime != nil {\n\t\tjobStatus.EndTime = &batchJob.Status.EndTime.Time\n\t}\n\n\tqueueMetrics, err := getQueueMetrics(jobKey)\n\tif aws.IsNonExistentQueueErr(err) {\n\t\tjobStatus.BatchesInQueue = 0\n\t\tjobStatus.TotalBatchCount = 0\n\t} else if err != nil {\n\t\treturn nil, err\n\t} else {\n\t\tjobStatus.BatchesInQueue = queueMetrics.TotalUserMessages()\n\n\t\tif batchJob.Status.Status == status.JobEnqueuing {\n\t\t\tjobStatus.TotalBatchCount = queueMetrics.TotalUserMessages()\n\t\t}\n\t}\n\n\tjobStatus.WorkerCounts = batchJob.Status.WorkerCounts\n\n\treturn &jobStatus, nil\n}\n\nfunc getJobStatusFromJobState(jobState *job.State) (*status.BatchJobStatus, error) {\n\tjobKey := jobState.JobKey\n\n\tjobSpec, err := operator.DownloadBatchJobSpec(jobKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobStatus := status.BatchJobStatus{\n\t\tBatchJob: *jobSpec,\n\t\tEndTime:  jobState.EndTime,\n\t\tStatus:   jobState.Status,\n\t}\n\n\treturn &jobStatus, nil\n}\n\nfunc readMetricsFromS3(jobKey spec.JobKey) (*metrics.BatchMetrics, error) {\n\ts3Key := spec.JobMetricsKey(config.ClusterConfig.ClusterUID, userconfig.BatchAPIKind, jobKey.APIName, jobKey.ID)\n\tbatchMetrics := metrics.BatchMetrics{}\n\terr := config.AWS.ReadJSONFromS3(&batchMetrics, config.ClusterConfig.Bucket, s3Key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &batchMetrics, nil\n}\n\nfunc getJobEndpoint(apiSpec *spec.API, jobKey spec.JobKey) (string, error) {\n\tendpoint, err := operator.APIEndpoint(apiSpec)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tparsedURL, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tq := parsedURL.Query()\n\tq.Add(\"jobID\", jobKey.ID)\n\tparsedURL.RawQuery = q.Encode()\n\n\treturn parsedURL.String(), nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/batchapi/k8s_specs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchapi\n\nimport (\n\t\"context\"\n\t\"path\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\nconst _operatorService = \"operator\"\n\nfunc virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService {\n\treturn k8s.VirtualService(&k8s.VirtualServiceSpec{\n\t\tName:     workloads.K8sName(api.Name),\n\t\tGateways: []string{\"apis-gateway\"},\n\t\tDestinations: []k8s.Destination{{\n\t\t\tServiceName: _operatorService,\n\t\t\tWeight:      100,\n\t\t\tPort:        uint32(consts.ProxyPortInt32),\n\t\t}},\n\t\tPrefixPath:  api.Networking.Endpoint,\n\t\tRewrite:     pointer.String(path.Join(\"batch\", api.Name)),\n\t\tAnnotations: api.ToK8sAnnotations(),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":               api.Name,\n\t\t\t\"apiID\":                 api.ID,\n\t\t\t\"specID\":                api.SpecID,\n\t\t\t\"podID\":                 api.PodID,\n\t\t\t\"initialDeploymentTime\": s.Int64(api.InitialDeploymentTime),\n\t\t\t\"apiKind\":               api.Kind.String(),\n\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t},\n\t})\n}\n\nfunc applyK8sResources(api *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error {\n\tnewVirtualService := virtualServiceSpec(api)\n\n\tif prevVirtualService == nil {\n\t\t_, err := config.K8s.CreateVirtualService(newVirtualService)\n\t\treturn err\n\t}\n\n\t_, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService)\n\treturn err\n}\n\nfunc deleteK8sResources(apiName string) error {\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\terr := config.K8s.DeleteAllOf(\n\t\t\t\tcontext.Background(),\n\t\t\t\t&batch.BatchJob{},\n\t\t\t\tclient.InNamespace(config.K8s.Namespace),\n\t\t\t\tclient.MatchingLabels{\"apiName\": apiName},\n\t\t\t)\n\t\t\treturn client.IgnoreNotFound(err)\n\t\t},\n\t\tfunc() error {\n\t\t\t_, err := config.K8s.DeleteVirtualService(workloads.K8sName(apiName))\n\t\t\treturn err\n\t\t},\n\t)\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/batchapi/queue.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchapi\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/metrics\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n)\n\nfunc apiQueueNamePrefix(apiName string) string {\n\t// <sqs_prefix>_b_<api_name>_\n\treturn config.ClusterConfig.SQSNamePrefix() + \"b\" + clusterconfig.SQSQueueDelimiter +\n\t\tapiName + clusterconfig.SQSQueueDelimiter\n}\n\n// QueueName is cx_<hash of cluster name>_b_<api_name>_<job_id>.fifo\nfunc getJobQueueName(jobKey spec.JobKey) string {\n\treturn apiQueueNamePrefix(jobKey.APIName) + jobKey.ID + \".fifo\"\n}\n\nfunc getJobQueueURL(jobKey spec.JobKey) (string, error) {\n\toperatorAccountID, _, err := config.AWS.GetCachedAccountID()\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, \"failed to construct queue url\", \"unable to get account id\")\n\t}\n\n\treturn fmt.Sprintf(\"https://sqs.%s.amazonaws.com/%s/%s\", config.AWS.Region, operatorAccountID, getJobQueueName(jobKey)), nil\n}\n\nfunc getQueueMetrics(jobKey spec.JobKey) (*metrics.QueueMetrics, error) {\n\tqueueURL, err := getJobQueueURL(jobKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn getQueueMetricsFromURL(queueURL)\n}\n\nfunc getQueueMetricsFromURL(queueURL string) (*metrics.QueueMetrics, error) {\n\tattributes, err := config.AWS.GetAllQueueAttributes(queueURL)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"failed to get queue metrics\")\n\t}\n\n\tqMetrics := metrics.QueueMetrics{}\n\tparsedInt, ok := s.ParseInt(attributes[\"ApproximateNumberOfMessages\"])\n\tif ok {\n\t\tqMetrics.Visible = parsedInt\n\t}\n\n\tparsedInt, ok = s.ParseInt(attributes[\"ApproximateNumberOfMessagesNotVisible\"])\n\tif ok {\n\t\tqMetrics.NotVisible = parsedInt\n\t}\n\n\treturn &qMetrics, nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/batchapi/s3_iterator.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchapi\n\nimport (\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/gobwas/glob\"\n)\n\n// Takes in a function(shouldSkip, bucketName, s3.Object)\nfunc s3IteratorFromLister(s3Lister schema.S3Lister, fn func(string, *s3.Object) (bool, error)) (int64, error) {\n\tincludeGlobPatterns := make([]glob.Glob, 0, len(s3Lister.Includes))\n\n\tfor _, includePattern := range s3Lister.Includes {\n\t\tglobExpression, err := glob.Compile(includePattern, '/')\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, \"failed to interpret glob pattern\", includePattern)\n\t\t}\n\t\tincludeGlobPatterns = append(includeGlobPatterns, globExpression)\n\t}\n\n\texcludeGlobPatterns := make([]glob.Glob, 0, len(s3Lister.Excludes))\n\tfor _, excludePattern := range s3Lister.Excludes {\n\t\tglobExpression, err := glob.Compile(excludePattern, '/')\n\t\tif err != nil {\n\t\t\treturn 0, errors.Wrap(err, \"failed to interpret glob pattern\", excludePattern)\n\t\t}\n\t\texcludeGlobPatterns = append(excludeGlobPatterns, globExpression)\n\t}\n\n\tvar numResults int64\n\n\tfor _, s3Path := range s3Lister.S3Paths {\n\t\tbucket, key, err := aws.SplitS3Path(s3Path)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\tawsClientForBucket, err := aws.NewFromClientS3Path(s3Path, config.AWS)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\terr = awsClientForBucket.S3Iterator(bucket, key, false, nil, nil, func(s3Obj *s3.Object) (bool, error) {\n\t\t\ts3FilePath := aws.S3Path(bucket, *s3Obj.Key)\n\n\t\t\tshouldSkip := false\n\t\t\tif len(includeGlobPatterns) > 0 {\n\t\t\t\tshouldSkip = true\n\t\t\t\tfor _, includeGlobPattern := range includeGlobPatterns {\n\t\t\t\t\tif includeGlobPattern.Match(s3FilePath) {\n\t\t\t\t\t\tshouldSkip = false\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor _, excludeGlobPattern := range excludeGlobPatterns {\n\t\t\t\tif excludeGlobPattern.Match(s3FilePath) {\n\t\t\t\t\tshouldSkip = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif !shouldSkip {\n\t\t\t\tshouldContinue, err := fn(bucket, s3Obj)\n\t\t\t\tnumResults++\n\t\t\t\tif s3Lister.MaxResults != nil && numResults >= *s3Lister.MaxResults {\n\t\t\t\t\tshouldContinue = false\n\t\t\t\t}\n\t\t\t\treturn shouldContinue, err\n\t\t\t}\n\n\t\t\treturn true, nil\n\t\t})\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\tif s3Lister.MaxResults != nil && numResults >= *s3Lister.MaxResults {\n\t\t\treturn numResults, nil\n\t\t}\n\t}\n\n\treturn numResults, nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/batchapi/validations.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage batchapi\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\tawslib \"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\tcr \"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/gobwas/glob\"\n)\n\nconst (\n\t_messageSizeLimit = 250 * 1024 // normally its 256 * 1024 but reserve 6k for message attributes\n)\n\nfunc validateJobSubmissionSchema(submission *schema.BatchJobSubmission) error {\n\tprovidedKeys := []string{}\n\tif submission.ItemList != nil {\n\t\tprovidedKeys = append(providedKeys, schema.ItemListKey)\n\t}\n\tif submission.FilePathLister != nil {\n\t\tprovidedKeys = append(providedKeys, schema.FilePathListerKey)\n\t}\n\tif submission.DelimitedFiles != nil {\n\t\tprovidedKeys = append(providedKeys, schema.DelimitedFilesKey)\n\t}\n\n\tif len(providedKeys) == 0 {\n\t\treturn job.ErrorSpecifyExactlyOneKey(schema.ItemListKey, schema.FilePathListerKey, schema.DelimitedFilesKey)\n\t}\n\n\tif len(providedKeys) > 1 {\n\t\treturn job.ErrorConflictingFields(providedKeys[0], providedKeys[1:]...)\n\t}\n\n\tif submission.ItemList != nil {\n\t\tif len(submission.ItemList.Items) == 0 {\n\t\t\treturn errors.Wrap(cr.ErrorTooFewElements(1), schema.ItemsKey)\n\t\t}\n\n\t\tfor i, batch := range submission.ItemList.Items {\n\t\t\tif len(batch) > _messageSizeLimit {\n\t\t\t\treturn ErrorItemSizeExceedsLimit(i, len(batch), _messageSizeLimit)\n\t\t\t}\n\t\t}\n\n\t\tif submission.ItemList.BatchSize < 1 {\n\t\t\treturn errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.ItemList.BatchSize, 1), schema.ItemListKey, schema.BatchSizeKey)\n\t\t}\n\t}\n\n\tif submission.FilePathLister != nil {\n\t\tif submission.FilePathLister.BatchSize < 1 {\n\t\t\treturn errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.FilePathLister.BatchSize, 1), schema.FilePathListerKey, schema.BatchSizeKey)\n\t\t}\n\t}\n\n\tif submission.DelimitedFiles != nil {\n\t\tif submission.DelimitedFiles.BatchSize < 1 {\n\t\t\treturn errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.DelimitedFiles.BatchSize, 1), schema.DelimitedFilesKey, schema.BatchSizeKey)\n\t\t}\n\t}\n\n\tif submission.Workers <= 0 {\n\t\treturn errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.Workers, 1), schema.WorkersKey)\n\t}\n\n\tif submission.Timeout != nil && *submission.Timeout <= 0 {\n\t\treturn errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.Timeout, 1), schema.TimeoutKey)\n\t}\n\n\tif submission.SQSDeadLetterQueue != nil {\n\t\tif len(submission.SQSDeadLetterQueue.ARN) == 0 {\n\t\t\treturn errors.Wrap(cr.ErrorCannotBeEmpty(), schema.SQSDeadLetterQueueKey, schema.ARNKey)\n\t\t}\n\t\tif submission.SQSDeadLetterQueue.MaxReceiveCount < 1 {\n\t\t\treturn errors.Wrap(cr.ErrorMustBeGreaterThanOrEqualTo(submission.SQSDeadLetterQueue.MaxReceiveCount, 1), schema.SQSDeadLetterQueueKey, schema.MaxReceiveCountKey)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc validateJobSubmission(submission *schema.BatchJobSubmission) error {\n\terr := validateJobSubmissionSchema(submission)\n\tif err != nil {\n\t\treturn errors.Append(err, fmt.Sprintf(\"\\n\\njob submission schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t}\n\n\tif submission.FilePathLister != nil {\n\t\terr := validateS3Lister(&submission.FilePathLister.S3Lister)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, schema.FilePathListerKey)\n\t\t}\n\t}\n\n\tif submission.DelimitedFiles != nil {\n\t\terr := validateS3Lister(&submission.DelimitedFiles.S3Lister)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, schema.DelimitedFilesKey)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc validateS3Lister(s3Lister *schema.S3Lister) error {\n\tif len(s3Lister.S3Paths) == 0 {\n\t\treturn errors.Wrap(cr.ErrorTooFewElements(1), schema.S3PathsKey)\n\t}\n\n\tfor _, globPattern := range s3Lister.Includes {\n\t\t_, err := glob.Compile(globPattern, '/')\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, schema.IncludesKey, globPattern)\n\t\t}\n\t}\n\n\tfor _, globPattern := range s3Lister.Excludes {\n\t\t_, err := glob.Compile(globPattern, '/')\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, schema.ExcludesKey, globPattern)\n\t\t}\n\t}\n\n\tfor _, s3Path := range s3Lister.S3Paths {\n\t\tif !awslib.IsValidS3Path(s3Path) {\n\t\t\treturn awslib.ErrorInvalidS3Path(s3Path)\n\t\t}\n\t}\n\n\tshortCircuitLister := schema.S3Lister{\n\t\tS3Paths:    s3Lister.S3Paths,\n\t\tIncludes:   s3Lister.Includes,\n\t\tExcludes:   s3Lister.Excludes,\n\t\tMaxResults: pointer.Int64(1),\n\t}\n\tnumResults, err := s3IteratorFromLister(shortCircuitLister, func(objPath string, s3Obj *s3.Object) (bool, error) {\n\t\treturn false, nil\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif numResults == 0 {\n\t\treturn ErrorNoS3FilesFound()\n\t}\n\n\treturn nil\n}\n\nfunc listFilesDryRun(s3Lister *schema.S3Lister) ([]string, error) {\n\tvar s3Files []string\n\tfor _, s3Path := range s3Lister.S3Paths {\n\t\tif !awslib.IsValidS3Path(s3Path) {\n\t\t\treturn nil, awslib.ErrorInvalidS3Path(s3Path)\n\t\t}\n\t}\n\n\t_, err := s3IteratorFromLister(*s3Lister, func(bucket string, s3Obj *s3.Object) (bool, error) {\n\t\ts3Files = append(s3Files, awslib.S3Path(bucket, *s3Obj.Key))\n\t\treturn true, nil\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(s3Files) == 0 {\n\t\treturn nil, ErrorNoS3FilesFound()\n\t}\n\n\treturn s3Files, nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/cache.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage job\n\nimport (\n\t\"path\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\nfunc ListAllInProgressJobKeysByAPI(kind userconfig.Kind, apiName string) ([]spec.JobKey, error) {\n\treturn listAllInProgressJobKeysByAPI(kind, &apiName)\n}\n\nfunc ListAllInProgressJobKeys(kind userconfig.Kind) ([]spec.JobKey, error) {\n\treturn listAllInProgressJobKeysByAPI(kind, nil)\n}\n\nfunc DeleteInProgressFile(jobKey spec.JobKey) error {\n\terr := config.AWS.DeleteS3File(config.ClusterConfig.Bucket, inProgressKey(jobKey))\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc DeleteAllInProgressFilesByAPI(kind userconfig.Kind, apiName string) error {\n\terr := config.AWS.DeleteS3Prefix(config.ClusterConfig.Bucket, allInProgressForAPIKey(kind, apiName), true)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc listAllInProgressJobKeysByAPI(kind userconfig.Kind, apiName *string) ([]spec.JobKey, error) {\n\t_, ok := _jobKinds[kind]\n\tif !ok {\n\t\treturn nil, ErrorInvalidJobKind(kind)\n\t}\n\n\tvar jobPath string\n\tif apiName != nil {\n\t\tjobPath = allInProgressForAPIKey(kind, *apiName)\n\t} else {\n\t\tjobPath = allInProgressKey(kind)\n\t}\n\n\ts3Objects, err := config.AWS.ListS3Dir(config.ClusterConfig.Bucket, jobPath, false, nil, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobKeys := make([]spec.JobKey, 0, len(s3Objects))\n\tfor _, obj := range s3Objects {\n\t\tif obj != nil {\n\t\t\tjobKeys = append(jobKeys, jobKeyFromInProgressKey(*obj.Key))\n\t\t}\n\t}\n\treturn jobKeys, nil\n}\n\nfunc uploadInProgressFile(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, inProgressKey(jobKey))\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// e.g. <cluster_uid>/jobs/<job_api_kind>/in_progress\nfunc allInProgressKey(kind userconfig.Kind) string {\n\treturn path.Join(\n\t\tconfig.ClusterConfig.ClusterUID, _jobsPrefix, kind.String(), _inProgressFilePrefix,\n\t)\n}\n\n// e.g. <cluster_uid>/jobs/<job_api_kind>/in_progress/<api_name>\nfunc allInProgressForAPIKey(kind userconfig.Kind, apiName string) string {\n\treturn path.Join(allInProgressKey(kind), apiName)\n}\n\n// e.g. <cluster_uid>/jobs/<job_api_kind>/in_progress/<api_name>/<job_id>\nfunc inProgressKey(jobKey spec.JobKey) string {\n\treturn path.Join(allInProgressForAPIKey(jobKey.Kind, jobKey.APIName), jobKey.ID)\n}\n\nfunc jobKeyFromInProgressKey(s3Key string) spec.JobKey {\n\tpathSplit := strings.Split(s3Key, \"/\")\n\n\tkind := pathSplit[len(pathSplit)-4]\n\tapiName := pathSplit[len(pathSplit)-2]\n\tjobID := pathSplit[len(pathSplit)-1]\n\n\treturn spec.JobKey{APIName: apiName, ID: jobID, Kind: userconfig.KindFromString(kind)}\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/consts.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage job\n\nimport \"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\nconst (\n\t_jobsPrefix            = \"jobs\"\n\t_inProgressFilePrefix  = \"in_progress\"\n\t_enqueuingLivenessFile = \"enqueuing_liveness\"\n)\n\nvar _jobKinds = map[userconfig.Kind]bool{\n\tuserconfig.TaskAPIKind:  true,\n\tuserconfig.BatchAPIKind: true,\n}\n\nfunc LivenessFile() string {\n\treturn _enqueuingLivenessFile\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage job\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\nconst (\n\tErrInvalidJobKind           = \"job.invalid_kind\"\n\tErrJobNotFound              = \"job.not_found\"\n\tErrJobIsNotInProgress       = \"job.job_is_not_in_progress\"\n\tErrJobHasAlreadyBeenStopped = \"job.job_has_already_been_stopped\"\n\tErrConflictingFields        = \"job.conflicting_fields\"\n\tErrSpecifyExactlyOneKey     = \"job.specify_exactly_one_key\"\n)\n\nfunc ErrorInvalidJobKind(kind userconfig.Kind) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidJobKind,\n\t\tMessage: fmt.Sprintf(\"invalid job kind %s\", kind.String()),\n\t})\n}\n\nfunc ErrorJobNotFound(jobKey spec.JobKey) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrJobNotFound,\n\t\tMessage: fmt.Sprintf(\"unable to find %s job %s\", jobKey.Kind.String(), jobKey.UserString()),\n\t})\n}\n\nfunc ErrorJobIsNotInProgress(kind userconfig.Kind) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrJobIsNotInProgress,\n\t\tMessage: fmt.Sprintf(\"cannot stop %s job because it is not in progress\", kind.String()),\n\t})\n}\n\nfunc ErrorJobHasAlreadyBeenStopped(kind userconfig.Kind) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrJobHasAlreadyBeenStopped,\n\t\tMessage: fmt.Sprintf(\"%s job has already been stopped\", kind.String()),\n\t})\n}\n\nfunc ErrorConflictingFields(key string, keys ...string) error {\n\tallKeys := append([]string{key}, keys...)\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrConflictingFields,\n\t\tMessage: fmt.Sprintf(\"please specify either the %s field (but not more than one at the same time)\", s.StrsOr(allKeys)),\n\t})\n}\n\nfunc ErrorSpecifyExactlyOneKey(key string, keys ...string) error {\n\tallKeys := append([]string{key}, keys...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSpecifyExactlyOneKey,\n\t\tMessage: fmt.Sprintf(\"specify exactly one of the following keys: %s\", s.StrsOr(allKeys)),\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/state.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage job\n\nimport (\n\t\"path\"\n\t\"path/filepath\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\nconst (\n\t_averageFilesPerJobState = 10\n)\n\ntype State struct {\n\tspec.JobKey\n\tStatus         status.JobCode\n\tLastUpdatedMap map[string]time.Time\n\tEndTime        *time.Time\n}\n\nfunc (j State) GetLastUpdated() time.Time {\n\tlastUpdated := time.Time{}\n\n\tfor _, fileLastUpdated := range j.LastUpdatedMap {\n\t\tif lastUpdated.After(fileLastUpdated) {\n\t\t\tlastUpdated = fileLastUpdated\n\t\t}\n\t}\n\n\treturn lastUpdated\n}\n\nfunc (j State) GetFirstCreated() time.Time {\n\tfirstCreated := time.Unix(1<<63-62135596801, 999999999) // Max time\n\n\tfor _, fileLastUpdated := range j.LastUpdatedMap {\n\t\tif firstCreated.After(fileLastUpdated) {\n\t\t\tfirstCreated = fileLastUpdated\n\t\t}\n\t}\n\n\treturn firstCreated\n}\n\n// Doesn't assume only status files are present. The order below matters.\nfunc GetTaskStatusCode(lastUpdatedMap map[string]time.Time) status.JobCode {\n\tif _, ok := lastUpdatedMap[status.JobStopped.String()]; ok {\n\t\treturn status.JobStopped\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobTimedOut.String()]; ok {\n\t\treturn status.JobTimedOut\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobWorkerOOM.String()]; ok {\n\t\treturn status.JobWorkerOOM\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobWorkerError.String()]; ok {\n\t\treturn status.JobWorkerError\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobEnqueueFailed.String()]; ok {\n\t\treturn status.JobEnqueueFailed\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobUnexpectedError.String()]; ok {\n\t\treturn status.JobUnexpectedError\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobCompletedWithFailures.String()]; ok {\n\t\treturn status.JobCompletedWithFailures\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobSucceeded.String()]; ok {\n\t\treturn status.JobSucceeded\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobRunning.String()]; ok {\n\t\treturn status.JobRunning\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobEnqueuing.String()]; ok {\n\t\treturn status.JobEnqueuing\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobPending.String()]; ok {\n\t\treturn status.JobPending\n\t}\n\n\treturn status.JobUnknown\n}\n\nfunc GetBatchStatusCode(lastUpdatedMap map[string]time.Time) status.JobCode {\n\tif _, ok := lastUpdatedMap[status.JobTimedOut.String()]; ok {\n\t\treturn status.JobTimedOut\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobWorkerOOM.String()]; ok {\n\t\treturn status.JobWorkerOOM\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobWorkerError.String()]; ok {\n\t\treturn status.JobWorkerError\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobEnqueueFailed.String()]; ok {\n\t\treturn status.JobEnqueueFailed\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobUnexpectedError.String()]; ok {\n\t\treturn status.JobUnexpectedError\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobCompletedWithFailures.String()]; ok {\n\t\treturn status.JobCompletedWithFailures\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobSucceeded.String()]; ok {\n\t\treturn status.JobSucceeded\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobStopped.String()]; ok {\n\t\treturn status.JobStopped\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobRunning.String()]; ok {\n\t\treturn status.JobRunning\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobEnqueuing.String()]; ok {\n\t\treturn status.JobEnqueuing\n\t}\n\n\tif _, ok := lastUpdatedMap[status.JobPending.String()]; ok {\n\t\treturn status.JobPending\n\t}\n\n\treturn status.JobUnknown\n}\n\nfunc GetJobState(jobKey spec.JobKey) (*State, error) {\n\ts3Objects, err := config.AWS.ListS3Prefix(config.ClusterConfig.Bucket, jobKey.Prefix(config.ClusterConfig.ClusterUID), false, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"failed to get job state\", jobKey.UserString())\n\t}\n\n\tif len(s3Objects) == 0 {\n\t\treturn nil, errors.Wrap(ErrorJobNotFound(jobKey), \"failed to get job state\")\n\t}\n\n\tlastUpdatedMap := map[string]time.Time{}\n\tfor _, object := range s3Objects {\n\t\tlastUpdatedMap[filepath.Base(*object.Key)] = *object.LastModified\n\t}\n\n\tjobState := getJobStateFromFiles(jobKey, lastUpdatedMap)\n\treturn &jobState, nil\n}\n\nfunc getJobStateFromFiles(jobKey spec.JobKey, lastUpdatedFileMap map[string]time.Time) State {\n\tvar statusCode status.JobCode\n\tswitch jobKey.Kind {\n\tcase userconfig.BatchAPIKind:\n\t\tstatusCode = GetBatchStatusCode(lastUpdatedFileMap)\n\tcase userconfig.TaskAPIKind:\n\t\tstatusCode = GetTaskStatusCode(lastUpdatedFileMap)\n\t}\n\n\tvar jobEndTime *time.Time\n\tif statusCode.IsCompleted() {\n\t\tif endTime, ok := lastUpdatedFileMap[statusCode.String()]; ok {\n\t\t\tjobEndTime = &endTime\n\t\t}\n\t}\n\n\treturn State{\n\t\tJobKey:         jobKey,\n\t\tLastUpdatedMap: lastUpdatedFileMap,\n\t\tStatus:         statusCode,\n\t\tEndTime:        jobEndTime,\n\t}\n}\n\nfunc GetMostRecentlySubmittedJobStates(apiName string, count int, kind userconfig.Kind) ([]*State, error) {\n\t// a single job state may include 5 files on average, overshoot the number of files needed\n\tapiPrefix := strings.EnsureSuffix(spec.JobAPIPrefix(config.ClusterConfig.ClusterUID, kind, apiName), \"/\")\n\n\ts3Objects, err := config.AWS.ListS3Prefix(\n\t\tconfig.ClusterConfig.Bucket,\n\t\tapiPrefix,\n\t\tfalse,\n\t\tpointer.Int64(int64(count*_averageFilesPerJobState)),\n\t\tnil,\n\t)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// job id -> file name -> last update timestamp\n\tlastUpdatedMaps := map[string]map[string]time.Time{}\n\tvar jobIDOrder []string\n\tfor _, object := range s3Objects {\n\t\tif object == nil {\n\t\t\tcontinue\n\t\t}\n\t\tfileName := filepath.Base(*object.Key)\n\t\tjobID := filepath.Base(filepath.Dir(*object.Key))\n\t\tif _, ok := lastUpdatedMaps[jobID]; !ok {\n\t\t\tjobIDOrder = append(jobIDOrder, jobID)\n\t\t\tlastUpdatedMaps[jobID] = map[string]time.Time{fileName: *object.LastModified}\n\t\t} else {\n\t\t\tlastUpdatedMaps[jobID][fileName] = *object.LastModified\n\t\t}\n\t}\n\n\tjobStates := make([]*State, 0, count)\n\n\tjobStateCount := 0\n\tfor _, jobID := range jobIDOrder {\n\n\t\t// it is possible to have fragmented deletes, spec.json should always be there\n\t\t_, found := lastUpdatedMaps[jobID][\"spec.json\"]\n\t\tif !found {\n\t\t\tgo config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, path.Join(apiPrefix, jobID), true)\n\t\t\tcontinue\n\t\t}\n\n\t\tjobState := getJobStateFromFiles(spec.JobKey{\n\t\t\tAPIName: apiName,\n\t\t\tID:      jobID,\n\t\t\tKind:    kind,\n\t\t}, lastUpdatedMaps[jobID])\n\t\tjobStates = append(jobStates, &jobState)\n\n\t\tjobStateCount++\n\t\tif jobStateCount == count {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn jobStates, nil\n}\n\nfunc SetStatusForJob(jobKey spec.JobKey, jobStatus status.JobCode) error {\n\tswitch jobStatus {\n\tcase status.JobEnqueuing:\n\t\treturn SetEnqueuingStatus(jobKey)\n\tcase status.JobRunning:\n\t\treturn SetRunningStatus(jobKey)\n\tcase status.JobEnqueueFailed:\n\t\treturn SetEnqueueFailedStatus(jobKey)\n\tcase status.JobCompletedWithFailures:\n\t\treturn SetCompletedWithFailuresStatus(jobKey)\n\tcase status.JobSucceeded:\n\t\treturn SetSucceededStatus(jobKey)\n\tcase status.JobUnexpectedError:\n\t\treturn SetUnexpectedErrorStatus(jobKey)\n\tcase status.JobWorkerError:\n\t\treturn SetWorkerErrorStatus(jobKey)\n\tcase status.JobWorkerOOM:\n\t\treturn SetWorkerOOMStatus(jobKey)\n\tcase status.JobTimedOut:\n\t\treturn SetTimedOutStatus(jobKey)\n\tcase status.JobStopped:\n\t\treturn SetStoppedStatus(jobKey)\n\t}\n\treturn nil\n}\n\nfunc UpdateLiveness(jobKey spec.JobKey) error {\n\ts3Key := path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), _enqueuingLivenessFile)\n\terr := config.AWS.UploadJSONToS3(time.Now(), config.ClusterConfig.Bucket, s3Key)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to update liveness\", jobKey.UserString())\n\t}\n\treturn nil\n}\n\nfunc SetEnqueuingStatus(jobKey spec.JobKey) error {\n\terr := UpdateLiveness(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobEnqueuing.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = uploadInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetFailedStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobEnqueueFailed.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetRunningStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobRunning.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = uploadInProgressFile(jobKey) // in progress file should already be there but just in case\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetStoppedStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobStopped.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetSucceededStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobSucceeded.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetCompletedWithFailuresStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobCompletedWithFailures.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetWorkerErrorStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobWorkerError.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetWorkerOOMStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobWorkerOOM.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetEnqueueFailedStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobEnqueueFailed.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetUnexpectedErrorStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobUnexpectedError.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc SetTimedOutStatus(jobKey spec.JobKey) error {\n\terr := config.AWS.UploadStringToS3(\"\", config.ClusterConfig.Bucket, path.Join(jobKey.Prefix(config.ClusterConfig.ClusterUID), status.JobTimedOut.String()))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = DeleteInProgressFile(jobKey)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/taskapi/api.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage taskapi\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nconst _taskDashboardUID = \"taskapi\"\n\n// UpdateAPI deploys or update a task api without triggering any task\nfunc UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) {\n\tprevVirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiConfig.Name))\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tinitialDeploymentTime := time.Now().UnixNano()\n\tif prevVirtualService != nil && prevVirtualService.Labels[\"initialDeploymentTime\"] != \"\" {\n\t\tvar err error\n\t\tinitialDeploymentTime, err = k8s.ParseInt64Label(prevVirtualService, \"initialDeploymentTime\")\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\t}\n\n\tapi := spec.GetAPISpec(apiConfig, initialDeploymentTime, \"\", config.ClusterConfig.ClusterUID) // Deployment ID not needed for TaskAPI spec\n\n\tif prevVirtualService == nil {\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"upload api spec\")\n\t\t}\n\n\t\terr = applyK8sResources(api, prevVirtualService)\n\t\tif err != nil {\n\t\t\troutines.RunWithPanicHandler(func() {\n\t\t\t\tdeleteK8sResources(api.Name)\n\t\t\t})\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"created %s\", api.Resource.UserString()), nil\n\t}\n\n\tif prevVirtualService.Labels[\"specID\"] != api.SpecID {\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"upload api spec\")\n\t\t}\n\n\t\terr = applyK8sResources(api, prevVirtualService)\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"updated %s\", api.Resource.UserString()), nil\n\t}\n\n\treturn api, fmt.Sprintf(\"%s is up to date\", api.Resource.UserString()), nil\n}\n\n// DeleteAPI deletes a task api\nfunc DeleteAPI(apiName string, keepCache bool) error {\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\treturn deleteK8sResources(apiName)\n\t\t},\n\t\tfunc() error {\n\t\t\tif keepCache {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn deleteS3Resources(apiName)\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc deleteS3Resources(apiName string) error {\n\t_ = job.DeleteAllInProgressFilesByAPI(userconfig.TaskAPIKind, apiName) // not useful xml error is thrown, swallow the error\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tprefix := filepath.Join(config.ClusterConfig.ClusterUID, \"apis\", apiName)\n\t\t\treturn config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true)\n\t\t},\n\t\tfunc() error {\n\t\t\tprefix := spec.JobAPIPrefix(config.ClusterConfig.ClusterUID, userconfig.TaskAPIKind, apiName)\n\t\t\tgo func() {\n\t\t\t\t_ = config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true) // deleting job files may take a while\n\t\t\t}()\n\t\t\treturn nil\n\t\t},\n\t)\n}\n\n// GetAllAPIs returns all task APIs, for each API returning the most recently submitted job and all running jobs\nfunc GetAllAPIs(virtualServices []istioclientnetworking.VirtualService, k8sJobs []kbatch.Job, pods []kcore.Pod) ([]schema.APIResponse, error) {\n\ttaskAPIsMap := map[string]*schema.APIResponse{}\n\n\tjobIDToK8sJobMap := map[string]*kbatch.Job{}\n\tfor i, kJob := range k8sJobs {\n\t\tjobIDToK8sJobMap[kJob.Labels[\"jobID\"]] = &k8sJobs[i]\n\t}\n\n\tjobIDToPodsMap := map[string][]kcore.Pod{}\n\tfor _, pod := range pods {\n\t\tif pod.Labels[\"jobID\"] != \"\" {\n\t\t\tjobIDToPodsMap[pod.Labels[\"jobID\"]] = append(jobIDToPodsMap[pod.Labels[\"jobID\"]], pod)\n\t\t}\n\t}\n\n\tfor i := range virtualServices {\n\t\tapiName := virtualServices[i].Labels[\"apiName\"]\n\n\t\tmetadata, err := spec.MetadataFromVirtualService(&virtualServices[i])\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"api %s\", apiName))\n\t\t}\n\n\t\tjobStates, err := job.GetMostRecentlySubmittedJobStates(metadata.Name, 1, userconfig.TaskAPIKind)\n\n\t\tjobStatuses := []status.TaskJobStatus{}\n\t\tif len(jobStates) > 0 {\n\t\t\tjobStatus, err := getJobStatusFromJobState(jobStates[0], jobIDToK8sJobMap[jobStates[0].ID], jobIDToPodsMap[jobStates[0].ID])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tjobStatuses = append(jobStatuses, *jobStatus)\n\t\t}\n\n\t\ttaskAPIsMap[metadata.Name] = &schema.APIResponse{\n\t\t\tMetadata:        metadata,\n\t\t\tTaskJobStatuses: jobStatuses,\n\t\t}\n\t}\n\n\tinProgressJobKeys, err := job.ListAllInProgressJobKeys(userconfig.TaskAPIKind)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, jobKey := range inProgressJobKeys {\n\t\talreadyAdded := false\n\t\tfor _, jobStatus := range taskAPIsMap[jobKey.APIName].TaskJobStatuses {\n\t\t\tif jobStatus.ID == jobKey.ID {\n\t\t\t\talreadyAdded = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif alreadyAdded {\n\t\t\tcontinue\n\t\t}\n\n\t\tjobStatus, err := getJobStatusFromK8sJob(jobKey, jobIDToK8sJobMap[jobKey.ID], jobIDToPodsMap[jobKey.ID])\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif jobStatus.Status.IsInProgress() {\n\t\t\ttaskAPIsMap[jobKey.APIName].TaskJobStatuses = append(taskAPIsMap[jobKey.APIName].TaskJobStatuses, *jobStatus)\n\t\t}\n\t}\n\n\ttaskAPIList := make([]schema.APIResponse, 0, len(taskAPIsMap))\n\n\tfor _, taskAPI := range taskAPIsMap {\n\t\ttaskAPIList = append(taskAPIList, *taskAPI)\n\t}\n\n\treturn taskAPIList, nil\n}\n\n// GetAPIByName returns a single task API and its most recently submitted job along with all running task jobs\nfunc GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) {\n\tmetadata, err := spec.MetadataFromVirtualService(deployedResource.VirtualService)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tapi, err := operator.DownloadAPISpec(deployedResource.Name, metadata.APIID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tk8sJobs, err := config.K8s.ListJobsByLabel(\"apiName\", deployedResource.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobIDToK8sJobMap := map[string]*kbatch.Job{}\n\tfor i, kJob := range k8sJobs {\n\t\tjobIDToK8sJobMap[kJob.Labels[\"jobID\"]] = &k8sJobs[i]\n\t}\n\n\tendpoint, err := operator.APIEndpoint(api)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tpods, err := config.K8s.ListPodsByLabel(\"apiName\", deployedResource.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobIDToPodsMap := map[string][]kcore.Pod{}\n\tfor _, pod := range pods {\n\t\tjobIDToPodsMap[pod.Labels[\"jobID\"]] = append(jobIDToPodsMap[pod.Labels[\"jobID\"]], pod)\n\t}\n\n\tinProgressJobKeys, err := job.ListAllInProgressJobKeysByAPI(userconfig.TaskAPIKind, deployedResource.Name)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobStatuses := []status.TaskJobStatus{}\n\tjobIDSet := strset.New()\n\tfor _, jobKey := range inProgressJobKeys {\n\t\tjobStatus, err := getJobStatusFromK8sJob(jobKey, jobIDToK8sJobMap[jobKey.ID], jobIDToPodsMap[jobKey.ID])\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tjobStatuses = append(jobStatuses, *jobStatus)\n\t\tjobIDSet.Add(jobKey.ID)\n\t}\n\n\tif len(jobStatuses) < 10 {\n\t\tjobStates, err := job.GetMostRecentlySubmittedJobStates(deployedResource.Name, 10+len(jobStatuses), userconfig.TaskAPIKind)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfor _, jobState := range jobStates {\n\t\t\tif jobIDSet.Has(jobState.ID) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tjobIDSet.Add(jobState.ID)\n\n\t\t\tjobStatus, err := getJobStatusFromJobState(jobState, nil, nil)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tjobStatuses = append(jobStatuses, *jobStatus)\n\t\t\tif len(jobStatuses) == 10 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tdashboardURL := pointer.String(getDashboardURL(api.Name))\n\n\treturn []schema.APIResponse{\n\t\t{\n\t\t\tSpec:            api,\n\t\t\tMetadata:        metadata,\n\t\t\tTaskJobStatuses: jobStatuses,\n\t\t\tEndpoint:        &endpoint,\n\t\t\tDashboardURL:    dashboardURL,\n\t\t},\n\t}, nil\n}\n\nfunc getDashboardURL(apiName string) string {\n\tloadBalancerURL, err := operator.LoadBalancerURL()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\n\tdashboardURL := fmt.Sprintf(\n\t\t\"%s/dashboard/d/%s/taskapi?orgId=1&refresh=30s&var-api_name=%s\",\n\t\tloadBalancerURL, _taskDashboardUID, apiName,\n\t)\n\n\treturn dashboardURL\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/taskapi/cron.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage taskapi\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nconst (\n\tManageJobResourcesCronPeriod = 60 * time.Second\n\t_k8sJobExistenceGracePeriod  = 10 * time.Second\n)\n\nvar operatorLogger = logging.GetLogger()\nvar _inProgressJobSpecMap = map[string]*spec.TaskJob{}\n\nfunc ManageJobResources() error {\n\tinProgressJobKeys, err := job.ListAllInProgressJobKeys(userconfig.TaskAPIKind)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tinProgressJobIDSet := strset.Set{}\n\tfor _, jobKey := range inProgressJobKeys {\n\t\tinProgressJobIDSet.Add(jobKey.ID)\n\t}\n\n\tfor jobID := range _inProgressJobSpecMap {\n\t\tif !inProgressJobIDSet.Has(jobID) {\n\t\t\tdelete(_inProgressJobSpecMap, jobID)\n\t\t}\n\t}\n\n\tjobs, err := config.K8s.ListJobs(\n\t\t&kmeta.ListOptions{\n\t\t\tLabelSelector: klabels.SelectorFromSet(\n\t\t\t\tmap[string]string{\"apiKind\": userconfig.TaskAPIKind.String()},\n\t\t\t).String(),\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tk8sJobMap := map[string]kbatch.Job{}\n\tk8sJobIDSet := strset.Set{}\n\tfor _, kJob := range jobs {\n\t\tk8sJobMap[kJob.Labels[\"jobID\"]] = kJob\n\t\tk8sJobIDSet.Add(kJob.Labels[\"jobID\"])\n\t}\n\n\tfor _, jobKey := range inProgressJobKeys {\n\t\tjobLogger, err := operator.GetJobLogger(jobKey)\n\t\tif err != nil {\n\t\t\ttelemetry.Error(err)\n\t\t\toperatorLogger.Error(err)\n\t\t\tcontinue\n\t\t}\n\n\t\tk8sJob, jobFound := k8sJobMap[jobKey.ID]\n\n\t\tjobState, err := job.GetJobState(jobKey)\n\t\tif err != nil {\n\t\t\tjobLogger.Error(err)\n\t\t\tjobLogger.Error(\"terminating job and cleaning up job resources\")\n\t\t\terr := errors.FirstError(\n\t\t\t\tjob.DeleteInProgressFile(jobKey),\n\t\t\t\tdeleteJobRuntimeResources(jobKey),\n\t\t\t\trecordFailure(jobKey),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\ttelemetry.Error(err)\n\t\t\t\toperatorLogger.Error(err)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tif !jobState.Status.IsInProgress() {\n\t\t\t// best effort cleanup\n\t\t\t_ = job.DeleteInProgressFile(jobKey)\n\t\t\t_ = deleteJobRuntimeResources(jobKey)\n\t\t\tcontinue\n\t\t}\n\n\t\t// reconcile job state and k8s job\n\t\tnewStatusCode, msg := reconcileInProgressJob(jobState, jobFound)\n\t\tif err != nil {\n\t\t\ttelemetry.Error(err)\n\t\t\toperatorLogger.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\tif newStatusCode != jobState.Status {\n\t\t\tjobLogger.Error(msg)\n\t\t\terr := job.SetStatusForJob(jobKey, newStatusCode)\n\t\t\tif err != nil {\n\t\t\t\ttelemetry.Error(err)\n\t\t\t\toperatorLogger.Error(err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif _, ok := _inProgressJobSpecMap[jobKey.ID]; !ok {\n\t\t\tjobSpec, err := operator.DownloadTaskJobSpec(jobKey)\n\t\t\tif err != nil {\n\t\t\t\tjobLogger.Error(err)\n\t\t\t\tjobLogger.Error(\"terminating job and cleaning up job resources\")\n\t\t\t\terr := errors.FirstError(\n\t\t\t\t\tjob.DeleteInProgressFile(jobKey),\n\t\t\t\t\tdeleteJobRuntimeResources(jobKey),\n\t\t\t\t\trecordFailure(jobKey),\n\t\t\t\t)\n\t\t\t\tif err != nil {\n\t\t\t\t\ttelemetry.Error(err)\n\t\t\t\t\toperatorLogger.Error(err)\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_inProgressJobSpecMap[jobKey.ID] = jobSpec\n\t\t}\n\t\tjobSpec := _inProgressJobSpecMap[jobKey.ID]\n\n\t\tif jobSpec.Timeout != nil && time.Since(jobSpec.StartTime) > time.Second*time.Duration(*jobSpec.Timeout) {\n\t\t\tjobLogger.Errorf(\"terminating job after exceeding the specified timeout of %d seconds\", *jobSpec.Timeout)\n\t\t\terr := errors.FirstError(\n\t\t\t\tjob.SetTimedOutStatus(jobKey),\n\t\t\t\tdeleteJobRuntimeResources(jobKey),\n\t\t\t\trecordFailure(jobKey),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\ttelemetry.Error(err)\n\t\t\t\toperatorLogger.Error(err)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tif jobState.Status == status.JobRunning {\n\t\t\terr = checkIfJobCompleted(jobKey, jobSpec.StartTime, k8sJob)\n\t\t\tif err != nil {\n\t\t\t\ttelemetry.Error(err)\n\t\t\t\toperatorLogger.Error(err)\n\t\t\t}\n\t\t}\n\t}\n\n\t// existing K8s job but job is not in progress\n\tfor jobID := range strset.Difference(k8sJobIDSet, inProgressJobIDSet) {\n\t\tjobKey := spec.JobKey{\n\t\t\tAPIName: k8sJobMap[jobID].Labels[\"apiName\"],\n\t\t\tID:      k8sJobMap[jobID].Labels[\"jobID\"],\n\t\t}\n\n\t\terr := deleteJobRuntimeResources(jobKey)\n\t\tif err != nil {\n\t\t\ttelemetry.Error(err)\n\t\t\toperatorLogger.Error(err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// verifies k8s job exists for a job in running status, if verification fails return a job code to reflect the state\nfunc reconcileInProgressJob(jobState *job.State, jobFound bool) (status.JobCode, string) {\n\tif jobState.Status == status.JobRunning {\n\t\tif time.Since(jobState.LastUpdatedMap[status.JobRunning.String()]) <= _k8sJobExistenceGracePeriod {\n\t\t\treturn jobState.Status, \"\"\n\t\t}\n\n\t\tif !jobFound { // unexpected k8s job missing\n\t\t\treturn status.JobUnexpectedError, fmt.Sprintf(\"terminating job %s; unable to find kubernetes job\", jobState.JobKey.UserString())\n\t\t}\n\t}\n\n\treturn jobState.Status, \"\"\n}\n\nfunc checkIfJobCompleted(jobKey spec.JobKey, jobStartTime time.Time, k8sJob kbatch.Job) error {\n\tpods, _ := config.K8s.ListPodsByLabel(\"jobID\", jobKey.ID)\n\tfor i := range pods {\n\t\tif k8s.WasPodOOMKilled(&pods[i]) {\n\t\t\treturn errors.FirstError(\n\t\t\t\tjob.SetWorkerOOMStatus(jobKey),\n\t\t\t\tdeleteJobRuntimeResources(jobKey),\n\t\t\t\trecordFailure(jobKey),\n\t\t\t)\n\t\t}\n\t}\n\n\tif int(k8sJob.Status.Failed) == 1 {\n\t\treturn errors.FirstError(\n\t\t\tjob.SetWorkerErrorStatus(jobKey),\n\t\t\tdeleteJobRuntimeResources(jobKey),\n\t\t\trecordFailure(jobKey),\n\t\t)\n\t} else if int(k8sJob.Status.Succeeded) == 1 && len(pods) > 0 {\n\t\treturn errors.FirstError(\n\t\t\tjob.SetSucceededStatus(jobKey),\n\t\t\tdeleteJobRuntimeResources(jobKey),\n\t\t\trecordSuccess(jobKey),\n\t\t\trecordTimePerTask(jobKey, time.Since(jobStartTime)),\n\t\t)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/taskapi/job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage taskapi\n\nimport (\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n)\n\nfunc SubmitJob(apiName string, submission *schema.TaskJobSubmission) (*spec.TaskJob, error) {\n\terr := validateJobSubmission(submission)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiName))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tapiID := virtualService.Labels[\"apiID\"]\n\n\tapiSpec, err := operator.DownloadAPISpec(apiName, apiID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobID := spec.MonotonicallyDecreasingID()\n\n\tjobKey := spec.JobKey{\n\t\tAPIName: apiSpec.Name,\n\t\tID:      jobID,\n\t\tKind:    apiSpec.Kind,\n\t}\n\n\tjobSpec := spec.TaskJob{\n\t\tJobKey:               jobKey,\n\t\tRuntimeTaskJobConfig: submission.RuntimeTaskJobConfig,\n\t\tAPIID:                apiSpec.ID,\n\t\tSpecID:               apiSpec.SpecID,\n\t\tPodID:                apiSpec.PodID,\n\t\tStartTime:            time.Now(),\n\t}\n\n\tif err := uploadJobSpec(&jobSpec); err != nil {\n\t\treturn nil, err\n\t}\n\n\tdeployJob(apiSpec, &jobSpec)\n\n\treturn &jobSpec, nil\n}\n\nfunc uploadJobSpec(jobSpec *spec.TaskJob) error {\n\tif err := config.AWS.UploadJSONToS3(\n\t\tjobSpec, config.ClusterConfig.Bucket, jobSpec.SpecFilePath(config.ClusterConfig.ClusterUID),\n\t); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc deployJob(apiSpec *spec.API, jobSpec *spec.TaskJob) {\n\terr := createJobConfigMap(*apiSpec, *jobSpec)\n\tif err != nil {\n\t\thandleJobSubmissionError(jobSpec.JobKey, err)\n\t}\n\n\terr = createK8sJob(apiSpec, jobSpec)\n\tif err != nil {\n\t\thandleJobSubmissionError(jobSpec.JobKey, err)\n\t}\n\n\terr = job.SetRunningStatus(jobSpec.JobKey)\n\tif err != nil {\n\t\thandleJobSubmissionError(jobSpec.JobKey, err)\n\t}\n}\n\nfunc createJobConfigMap(apiSpec spec.API, jobSpec spec.TaskJob) error {\n\tconfigMapConfig := workloads.ConfigMapConfig{\n\t\tTaskJob: &jobSpec,\n\t}\n\n\tconfigMapData, err := configMapConfig.GenerateConfigMapData()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn createK8sConfigMap(k8sConfigMap(apiSpec, jobSpec, configMapData))\n}\n\nfunc handleJobSubmissionError(jobKey spec.JobKey, jobErr error) {\n\tjobLogger, err := operator.GetJobLogger(jobKey)\n\tif err != nil {\n\t\ttelemetry.Error(err)\n\t\toperatorLogger.Error(err)\n\t\treturn\n\t}\n\tjobLogger.Error(jobErr.Error())\n\terr = errors.FirstError(\n\t\tjob.SetUnexpectedErrorStatus(jobKey),\n\t\tdeleteJobRuntimeResources(jobKey),\n\t)\n\tif err != nil {\n\t\ttelemetry.Error(err)\n\t\terrors.PrintError(err)\n\t}\n}\n\nfunc deleteJobRuntimeResources(jobKey spec.JobKey) error {\n\treturn errors.FirstError(\n\t\tdeleteK8sJob(jobKey),\n\t\tdeleteK8sConfigMap(jobKey),\n\t)\n}\n\nfunc StopJob(jobKey spec.JobKey) error {\n\tjobState, err := job.GetJobState(jobKey)\n\tif err != nil {\n\t\troutines.RunWithPanicHandler(func() {\n\t\t\tdeleteJobRuntimeResources(jobKey)\n\t\t})\n\t\treturn err\n\t}\n\n\tif !jobState.Status.IsInProgress() {\n\t\troutines.RunWithPanicHandler(func() {\n\t\t\tdeleteJobRuntimeResources(jobKey)\n\t\t})\n\t\treturn errors.Wrap(job.ErrorJobIsNotInProgress(jobKey.Kind), jobKey.UserString())\n\t}\n\n\tjobLogger, err := operator.GetJobLogger(jobKey)\n\tif err == nil {\n\t\tjobLogger.Warn(\"request received to stop job; performing cleanup...\")\n\t}\n\n\treturn errors.FirstError(\n\t\tdeleteJobRuntimeResources(jobKey),\n\t\tjob.SetStoppedStatus(jobKey),\n\t)\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/taskapi/job_status.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n  http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage taskapi\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nfunc GetJobStatus(jobKey spec.JobKey) (*status.TaskJobStatus, error) {\n\tjobState, err := job.GetJobState(jobKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tk8sJob, err := config.K8s.GetJob(jobKey.K8sName())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tpods, err := config.K8s.ListPodsByLabels(map[string]string{\"apiName\": jobKey.APIName, \"jobID\": jobKey.ID})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn getJobStatusFromJobState(jobState, k8sJob, pods)\n}\n\nfunc getJobStatusFromK8sJob(jobKey spec.JobKey, k8sJob *kbatch.Job, pods []kcore.Pod) (*status.TaskJobStatus, error) {\n\tjobState, err := job.GetJobState(jobKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn getJobStatusFromJobState(jobState, k8sJob, pods)\n}\n\nfunc getJobStatusFromJobState(jobState *job.State, k8sJob *kbatch.Job, pods []kcore.Pod) (*status.TaskJobStatus, error) {\n\tjobKey := jobState.JobKey\n\n\tjobSpec, err := operator.DownloadTaskJobSpec(jobKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tjobStatus := status.TaskJobStatus{\n\t\tTaskJob: *jobSpec,\n\t\tEndTime: jobState.EndTime,\n\t\tStatus:  jobState.Status,\n\t}\n\n\tif jobState.Status.IsInProgress() && k8sJob != nil {\n\t\tworkerCounts := job.GetWorkerCountsForJob(*k8sJob, pods)\n\t\tjobStatus.WorkerCounts = &workerCounts\n\t}\n\n\treturn &jobStatus, nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/taskapi/k8s_specs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage taskapi\n\nimport (\n\t\"path\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nconst _operatorService = \"operator\"\n\nfunc virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService {\n\treturn k8s.VirtualService(&k8s.VirtualServiceSpec{\n\t\tName:     workloads.K8sName(api.Name),\n\t\tGateways: []string{\"apis-gateway\"},\n\t\tDestinations: []k8s.Destination{{\n\t\t\tServiceName: _operatorService,\n\t\t\tWeight:      100,\n\t\t\tPort:        uint32(consts.ProxyPortInt32),\n\t\t}},\n\t\tPrefixPath:  api.Networking.Endpoint,\n\t\tRewrite:     pointer.String(path.Join(\"tasks\", api.Name)),\n\t\tAnnotations: api.ToK8sAnnotations(),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":               api.Name,\n\t\t\t\"apiID\":                 api.ID,\n\t\t\t\"specID\":                api.SpecID,\n\t\t\t\"podID\":                 api.PodID,\n\t\t\t\"initialDeploymentTime\": s.Int64(api.InitialDeploymentTime),\n\t\t\t\"apiKind\":               api.Kind.String(),\n\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t},\n\t})\n}\n\nfunc k8sJobSpec(api *spec.API, job *spec.TaskJob) *kbatch.Job {\n\tcontainers, volumes := workloads.TaskContainers(*api, &job.JobKey)\n\n\treturn k8s.Job(&k8s.JobSpec{\n\t\tName:        job.JobKey.K8sName(),\n\t\tParallelism: int32(job.Workers),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":        api.Name,\n\t\t\t\"apiID\":          api.ID,\n\t\t\t\"specID\":         api.SpecID,\n\t\t\t\"podID\":          api.PodID,\n\t\t\t\"jobID\":          job.ID,\n\t\t\t\"apiKind\":        api.Kind.String(),\n\t\t\t\"cortex.dev/api\": \"true\",\n\t\t},\n\t\tPodSpec: k8s.PodSpec{\n\t\t\tLabels: map[string]string{\n\t\t\t\t\"apiName\":        api.Name,\n\t\t\t\t\"podID\":          api.PodID,\n\t\t\t\t\"jobID\":          job.ID,\n\t\t\t\t\"apiKind\":        api.Kind.String(),\n\t\t\t\t\"cortex.dev/api\": \"true\",\n\t\t\t},\n\t\t\tAnnotations: map[string]string{\n\t\t\t\t\"traffic.sidecar.istio.io/excludeOutboundIPRanges\": \"0.0.0.0/0\",\n\t\t\t\t\"cluster-autoscaler.kubernetes.io/safe-to-evict\":   \"false\",\n\t\t\t},\n\t\t\tK8sPodSpec: kcore.PodSpec{\n\t\t\t\tRestartPolicy: \"Never\",\n\t\t\t\tInitContainers: []kcore.Container{\n\t\t\t\t\tworkloads.KubexitInitContainer(),\n\t\t\t\t},\n\t\t\t\tContainers:         containers,\n\t\t\t\tNodeSelector:       workloads.NodeSelectors(),\n\t\t\t\tTolerations:        workloads.GenerateResourceTolerations(),\n\t\t\t\tAffinity:           workloads.GenerateNodeAffinities(api.NodeGroups),\n\t\t\t\tVolumes:            volumes,\n\t\t\t\tServiceAccountName: workloads.ServiceAccountName,\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc k8sConfigMap(api spec.API, job spec.TaskJob, configMapData map[string]string) kcore.ConfigMap {\n\treturn *k8s.ConfigMap(&k8s.ConfigMapSpec{\n\t\tName: job.JobKey.K8sName(),\n\t\tData: configMapData,\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":        api.Name,\n\t\t\t\"apiKind\":        api.Kind.String(),\n\t\t\t\"cortex.dev/api\": \"true\",\n\t\t},\n\t})\n}\n\nfunc applyK8sResources(api *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error {\n\tnewVirtualService := virtualServiceSpec(api)\n\n\tif prevVirtualService == nil {\n\t\t_, err := config.K8s.CreateVirtualService(newVirtualService)\n\t\treturn err\n\t}\n\n\t_, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService)\n\treturn err\n}\n\nfunc deleteK8sResources(apiName string) error {\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\t_, err := config.K8s.DeleteJobs(&kmeta.ListOptions{\n\t\t\t\tLabelSelector: klabels.SelectorFromSet(\n\t\t\t\t\tmap[string]string{\n\t\t\t\t\t\t\"apiName\": apiName,\n\t\t\t\t\t\t\"apiKind\": userconfig.TaskAPIKind.String(),\n\t\t\t\t\t}).String(),\n\t\t\t})\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\t_, err := config.K8s.DeleteVirtualService(workloads.K8sName(apiName))\n\t\t\treturn err\n\t\t},\n\t)\n}\n\nfunc deleteK8sJob(jobKey spec.JobKey) error {\n\t_, err := config.K8s.DeleteJobs(&kmeta.ListOptions{\n\t\tLabelSelector: klabels.SelectorFromSet(\n\t\t\tmap[string]string{\n\t\t\t\t\"apiName\": jobKey.APIName,\n\t\t\t\t\"apiKind\": userconfig.TaskAPIKind.String(),\n\t\t\t\t\"jobID\":   jobKey.ID,\n\t\t\t}).String(),\n\t})\n\treturn err\n}\n\nfunc createK8sJob(apiSpec *spec.API, jobSpec *spec.TaskJob) error {\n\tk8sJob := k8sJobSpec(apiSpec, jobSpec)\n\n\t_, err := config.K8s.CreateJob(k8sJob)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc deleteK8sConfigMap(jobKey spec.JobKey) error {\n\t_, err := config.K8s.DeleteConfigMap(jobKey.K8sName())\n\treturn err\n}\n\nfunc createK8sConfigMap(configMap kcore.ConfigMap) error {\n\t_, err := config.K8s.CreateConfigMap(&configMap)\n\treturn err\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/taskapi/metrics.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage taskapi\n\nimport (\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n)\n\nfunc recordSuccess(jobKey spec.JobKey) error {\n\ttags := []string{\n\t\t\"api_name:\" + jobKey.APIName,\n\t\t\"job_id:\" + jobKey.ID,\n\t}\n\terr := config.MetricsClient.Incr(\"cortex_task_succeeded\", tags, 1.0)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\treturn nil\n}\n\nfunc recordFailure(jobKey spec.JobKey) error {\n\ttags := []string{\n\t\t\"api_name:\" + jobKey.APIName,\n\t\t\"job_id:\" + jobKey.ID,\n\t}\n\terr := config.MetricsClient.Incr(\"cortex_task_failed\", tags, 1.0)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\treturn nil\n}\n\nfunc recordTimePerTask(jobKey spec.JobKey, elapsedTime time.Duration) error {\n\ttags := []string{\n\t\t\"api_name:\" + jobKey.APIName,\n\t\t\"job_id:\" + jobKey.ID,\n\t}\n\terr := config.MetricsClient.Histogram(\"cortex_time_per_task\", elapsedTime.Seconds(), tags, 1.0)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/taskapi/validations.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage taskapi\n\nimport (\n\tcr \"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n)\n\nfunc validateJobSubmission(submission *schema.TaskJobSubmission) error {\n\tif submission.Workers != 1 {\n\t\treturn errors.Wrap(cr.ErrorInvalidInt(submission.Workers, 1), schema.WorkersKey)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/job/worker_stats.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage job\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nfunc GetWorkerCountsForJob(k8sJob kbatch.Job, pods []kcore.Pod) status.WorkerCounts {\n\tif k8sJob.Status.Failed > 0 {\n\t\treturn status.WorkerCounts{\n\t\t\tFailed: *k8sJob.Spec.Parallelism, // When one worker fails, the rest of the pods get deleted so you won't be able to get their statuses\n\t\t}\n\t}\n\n\tworkerCounts := status.WorkerCounts{}\n\tfor i := range pods {\n\t\taddPodToWorkerCounts(&pods[i], &workerCounts)\n\t}\n\n\treturn workerCounts\n}\n\nfunc addPodToWorkerCounts(pod *kcore.Pod, workerCounts *status.WorkerCounts) {\n\tif k8s.IsPodReady(pod) {\n\t\tworkerCounts.Ready++\n\t\treturn\n\t}\n\n\tswitch k8s.GetPodStatus(pod) {\n\tcase k8s.PodStatusPending:\n\t\tworkerCounts.Pending++\n\tcase k8s.PodStatusStalled:\n\t\tworkerCounts.Stalled++\n\tcase k8s.PodStatusCreating:\n\t\tworkerCounts.Creating++\n\tcase k8s.PodStatusNotReady:\n\t\tworkerCounts.NotReady++\n\tcase k8s.PodStatusErrImagePull:\n\t\tworkerCounts.ErrImagePull++\n\tcase k8s.PodStatusTerminating:\n\t\tworkerCounts.Terminating++\n\tcase k8s.PodStatusFailed:\n\t\tworkerCounts.Failed++\n\tcase k8s.PodStatusKilled:\n\t\tworkerCounts.Killed++\n\tcase k8s.PodStatusKilledOOM:\n\t\tworkerCounts.KilledOOM++\n\tcase k8s.PodStatusSucceeded:\n\t\tworkerCounts.Succeeded++\n\tcase k8s.PodStatusUnknown:\n\t\tworkerCounts.Unknown++\n\t}\n}\n"
  },
  {
    "path": "pkg/operator/resources/realtimeapi/api.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage realtimeapi\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"sort\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nconst _realtimeDashboardUID = \"realtimeapi\"\n\nfunc generateDeploymentID() string {\n\treturn k8s.RandomName()[:10]\n}\n\nfunc UpdateAPI(apiConfig *userconfig.API, force bool) (*spec.API, string, error) {\n\tprevDeployment, prevService, prevVirtualService, err := getK8sResources(apiConfig.Name)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tinitialDeploymentTime := time.Now().UnixNano()\n\tdeploymentID := generateDeploymentID()\n\tif prevVirtualService != nil && prevVirtualService.Labels[\"initialDeploymentTime\"] != \"\" {\n\t\tvar err error\n\t\tinitialDeploymentTime, err = k8s.ParseInt64Label(prevVirtualService, \"initialDeploymentTime\")\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\t\tdeploymentID = prevVirtualService.Labels[\"deploymentID\"]\n\t}\n\n\tapi := spec.GetAPISpec(apiConfig, initialDeploymentTime, deploymentID, config.ClusterConfig.ClusterUID)\n\n\tif prevDeployment == nil {\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"upload api spec\")\n\t\t}\n\n\t\tif err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil {\n\t\t\troutines.RunWithPanicHandler(func() {\n\t\t\t\t_ = deleteK8sResources(api.Name)\n\t\t\t})\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"creating %s\", api.Resource.UserString()), nil\n\t}\n\n\tif prevVirtualService.Labels[\"specID\"] != api.SpecID || prevVirtualService.Labels[\"deploymentID\"] != api.DeploymentID {\n\t\tisUpdating, err := isAPIUpdating(prevDeployment)\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\t\tif isUpdating && !force {\n\t\t\treturn nil, \"\", ErrorAPIUpdating(api.Name)\n\t\t}\n\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"upload api spec\")\n\t\t}\n\n\t\tif err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\t\treturn api, fmt.Sprintf(\"updating %s\", api.Resource.UserString()), nil\n\t}\n\n\t// deployment didn't change\n\tisUpdating, err := isAPIUpdating(prevDeployment)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\tif isUpdating {\n\t\treturn api, fmt.Sprintf(\"%s is already updating\", api.Resource.UserString()), nil\n\t}\n\treturn api, fmt.Sprintf(\"%s is up to date\", api.Resource.UserString()), nil\n}\n\nfunc RefreshAPI(apiName string, force bool) (string, error) {\n\tprevDeployment, prevService, prevVirtualService, err := getK8sResources(apiName)\n\tif err != nil {\n\t\treturn \"\", err\n\t} else if prevDeployment == nil || prevVirtualService == nil {\n\t\treturn \"\", errors.ErrorUnexpected(\"unable to find deployment\", apiName)\n\t}\n\n\tisUpdating, err := isAPIUpdating(prevDeployment)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif isUpdating && !force {\n\t\treturn \"\", ErrorAPIUpdating(apiName)\n\t}\n\n\tapiID, err := k8s.GetLabel(prevDeployment, \"apiID\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tapi, err := operator.DownloadAPISpec(apiName, apiID)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tinitialDeploymentTime, err := k8s.ParseInt64Label(prevVirtualService, \"initialDeploymentTime\")\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tapi = spec.GetAPISpec(api.API, initialDeploymentTime, generateDeploymentID(), config.ClusterConfig.ClusterUID)\n\n\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\treturn \"\", errors.Wrap(err, \"upload api spec\")\n\t}\n\n\tif err := applyK8sResources(api, prevDeployment, prevService, prevVirtualService); err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn fmt.Sprintf(\"updating %s\", api.Resource.UserString()), nil\n}\n\nfunc DeleteAPI(apiName string, keepCache bool) error {\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\treturn deleteK8sResources(apiName)\n\t\t},\n\t\tfunc() error {\n\t\t\tif keepCache {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\t// best effort deletion, swallow errors because there could be weird error messages\n\t\t\t_ = deleteBucketResources(apiName)\n\t\t\treturn nil\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc GetAllAPIs(deployments []kapps.Deployment) ([]schema.APIResponse, error) {\n\trealtimeAPIs := make([]schema.APIResponse, len(deployments))\n\tmappedRealtimeAPIs := make(map[string]schema.APIResponse, len(deployments))\n\tapiNames := make([]string, len(deployments))\n\n\tfor i := range deployments {\n\t\tapiName := deployments[i].Labels[\"apiName\"]\n\t\tapiNames[i] = apiName\n\n\t\tmetadata, err := spec.MetadataFromDeployment(&deployments[i])\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"api %s\", apiName))\n\t\t}\n\t\tmappedRealtimeAPIs[apiName] = schema.APIResponse{\n\t\t\tStatus:   status.FromDeployment(&deployments[i]),\n\t\t\tMetadata: metadata,\n\t\t}\n\t}\n\n\tsort.Strings(apiNames)\n\tfor i := range apiNames {\n\t\trealtimeAPIs[i] = mappedRealtimeAPIs[apiNames[i]]\n\t}\n\n\treturn realtimeAPIs, nil\n}\n\nfunc GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) {\n\tdeployment, err := config.K8s.GetDeployment(workloads.K8sName(deployedResource.Name))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif deployment == nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to find deployment\", deployedResource.Name)\n\t}\n\n\tapiStatus := status.FromDeployment(deployment)\n\tapiMetadata, err := spec.MetadataFromDeployment(deployment)\n\tif err != nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to obtain metadata\", deployedResource.Name)\n\t}\n\n\tapi, err := operator.DownloadAPISpec(apiMetadata.Name, apiMetadata.APIID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tapiEndpoint, err := operator.APIEndpoint(api)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdashboardURL := pointer.String(getDashboardURL(api.Name))\n\n\treturn []schema.APIResponse{\n\t\t{\n\t\t\tSpec:         api,\n\t\t\tMetadata:     apiMetadata,\n\t\t\tStatus:       apiStatus,\n\t\t\tEndpoint:     &apiEndpoint,\n\t\t\tDashboardURL: dashboardURL,\n\t\t},\n\t}, nil\n}\n\nfunc DescribeAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) {\n\tdeployment, err := config.K8s.GetDeployment(workloads.K8sName(deployedResource.Name))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif deployment == nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to find deployment\", deployedResource.Name)\n\t}\n\n\tapiStatus := status.FromDeployment(deployment)\n\tapiMetadata, err := spec.MetadataFromDeployment(deployment)\n\tif err != nil {\n\t\treturn nil, errors.ErrorUnexpected(\"unable to obtain metadata\", deployedResource.Name)\n\t}\n\n\tpods, err := config.K8s.ListPodsByLabel(\"apiName\", deployment.Labels[\"apiName\"])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tapiStatus.ReplicaCounts = GetReplicaCounts(deployment, pods)\n\n\tapiEndpoint, err := operator.APIEndpointFromResource(deployedResource)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdashboardURL := pointer.String(getDashboardURL(deployedResource.Name))\n\n\treturn []schema.APIResponse{\n\t\t{\n\t\t\tMetadata:     apiMetadata,\n\t\t\tStatus:       apiStatus,\n\t\t\tEndpoint:     &apiEndpoint,\n\t\t\tDashboardURL: dashboardURL,\n\t\t},\n\t}, nil\n}\n\nfunc getK8sResources(apiName string) (*kapps.Deployment, *kcore.Service, *istioclientnetworking.VirtualService, error) {\n\tvar deployment *kapps.Deployment\n\tvar service *kcore.Service\n\tvar virtualService *istioclientnetworking.VirtualService\n\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tdeployment, err = config.K8s.GetDeployment(workloads.K8sName(apiName))\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tservice, err = config.K8s.GetService(workloads.K8sName(apiName))\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tvirtualService, err = config.K8s.GetVirtualService(workloads.K8sName(apiName))\n\t\t\treturn err\n\t\t},\n\t)\n\n\treturn deployment, service, virtualService, err\n}\n\nfunc applyK8sResources(api *spec.API, prevDeployment *kapps.Deployment, prevService *kcore.Service, prevVirtualService *istioclientnetworking.VirtualService) error {\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\treturn applyK8sDeployment(api, prevDeployment)\n\t\t},\n\t\tfunc() error {\n\t\t\treturn applyK8sService(api, prevService)\n\t\t},\n\t\tfunc() error {\n\t\t\treturn applyK8sVirtualService(api, prevVirtualService)\n\t\t},\n\t)\n}\n\nfunc applyK8sDeployment(api *spec.API, prevDeployment *kapps.Deployment) error {\n\tnewDeployment := deploymentSpec(api, prevDeployment)\n\n\tif prevDeployment == nil {\n\t\t_, err := config.K8s.CreateDeployment(newDeployment)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if prevDeployment.Status.ReadyReplicas == 0 {\n\t\t// Delete deployment if it never became ready\n\t\t_, _ = config.K8s.DeleteDeployment(workloads.K8sName(api.Name))\n\t\t_, err := config.K8s.CreateDeployment(newDeployment)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\t_, err := config.K8s.UpdateDeployment(newDeployment)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc applyK8sService(api *spec.API, prevService *kcore.Service) error {\n\tnewService := serviceSpec(api)\n\n\tif prevService == nil {\n\t\t_, err := config.K8s.CreateService(newService)\n\t\treturn err\n\t}\n\n\t_, err := config.K8s.UpdateService(prevService, newService)\n\treturn err\n}\n\nfunc applyK8sVirtualService(api *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error {\n\tnewVirtualService := virtualServiceSpec(api)\n\n\tif prevVirtualService == nil {\n\t\t_, err := config.K8s.CreateVirtualService(newVirtualService)\n\t\treturn err\n\t}\n\n\t_, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService)\n\treturn err\n}\n\nfunc deleteK8sResources(apiName string) error {\n\treturn parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\t_, err := config.K8s.DeleteDeployment(workloads.K8sName(apiName))\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\t_, err := config.K8s.DeleteService(workloads.K8sName(apiName))\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\t_, err := config.K8s.DeleteVirtualService(workloads.K8sName(apiName))\n\t\t\treturn err\n\t\t},\n\t)\n}\n\nfunc deleteBucketResources(apiName string) error {\n\tprefix := filepath.Join(config.ClusterConfig.ClusterUID, \"apis\", apiName)\n\treturn config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true)\n}\n\n// returns true if min_replicas are not ready and no updated replicas have errored\nfunc isAPIUpdating(deployment *kapps.Deployment) (bool, error) {\n\tpods, err := config.K8s.ListPodsByLabel(\"apiName\", deployment.Labels[\"apiName\"])\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treplicaCounts := GetReplicaCounts(deployment, pods)\n\n\tautoscalingSpec, err := userconfig.AutoscalingFromAnnotations(deployment)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif replicaCounts.Ready < autoscalingSpec.MinReplicas && replicaCounts.TotalFailed() == 0 {\n\t\treturn true, nil\n\t}\n\n\treturn false, nil\n}\n\nfunc isPodSpecLatest(deployment *kapps.Deployment, pod *kcore.Pod) bool {\n\treturn deployment.Spec.Template.Labels[\"podID\"] == pod.Labels[\"podID\"] &&\n\t\tdeployment.Spec.Template.Labels[\"deploymentID\"] == pod.Labels[\"deploymentID\"]\n}\n\nfunc getDashboardURL(apiName string) string {\n\tloadBalancerURL, err := operator.LoadBalancerURL()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\n\tdashboardURL := fmt.Sprintf(\n\t\t\"%s/dashboard/d/%s/realtimeapi?orgId=1&refresh=30s&var-api_name=%s\",\n\t\tloadBalancerURL, _realtimeDashboardUID, apiName,\n\t)\n\n\treturn dashboardURL\n}\n"
  },
  {
    "path": "pkg/operator/resources/realtimeapi/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage realtimeapi\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrAPIUpdating = \"realtimeapi.api_updating\"\n)\n\nfunc ErrorAPIUpdating(apiName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAPIUpdating,\n\t\tMessage: fmt.Sprintf(\"%s is updating (override with --force)\", apiName),\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/resources/realtimeapi/k8s_specs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage realtimeapi\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistionetworking \"istio.io/api/networking/v1beta1\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nvar _terminationGracePeriodSeconds int64 = 60 // seconds\n\nfunc deploymentSpec(api *spec.API, prevDeployment *kapps.Deployment) *kapps.Deployment {\n\tcontainers, volumes := workloads.RealtimeContainers(*api)\n\n\treturn k8s.Deployment(&k8s.DeploymentSpec{\n\t\tName:           workloads.K8sName(api.Name),\n\t\tReplicas:       getRequestedReplicasFromDeployment(*api, prevDeployment),\n\t\tMaxSurge:       pointer.String(api.UpdateStrategy.MaxSurge),\n\t\tMaxUnavailable: pointer.String(api.UpdateStrategy.MaxUnavailable),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":               api.Name,\n\t\t\t\"apiKind\":               api.Kind.String(),\n\t\t\t\"apiID\":                 api.ID,\n\t\t\t\"specID\":                api.SpecID,\n\t\t\t\"initialDeploymentTime\": s.Int64(api.InitialDeploymentTime),\n\t\t\t\"deploymentID\":          api.DeploymentID,\n\t\t\t\"podID\":                 api.PodID,\n\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t},\n\t\tAnnotations: api.ToK8sAnnotations(),\n\t\tSelector: map[string]string{\n\t\t\t\"apiName\": api.Name,\n\t\t\t\"apiKind\": api.Kind.String(),\n\t\t},\n\t\tPodSpec: k8s.PodSpec{\n\t\t\tLabels: map[string]string{\n\t\t\t\t\"apiName\":               api.Name,\n\t\t\t\t\"apiKind\":               api.Kind.String(),\n\t\t\t\t\"initialDeploymentTime\": s.Int64(api.InitialDeploymentTime),\n\t\t\t\t\"deploymentID\":          api.DeploymentID,\n\t\t\t\t\"podID\":                 api.PodID,\n\t\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t\t},\n\t\t\tAnnotations: map[string]string{\n\t\t\t\t\"traffic.sidecar.istio.io/excludeOutboundIPRanges\": \"0.0.0.0/0\",\n\t\t\t},\n\t\t\tK8sPodSpec: kcore.PodSpec{\n\t\t\t\tRestartPolicy:                 \"Always\",\n\t\t\t\tTerminationGracePeriodSeconds: pointer.Int64(_terminationGracePeriodSeconds),\n\t\t\t\tContainers:                    containers,\n\t\t\t\tNodeSelector:                  workloads.NodeSelectors(),\n\t\t\t\tTolerations:                   workloads.GenerateResourceTolerations(),\n\t\t\t\tAffinity:                      workloads.GenerateNodeAffinities(api.NodeGroups),\n\t\t\t\tVolumes:                       volumes,\n\t\t\t\tServiceAccountName:            workloads.ServiceAccountName,\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc serviceSpec(api *spec.API) *kcore.Service {\n\treturn k8s.Service(&k8s.ServiceSpec{\n\t\tName:        workloads.K8sName(api.Name),\n\t\tPortName:    \"http\",\n\t\tPort:        consts.ProxyPortInt32,\n\t\tTargetPort:  consts.ProxyPortInt32,\n\t\tAnnotations: api.ToK8sAnnotations(),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":        api.Name,\n\t\t\t\"apiKind\":        api.Kind.String(),\n\t\t\t\"cortex.dev/api\": \"true\",\n\t\t},\n\t\tSelector: map[string]string{\n\t\t\t\"apiName\": api.Name,\n\t\t\t\"apiKind\": api.Kind.String(),\n\t\t},\n\t})\n}\n\nfunc virtualServiceSpec(api *spec.API) *istioclientnetworking.VirtualService {\n\tvar activatorWeight int32\n\tif api.Autoscaling.InitReplicas == 0 {\n\t\tactivatorWeight = 100\n\t}\n\n\treturn k8s.VirtualService(&k8s.VirtualServiceSpec{\n\t\tName:     workloads.K8sName(api.Name),\n\t\tGateways: []string{\"apis-gateway\"},\n\t\tDestinations: []k8s.Destination{\n\t\t\t{\n\t\t\t\tServiceName: workloads.K8sName(api.Name),\n\t\t\t\tWeight:      100 - activatorWeight,\n\t\t\t\tPort:        uint32(consts.ProxyPortInt32),\n\t\t\t\tHeaders: &istionetworking.Headers{\n\t\t\t\t\tResponse: &istionetworking.Headers_HeaderOperations{\n\t\t\t\t\t\tSet: map[string]string{\n\t\t\t\t\t\t\tconsts.CortexOriginHeader: \"api\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tServiceName: consts.ActivatorName,\n\t\t\t\tWeight:      activatorWeight,\n\t\t\t\tPort:        uint32(consts.ActivatorPortInt32),\n\t\t\t\tHeaders: &istionetworking.Headers{\n\t\t\t\t\tRequest: &istionetworking.Headers_HeaderOperations{\n\t\t\t\t\t\tSet: map[string]string{\n\t\t\t\t\t\t\tconsts.CortexAPINameHeader: api.Name,\n\t\t\t\t\t\t\tconsts.CortexTargetServiceHeader: fmt.Sprintf(\n\t\t\t\t\t\t\t\t\"http://%s.%s:%d\",\n\t\t\t\t\t\t\t\tworkloads.K8sName(api.Name),\n\t\t\t\t\t\t\t\tconsts.DefaultNamespace,\n\t\t\t\t\t\t\t\tconsts.ProxyPortInt32,\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tResponse: &istionetworking.Headers_HeaderOperations{\n\t\t\t\t\t\tSet: map[string]string{\n\t\t\t\t\t\t\tconsts.CortexOriginHeader: consts.ActivatorName,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tPrefixPath:  api.Networking.Endpoint,\n\t\tRewrite:     pointer.String(\"/\"),\n\t\tRetries:     pointer.Int32(0),\n\t\tAnnotations: api.ToK8sAnnotations(),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":               api.Name,\n\t\t\t\"apiKind\":               api.Kind.String(),\n\t\t\t\"apiID\":                 api.ID,\n\t\t\t\"specID\":                api.SpecID,\n\t\t\t\"initialDeploymentTime\": s.Int64(api.InitialDeploymentTime),\n\t\t\t\"deploymentID\":          api.DeploymentID,\n\t\t\t\"podID\":                 api.PodID,\n\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t},\n\t})\n}\n\nfunc getRequestedReplicasFromDeployment(api spec.API, deployment *kapps.Deployment) int32 {\n\trequestedReplicas := api.Autoscaling.InitReplicas\n\n\tif deployment != nil && deployment.Spec.Replicas != nil && *deployment.Spec.Replicas > 0 {\n\t\trequestedReplicas = *deployment.Spec.Replicas\n\t}\n\n\tif requestedReplicas < api.Autoscaling.MinReplicas {\n\t\trequestedReplicas = api.Autoscaling.MinReplicas\n\t}\n\n\tif requestedReplicas > api.Autoscaling.MaxReplicas {\n\t\trequestedReplicas = api.Autoscaling.MaxReplicas\n\t}\n\n\treturn requestedReplicas\n}\n"
  },
  {
    "path": "pkg/operator/resources/realtimeapi/status.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage realtimeapi\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nfunc GetReplicaCounts(deployment *kapps.Deployment, pods []kcore.Pod) *status.ReplicaCounts {\n\tcounts := status.ReplicaCounts{}\n\tcounts.Requested = *deployment.Spec.Replicas\n\n\tfor i := range pods {\n\t\tpod := pods[i]\n\t\tif pod.Labels[\"apiName\"] != deployment.Labels[\"apiName\"] {\n\t\t\tcontinue\n\t\t}\n\t\taddPodToReplicaCounts(&pods[i], deployment, &counts)\n\t}\n\n\treturn &counts\n}\n\nfunc addPodToReplicaCounts(pod *kcore.Pod, deployment *kapps.Deployment, counts *status.ReplicaCounts) {\n\tlatest := false\n\tif isPodSpecLatest(deployment, pod) {\n\t\tlatest = true\n\t}\n\n\tisPodReady := k8s.IsPodReady(pod)\n\tif latest && isPodReady {\n\t\tcounts.Ready++\n\t\treturn\n\t} else if !latest && isPodReady {\n\t\tcounts.ReadyOutOfDate++\n\t\treturn\n\t}\n\n\tpodStatus := k8s.GetPodStatus(pod)\n\n\tif podStatus == k8s.PodStatusTerminating {\n\t\tcounts.Terminating++\n\t\treturn\n\t}\n\n\tif !latest {\n\t\treturn\n\t}\n\n\tswitch podStatus {\n\tcase k8s.PodStatusPending:\n\t\tcounts.Pending++\n\tcase k8s.PodStatusStalled:\n\t\tcounts.Stalled++\n\tcase k8s.PodStatusCreating:\n\t\tcounts.Creating++\n\tcase k8s.PodStatusReady:\n\t\tcounts.Ready++\n\tcase k8s.PodStatusNotReady:\n\t\tcounts.NotReady++\n\tcase k8s.PodStatusErrImagePull:\n\t\tcounts.ErrImagePull++\n\tcase k8s.PodStatusFailed:\n\t\tcounts.Failed++\n\tcase k8s.PodStatusKilled:\n\t\tcounts.Killed++\n\tcase k8s.PodStatusKilledOOM:\n\t\tcounts.KilledOOM++\n\tcase k8s.PodStatusUnknown:\n\t\tcounts.Unknown++\n\t}\n}\n"
  },
  {
    "path": "pkg/operator/resources/resources.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage resources\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\tbatch \"github.com/cortexlabs/cortex/pkg/crds/apis/batch/v1alpha1\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/logging\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/asyncapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/batchapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/job/taskapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/realtimeapi\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/resources/trafficsplitter\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkapps \"k8s.io/api/apps/v1\"\n\tkbatch \"k8s.io/api/batch/v1\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tklabels \"k8s.io/apimachinery/pkg/labels\"\n)\n\nvar operatorLogger = logging.GetLogger()\n\n// Returns an error if resource doesn't exist\nfunc GetDeployedResourceByName(resourceName string) (*operator.DeployedResource, error) {\n\tresource, err := GetDeployedResourceByNameOrNil(resourceName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif resource == nil {\n\t\treturn nil, ErrorAPINotDeployed(resourceName)\n\t}\n\n\treturn resource, nil\n}\n\nfunc GetDeployedResourceByNameOrNil(resourceName string) (*operator.DeployedResource, error) {\n\tvirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(resourceName))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif virtualService == nil {\n\t\treturn nil, nil\n\t}\n\n\treturn &operator.DeployedResource{\n\t\tResource: userconfig.Resource{\n\t\t\tName: virtualService.Labels[\"apiName\"],\n\t\t\tKind: userconfig.KindFromString(virtualService.Labels[\"apiKind\"]),\n\t\t},\n\t\tVirtualService: virtualService,\n\t}, nil\n}\n\nfunc Deploy(configFileName string, configBytes []byte, force bool) ([]schema.DeployResult, error) {\n\tapiConfigs, err := spec.ExtractAPIConfigs(configBytes, configFileName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\terr = ValidateClusterAPIs(apiConfigs)\n\tif err != nil {\n\t\terr = errors.Append(err, fmt.Sprintf(\"\\n\\napi configuration schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t\treturn nil, err\n\t}\n\n\t// This is done if user specifies RealtimeAPIs in same file as TrafficSplitter\n\tapiConfigs = append(ExclusiveFilterAPIsByKind(apiConfigs, userconfig.TrafficSplitterKind), InclusiveFilterAPIsByKind(apiConfigs, userconfig.TrafficSplitterKind)...)\n\n\tresults := make([]schema.DeployResult, 0, len(apiConfigs))\n\tfor i := range apiConfigs {\n\t\tapiConfig := apiConfigs[i]\n\n\t\tapi, msg, err := UpdateAPI(&apiConfig, force)\n\n\t\tresult := schema.DeployResult{\n\t\t\tMessage: msg,\n\t\t\tAPI:     api,\n\t\t}\n\n\t\tif err != nil {\n\t\t\tresult.Error = errors.ErrorStr(err)\n\t\t}\n\n\t\tresults = append(results, result)\n\t}\n\n\treturn results, nil\n}\n\nfunc UpdateAPI(apiConfig *userconfig.API, force bool) (*schema.APIResponse, string, error) {\n\tdeployedResource, err := GetDeployedResourceByNameOrNil(apiConfig.Name)\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tif deployedResource != nil && deployedResource.Kind != apiConfig.Kind {\n\t\treturn nil, \"\", ErrorCannotChangeKindOfDeployedAPI(apiConfig.Name, apiConfig.Kind, deployedResource.Kind)\n\t}\n\n\ttelemetry.Event(\"operator.deploy\", apiConfig.TelemetryEvent())\n\n\tvar api *spec.API\n\tvar msg string\n\tswitch apiConfig.Kind {\n\tcase userconfig.RealtimeAPIKind:\n\t\tapi, msg, err = realtimeapi.UpdateAPI(apiConfig, force)\n\tcase userconfig.BatchAPIKind:\n\t\tapi, msg, err = batchapi.UpdateAPI(apiConfig)\n\tcase userconfig.TaskAPIKind:\n\t\tapi, msg, err = taskapi.UpdateAPI(apiConfig)\n\tcase userconfig.AsyncAPIKind:\n\t\tapi, msg, err = asyncapi.UpdateAPI(*apiConfig, force)\n\tcase userconfig.TrafficSplitterKind:\n\t\tapi, msg, err = trafficsplitter.UpdateAPI(apiConfig)\n\tdefault:\n\t\treturn nil, \"\", ErrorOperationIsOnlySupportedForKind(\n\t\t\t*deployedResource, userconfig.RealtimeAPIKind,\n\t\t\tuserconfig.AsyncAPIKind,\n\t\t\tuserconfig.BatchAPIKind,\n\t\t\tuserconfig.TrafficSplitterKind,\n\t\t\tuserconfig.TaskAPIKind,\n\t\t) // unexpected\n\t}\n\n\tif err == nil && api != nil {\n\t\tapiEndpoint, _ := operator.APIEndpoint(api)\n\n\t\treturn &schema.APIResponse{\n\t\t\tSpec:     api,\n\t\t\tEndpoint: &apiEndpoint,\n\t\t}, msg, nil\n\t}\n\n\treturn nil, msg, err\n}\n\nfunc RefreshAPI(apiName string, force bool) (string, error) {\n\tdeployedResource, err := GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tswitch deployedResource.Kind {\n\tcase userconfig.RealtimeAPIKind:\n\t\treturn realtimeapi.RefreshAPI(apiName, force)\n\tcase userconfig.AsyncAPIKind:\n\t\treturn asyncapi.RefreshAPI(apiName, force)\n\tdefault:\n\t\treturn \"\", ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind)\n\t}\n}\n\nfunc DeleteAPI(apiName string, keepCache bool) (*schema.DeleteResponse, error) {\n\tdeployedResource, err := GetDeployedResourceByNameOrNil(apiName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif deployedResource == nil {\n\t\t// Delete anyways just to be sure everything is deleted\n\t\troutines.RunWithPanicHandler(func() {\n\t\t\terr := parallel.RunFirstErr(\n\t\t\t\tfunc() error {\n\t\t\t\t\treturn realtimeapi.DeleteAPI(apiName, keepCache)\n\t\t\t\t},\n\t\t\t\tfunc() error {\n\t\t\t\t\treturn batchapi.DeleteAPI(apiName, keepCache)\n\t\t\t\t},\n\t\t\t\tfunc() error {\n\t\t\t\t\treturn trafficsplitter.DeleteAPI(apiName, keepCache)\n\t\t\t\t},\n\t\t\t\tfunc() error {\n\t\t\t\t\treturn taskapi.DeleteAPI(apiName, keepCache)\n\t\t\t\t},\n\t\t\t\tfunc() error {\n\t\t\t\t\treturn asyncapi.DeleteAPI(apiName, keepCache)\n\t\t\t\t},\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\ttelemetry.Error(err)\n\t\t\t}\n\t\t})\n\t\treturn nil, ErrorAPINotDeployed(apiName)\n\t}\n\n\tswitch deployedResource.Kind {\n\tcase userconfig.RealtimeAPIKind:\n\t\terr := checkIfUsedByTrafficSplitter(apiName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\terr = realtimeapi.DeleteAPI(apiName, keepCache)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.TrafficSplitterKind:\n\t\terr := trafficsplitter.DeleteAPI(apiName, keepCache)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.BatchAPIKind:\n\t\terr := batchapi.DeleteAPI(apiName, keepCache)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.TaskAPIKind:\n\t\terr := taskapi.DeleteAPI(apiName, keepCache)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.AsyncAPIKind:\n\t\terr = asyncapi.DeleteAPI(apiName, keepCache)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tdefault:\n\t\treturn nil, ErrorOperationIsOnlySupportedForKind(*deployedResource, userconfig.RealtimeAPIKind, userconfig.AsyncAPIKind, userconfig.BatchAPIKind, userconfig.TrafficSplitterKind) // unexpected\n\t}\n\n\treturn &schema.DeleteResponse{\n\t\tMessage: fmt.Sprintf(\"deleting %s\", apiName),\n\t}, nil\n}\n\nfunc GetAPIs() ([]schema.APIResponse, error) {\n\tvar deployments []kapps.Deployment\n\tvar k8sTaskJobs []kbatch.Job\n\tvar taskAPIPods []kcore.Pod\n\tvar virtualServices []istioclientnetworking.VirtualService\n\tvar batchJobList batch.BatchJobList\n\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tdeployments, err = config.K8s.ListDeploymentsWithLabelKeys(\"apiName\")\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\ttaskAPIPods, err = config.K8s.ListPodsByLabel(\"apiKind\", userconfig.TaskAPIKind.String())\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tk8sTaskJobs, err = config.K8s.ListJobs(\n\t\t\t\t&kmeta.ListOptions{\n\t\t\t\t\tLabelSelector: klabels.SelectorFromSet(\n\t\t\t\t\t\tmap[string]string{\n\t\t\t\t\t\t\t\"apiKind\": userconfig.TaskAPIKind.String(),\n\t\t\t\t\t\t},\n\t\t\t\t\t).String(),\n\t\t\t\t},\n\t\t\t)\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\tvar err error\n\t\t\tvirtualServices, err = config.K8s.ListVirtualServicesWithLabelKeys(\"apiName\")\n\t\t\treturn err\n\t\t},\n\t\tfunc() error {\n\t\t\treturn config.K8s.List(context.Background(), &batchJobList)\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar realtimeAPIDeployments []kapps.Deployment\n\tvar asyncAPIDeployments []kapps.Deployment\n\tfor _, deployment := range deployments {\n\t\tswitch deployment.Labels[\"apiKind\"] {\n\t\tcase userconfig.RealtimeAPIKind.String():\n\t\t\trealtimeAPIDeployments = append(realtimeAPIDeployments, deployment)\n\t\tcase userconfig.AsyncAPIKind.String():\n\t\t\tasyncAPIDeployments = append(asyncAPIDeployments, deployment)\n\t\t}\n\t}\n\n\tvar batchAPIVirtualServices []istioclientnetworking.VirtualService\n\tvar taskAPIVirtualServices []istioclientnetworking.VirtualService\n\tvar trafficSplitterVirtualServices []istioclientnetworking.VirtualService\n\n\tfor _, vs := range virtualServices {\n\t\tswitch vs.Labels[\"apiKind\"] {\n\t\tcase userconfig.BatchAPIKind.String():\n\t\t\tbatchAPIVirtualServices = append(batchAPIVirtualServices, vs)\n\t\tcase userconfig.TrafficSplitterKind.String():\n\t\t\ttrafficSplitterVirtualServices = append(trafficSplitterVirtualServices, vs)\n\t\tcase userconfig.TaskAPIKind.String():\n\t\t\ttaskAPIVirtualServices = append(taskAPIVirtualServices, vs)\n\t\t}\n\t}\n\n\trealtimeAPIList, err := realtimeapi.GetAllAPIs(realtimeAPIDeployments)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar taskAPIList []schema.APIResponse\n\ttaskAPIList, err = taskapi.GetAllAPIs(taskAPIVirtualServices, k8sTaskJobs, taskAPIPods)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tbatchAPIList, err := batchapi.GetAllAPIs(batchAPIVirtualServices, batchJobList.Items)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tasyncAPIList, err := asyncapi.GetAllAPIs(asyncAPIDeployments)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\ttrafficSplitterList, err := trafficsplitter.GetAllAPIs(trafficSplitterVirtualServices)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tresponse := make([]schema.APIResponse, 0, len(realtimeAPIList)+len(batchAPIList)+len(trafficSplitterList))\n\n\tresponse = append(response, realtimeAPIList...)\n\tresponse = append(response, batchAPIList...)\n\tresponse = append(response, taskAPIList...)\n\tresponse = append(response, asyncAPIList...)\n\tresponse = append(response, trafficSplitterList...)\n\n\treturn response, nil\n}\n\nfunc GetAPI(apiName string) ([]schema.APIResponse, error) {\n\tdeployedResource, err := GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar apiResponse []schema.APIResponse\n\n\tswitch deployedResource.Kind {\n\tcase userconfig.RealtimeAPIKind:\n\t\tapiResponse, err = realtimeapi.GetAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.BatchAPIKind:\n\t\tapiResponse, err = batchapi.GetAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.TaskAPIKind:\n\t\tapiResponse, err = taskapi.GetAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.AsyncAPIKind:\n\t\tapiResponse, err = asyncapi.GetAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.TrafficSplitterKind:\n\t\tapiResponse, err = trafficsplitter.GetAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tdefault:\n\t\treturn nil, ErrorOperationIsOnlySupportedForKind(\n\t\t\t*deployedResource,\n\t\t\tuserconfig.RealtimeAPIKind, userconfig.BatchAPIKind,\n\t\t\tuserconfig.TaskAPIKind, userconfig.TrafficSplitterKind,\n\t\t\tuserconfig.AsyncAPIKind,\n\t\t) // unexpected\n\t}\n\n\t// Get past API deploy times\n\tif len(apiResponse) > 0 {\n\t\tapiResponse[0].APIVersions, err = getPastAPIDeploys(deployedResource.Name)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn apiResponse, nil\n}\n\nfunc GetAPIByID(apiName string, apiID string) ([]schema.APIResponse, error) {\n\t// check if the API is currently running, so that additional information can be returned\n\tdeployedResource, err := GetDeployedResourceByName(apiName)\n\tif err == nil && deployedResource != nil && deployedResource.ID() == apiID {\n\t\treturn GetAPI(apiName)\n\t}\n\n\t// search for the API spec with the old ID\n\tapiSpec, err := operator.DownloadAPISpec(apiName, apiID)\n\tif err != nil {\n\t\tif aws.IsGenericNotFoundErr(err) {\n\t\t\treturn nil, ErrorAPIIDNotFound(apiName, apiID)\n\t\t}\n\t\treturn nil, err\n\t}\n\n\treturn []schema.APIResponse{\n\t\t{\n\t\t\tSpec: apiSpec,\n\t\t},\n\t}, nil\n}\n\nfunc getPastAPIDeploys(apiName string) ([]schema.APIVersion, error) {\n\tvar apiVersions []schema.APIVersion\n\n\tapiIDs, err := config.AWS.ListS3DirOneLevel(config.ClusterConfig.Bucket, spec.KeysPrefix(apiName, config.ClusterConfig.ClusterUID), pointer.Int64(10), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, apiID := range apiIDs {\n\t\tlastUpdated, err := spec.TimeFromAPIID(apiID)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tapiVersions = append(apiVersions, schema.APIVersion{\n\t\t\tAPIID:       apiID,\n\t\t\tLastUpdated: lastUpdated.Unix(),\n\t\t})\n\t}\n\n\treturn apiVersions, nil\n}\n\n// checkIfUsedByTrafficSplitter checks if api is used by a deployed TrafficSplitter\nfunc checkIfUsedByTrafficSplitter(apiName string) error {\n\tvirtualServices, err := config.K8s.ListVirtualServicesByLabel(\"apiKind\", userconfig.TrafficSplitterKind.String())\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar usedByTrafficSplitters []string\n\tfor _, vs := range virtualServices {\n\t\ttrafficSplitterSpec, err := operator.DownloadAPISpec(vs.Labels[\"apiName\"], vs.Labels[\"apiID\"])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, api := range trafficSplitterSpec.APIs {\n\t\t\tif apiName == api.Name {\n\t\t\t\tusedByTrafficSplitters = append(usedByTrafficSplitters, trafficSplitterSpec.Name)\n\t\t\t}\n\t\t}\n\t}\n\tif len(usedByTrafficSplitters) > 0 {\n\t\treturn ErrorAPIUsedByTrafficSplitter(usedByTrafficSplitters)\n\t}\n\treturn nil\n}\n\nfunc DescribeAPI(apiName string) ([]schema.APIResponse, error) {\n\tdeployedResource, err := GetDeployedResourceByName(apiName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar apiResponse []schema.APIResponse\n\n\tswitch deployedResource.Kind {\n\tcase userconfig.RealtimeAPIKind:\n\t\tapiResponse, err = realtimeapi.DescribeAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tcase userconfig.AsyncAPIKind:\n\t\tapiResponse, err = asyncapi.DescribeAPIByName(deployedResource)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\tdefault:\n\t\treturn nil, ErrorOperationIsOnlySupportedForKind(\n\t\t\t*deployedResource,\n\t\t\tuserconfig.RealtimeAPIKind,\n\t\t\tuserconfig.AsyncAPIKind,\n\t\t) // unexpected\n\t}\n\n\treturn apiResponse, nil\n}\n"
  },
  {
    "path": "pkg/operator/resources/trafficsplitter/api.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage trafficsplitter\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/parallel\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/lib/routines\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/schema\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n)\n\n// UpdateAPI creates or updates a traffic splitter API kind\nfunc UpdateAPI(apiConfig *userconfig.API) (*spec.API, string, error) {\n\tprevVirtualService, err := config.K8s.GetVirtualService(workloads.K8sName(apiConfig.Name))\n\tif err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tinitialDeploymentTime := time.Now().UnixNano()\n\tif prevVirtualService != nil && prevVirtualService.Labels[\"initialDeploymentTime\"] != \"\" {\n\t\tvar err error\n\t\tinitialDeploymentTime, err = k8s.ParseInt64Label(prevVirtualService, \"initialDeploymentTime\")\n\t\tif err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\t}\n\n\tapi := spec.GetAPISpec(apiConfig, initialDeploymentTime, \"\", config.ClusterConfig.ClusterUID)\n\tif prevVirtualService == nil {\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"failed to upload api spec\")\n\t\t}\n\n\t\tif err := applyK8sVirtualService(api, prevVirtualService); err != nil {\n\t\t\troutines.RunWithPanicHandler(func() {\n\t\t\t\t_ = deleteK8sResources(api.Name)\n\t\t\t})\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"created %s\", api.Resource.UserString()), nil\n\t}\n\n\tif prevVirtualService.Labels[\"specID\"] != api.SpecID {\n\t\tif err := config.AWS.UploadJSONToS3(api, config.ClusterConfig.Bucket, api.Key); err != nil {\n\t\t\treturn nil, \"\", errors.Wrap(err, \"failed to upload api spec\")\n\t\t}\n\n\t\tif err := applyK8sVirtualService(api, prevVirtualService); err != nil {\n\t\t\treturn nil, \"\", err\n\t\t}\n\n\t\treturn api, fmt.Sprintf(\"updated %s\", api.Resource.UserString()), nil\n\t}\n\n\treturn api, fmt.Sprintf(\"%s is up to date\", api.Resource.UserString()), nil\n}\n\n// DeleteAPI deletes all the resources related to a given traffic splitter API\nfunc DeleteAPI(apiName string, keepCache bool) error {\n\terr := parallel.RunFirstErr(\n\t\tfunc() error {\n\t\t\treturn deleteK8sResources(apiName)\n\t\t},\n\t\tfunc() error {\n\t\t\tif keepCache {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\t// best effort deletion\n\t\t\t_ = deleteS3Resources(apiName)\n\t\t\treturn nil\n\t\t},\n\t)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc applyK8sVirtualService(trafficSplitter *spec.API, prevVirtualService *istioclientnetworking.VirtualService) error {\n\tnewVirtualService := virtualServiceSpec(trafficSplitter)\n\n\tif prevVirtualService == nil {\n\t\t_, err := config.K8s.CreateVirtualService(newVirtualService)\n\t\treturn err\n\t}\n\n\t_, err := config.K8s.UpdateVirtualService(prevVirtualService, newVirtualService)\n\treturn err\n}\n\nfunc getTrafficSplitterDestinations(trafficSplitter *spec.API) []k8s.Destination {\n\tdestinations := make([]k8s.Destination, len(trafficSplitter.APIs))\n\tfor i, api := range trafficSplitter.APIs {\n\t\tdestinations[i] = k8s.Destination{\n\t\t\tServiceName: workloads.K8sName(api.Name),\n\t\t\tWeight:      api.Weight,\n\t\t\tPort:        uint32(consts.ProxyPortInt32),\n\t\t\tShadow:      api.Shadow,\n\t\t}\n\t}\n\treturn destinations\n}\n\n// GetAllAPIs returns a list of metadata, in the form of schema.APIResponse, about all the created traffic splitter APIs\nfunc GetAllAPIs(virtualServices []istioclientnetworking.VirtualService) ([]schema.APIResponse, error) {\n\tvar trafficSplitters []schema.APIResponse\n\tfor i := range virtualServices {\n\t\tapiName := virtualServices[i].Labels[\"apiName\"]\n\n\t\tmetadata, err := spec.MetadataFromVirtualService(&virtualServices[i])\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"api %s\", apiName))\n\t\t}\n\n\t\tif metadata.Kind != userconfig.TrafficSplitterKind {\n\t\t\tcontinue\n\t\t}\n\n\t\ttargets, err := userconfig.TrafficSplitterTargetsFromAnnotations(&virtualServices[i])\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"api %s\", apiName))\n\t\t}\n\n\t\ttrafficSplitters = append(trafficSplitters, schema.APIResponse{\n\t\t\tMetadata:                  metadata,\n\t\t\tNumTrafficSplitterTargets: pointer.Int32(targets),\n\t\t})\n\t}\n\n\treturn trafficSplitters, nil\n}\n\n// GetAPIByName retrieves the metadata, in the form of schema.APIResponse, of a single traffic splitter API\nfunc GetAPIByName(deployedResource *operator.DeployedResource) ([]schema.APIResponse, error) {\n\tmetadata, err := spec.MetadataFromVirtualService(deployedResource.VirtualService)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tapi, err := operator.DownloadAPISpec(deployedResource.Name, metadata.APIID)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tendpoint, err := operator.APIEndpoint(api)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn []schema.APIResponse{\n\t\t{\n\t\t\tSpec:     api,\n\t\t\tMetadata: metadata,\n\t\t\tEndpoint: &endpoint,\n\t\t},\n\t}, nil\n}\n\nfunc deleteK8sResources(apiName string) error {\n\t_, err := config.K8s.DeleteVirtualService(workloads.K8sName(apiName))\n\treturn err\n}\n\nfunc deleteS3Resources(apiName string) error {\n\tprefix := filepath.Join(config.ClusterConfig.ClusterUID, \"apis\", apiName)\n\treturn config.AWS.DeleteS3Dir(config.ClusterConfig.Bucket, prefix, true)\n}\n"
  },
  {
    "path": "pkg/operator/resources/trafficsplitter/k8s_specs.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage trafficsplitter\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/workloads\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n)\n\nfunc virtualServiceSpec(trafficSplitter *spec.API) *istioclientnetworking.VirtualService {\n\treturn k8s.VirtualService(&k8s.VirtualServiceSpec{\n\t\tName:         workloads.K8sName(trafficSplitter.Name),\n\t\tGateways:     []string{\"apis-gateway\"},\n\t\tDestinations: getTrafficSplitterDestinations(trafficSplitter),\n\t\tExactPath:    trafficSplitter.Networking.Endpoint,\n\t\tRewrite:      pointer.String(\"/\"),\n\t\tRetries:      pointer.Int32(0),\n\t\tAnnotations:  trafficSplitter.ToK8sAnnotations(),\n\t\tLabels: map[string]string{\n\t\t\t\"apiName\":               trafficSplitter.Name,\n\t\t\t\"apiKind\":               trafficSplitter.Kind.String(),\n\t\t\t\"apiID\":                 trafficSplitter.ID,\n\t\t\t\"specID\":                trafficSplitter.SpecID,\n\t\t\t\"initialDeploymentTime\": s.Int64(trafficSplitter.InitialDeploymentTime),\n\t\t\t\"cortex.dev/api\":        \"true\",\n\t\t},\n\t})\n}\n"
  },
  {
    "path": "pkg/operator/resources/validations.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage resources\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/operator/operator\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n)\n\nfunc ValidateClusterAPIs(apis []userconfig.API) error {\n\tif len(apis) == 0 {\n\t\treturn spec.ErrorNoAPIs()\n\t}\n\n\tif len(config.ClusterConfig.NodeGroups) == 0 {\n\t\treturn ErrorNoNodeGroups()\n\t}\n\n\tvirtualServices, err := config.K8s.ListVirtualServices(nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdeployedRealtimeAPIs := strset.New()\n\tfor _, virtualService := range virtualServices {\n\t\tif virtualService.Labels[\"apiKind\"] == userconfig.RealtimeAPIKind.String() {\n\t\t\tdeployedRealtimeAPIs.Add(virtualService.Labels[\"apiName\"])\n\t\t}\n\t}\n\n\trealtimeAPIs := InclusiveFilterAPIsByKind(apis, userconfig.RealtimeAPIKind)\n\n\tfor i := range apis {\n\t\tapi := &apis[i]\n\t\tif api.Kind == userconfig.RealtimeAPIKind || api.Kind == userconfig.BatchAPIKind ||\n\t\t\tapi.Kind == userconfig.TaskAPIKind || api.Kind == userconfig.AsyncAPIKind {\n\n\t\t\tif err := spec.ValidateAPI(api, config.AWS, config.K8s); err != nil {\n\t\t\t\treturn errors.Wrap(err, api.Identify())\n\t\t\t}\n\n\t\t\tif err := validateEndpointCollisions(api, virtualServices); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif api.Kind == userconfig.TrafficSplitterKind {\n\t\t\tif err := spec.ValidateTrafficSplitter(api); err != nil {\n\t\t\t\treturn errors.Wrap(err, api.Identify())\n\t\t\t}\n\t\t\tif err := checkIfAPIExists(api.APIs, realtimeAPIs, deployedRealtimeAPIs); err != nil {\n\t\t\t\treturn errors.Wrap(err, api.Identify())\n\t\t\t}\n\t\t\tif err := validateEndpointCollisions(api, virtualServices); err != nil {\n\t\t\t\treturn errors.Wrap(err, api.Identify())\n\t\t\t}\n\t\t}\n\t}\n\n\tmaxMemMap, err := operator.UpdateMemoryCapacityConfigMap()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor i := range apis {\n\t\tapi := &apis[i]\n\t\tif api.Kind != userconfig.TrafficSplitterKind {\n\t\t\tif err := validateK8sCompute(api, maxMemMap); err != nil {\n\t\t\t\treturn errors.Wrap(err, api.Identify())\n\t\t\t}\n\t\t}\n\t}\n\n\tdups := spec.FindDuplicateNames(apis)\n\tif len(dups) > 0 {\n\t\treturn spec.ErrorDuplicateName(dups)\n\t}\n\tdups = findDuplicateEndpoints(apis)\n\tif len(dups) > 0 {\n\t\treturn spec.ErrorDuplicateEndpointInOneDeploy(dups)\n\t}\n\n\treturn nil\n}\n\nvar _nvidiaDevicePluginCPUReserve = kresource.MustParse(\"100m\")\nvar _nvidiaDevicePluginMemReserve = kresource.MustParse(\"100Mi\")\n\nvar _nvidiaDCGMExporterCPUReserve = kresource.MustParse(\"50m\")\nvar _nvidiaDCGMExporterMemReserve = kresource.MustParse(\"50Mi\")\n\nvar _neuronDevicePluginCPUReserve = kresource.MustParse(\"100m\")\nvar _neuronDevicePluginMemReserve = kresource.MustParse(\"100Mi\")\n\nfunc validateK8sCompute(api *userconfig.API, maxMemMap map[string]kresource.Quantity) error {\n\tclusterNodeGroupNames := strset.New(config.ClusterConfig.GetNodeGroupNames()...)\n\tfor _, ngName := range api.NodeGroups {\n\t\tif !clusterNodeGroupNames.Has(ngName) {\n\t\t\treturn errors.Wrap(ErrorInvalidNodeGroupSelector(ngName, config.ClusterConfig.GetNodeGroupNames()), userconfig.NodeGroupsKey)\n\t\t}\n\t}\n\n\tcompute := userconfig.GetPodComputeRequest(api)\n\n\tfor _, ng := range config.ClusterConfig.NodeGroups {\n\t\tif api.NodeGroups != nil && !slices.HasString(api.NodeGroups, ng.Name) {\n\t\t\tcontinue\n\t\t}\n\n\t\tnodeCPU, nodeMem, nodeGPU, nodeInf := getNodeCapacity(ng.InstanceType, maxMemMap)\n\n\t\tif compute.CPU != nil && nodeCPU.Cmp(compute.CPU.Quantity) < 0 {\n\t\t\tcontinue\n\t\t} else if compute.Mem != nil && nodeMem.Cmp(compute.Mem.Quantity) < 0 {\n\t\t\tcontinue\n\t\t} else if compute.GPU > nodeGPU {\n\t\t\tcontinue\n\t\t} else if compute.Inf > nodeInf {\n\t\t\tcontinue\n\t\t}\n\n\t\t// we found a node group that has capacity\n\t\treturn nil\n\t}\n\n\t// no nodegroups have capacity\n\treturn ErrorNoAvailableNodeComputeLimit(api, compute, maxMemMap)\n}\n\nfunc getNodeCapacity(instanceType string, maxMemMap map[string]kresource.Quantity) (kresource.Quantity, kresource.Quantity, int64, int64) {\n\tinstanceMetadata := aws.InstanceMetadatas[config.ClusterConfig.Region][instanceType]\n\n\tcpu := instanceMetadata.CPU.DeepCopy()\n\tcpu.Sub(consts.CortexCPUPodReserved)\n\tcpu.Sub(consts.CortexCPUK8sReserved)\n\n\tmem := maxMemMap[instanceType].DeepCopy()\n\tmem.Sub(consts.CortexMemPodReserved)\n\tmem.Sub(consts.CortexMemK8sReserved)\n\n\tgpu := instanceMetadata.GPU\n\tif gpu > 0 {\n\t\t// Reserve resources for nvidia device plugin daemonset\n\t\tcpu.Sub(_nvidiaDevicePluginCPUReserve)\n\t\tmem.Sub(_nvidiaDevicePluginMemReserve)\n\t\t// Reserve resources for nvidia dcgm prometheus exporter\n\t\tcpu.Sub(_nvidiaDCGMExporterCPUReserve)\n\t\tmem.Sub(_nvidiaDCGMExporterMemReserve)\n\t}\n\n\tinf := instanceMetadata.Inf\n\tif inf > 0 {\n\t\t// Reserve resources for neuron device plugin daemonset\n\t\tcpu.Sub(_neuronDevicePluginCPUReserve)\n\t\tmem.Sub(_neuronDevicePluginMemReserve)\n\t}\n\n\treturn cpu, mem, gpu, inf\n}\n\nfunc validateEndpointCollisions(api *userconfig.API, virtualServices []istioclientnetworking.VirtualService) error {\n\tfor i := range virtualServices {\n\t\tvirtualService := virtualServices[i]\n\t\tgateways := k8s.ExtractVirtualServiceGateways(&virtualService)\n\t\tif !gateways.Has(\"apis-gateway\") {\n\t\t\tcontinue\n\t\t}\n\n\t\tendpoints := k8s.ExtractVirtualServiceEndpoints(&virtualService)\n\t\tfor endpoint := range endpoints {\n\t\t\tif s.EnsureSuffix(endpoint, \"/\") == s.EnsureSuffix(*api.Networking.Endpoint, \"/\") && virtualService.Labels[\"apiName\"] != api.Name {\n\t\t\t\treturn errors.Wrap(spec.ErrorDuplicateEndpoint(virtualService.Labels[\"apiName\"]), userconfig.NetworkingKey, userconfig.EndpointKey, endpoint)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc findDuplicateEndpoints(apis []userconfig.API) []userconfig.API {\n\tendpoints := make(map[string][]userconfig.API)\n\n\tfor _, api := range apis {\n\t\tendpoints[*api.Networking.Endpoint] = append(endpoints[*api.Networking.Endpoint], api)\n\t}\n\n\tfor endpoint := range endpoints {\n\t\tif len(endpoints[endpoint]) > 1 {\n\t\t\treturn endpoints[endpoint]\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// InclusiveFilterAPIsByKind includes only provided Kinds\nfunc InclusiveFilterAPIsByKind(apis []userconfig.API, kindsToInclude ...userconfig.Kind) []userconfig.API {\n\tkindsToIncludeSet := strset.New()\n\tfor _, kind := range kindsToInclude {\n\t\tkindsToIncludeSet.Add(kind.String())\n\t}\n\tfileredAPIs := []userconfig.API{}\n\tfor _, api := range apis {\n\t\tif kindsToIncludeSet.Has(api.Kind.String()) {\n\t\t\tfileredAPIs = append(fileredAPIs, api)\n\t\t}\n\t}\n\treturn fileredAPIs\n}\n\nfunc ExclusiveFilterAPIsByKind(apis []userconfig.API, kindsToExclude ...userconfig.Kind) []userconfig.API {\n\tkindsToExcludeSet := strset.New()\n\tfor _, kind := range kindsToExclude {\n\t\tkindsToExcludeSet.Add(kind.String())\n\t}\n\tfileredAPIs := []userconfig.API{}\n\tfor _, api := range apis {\n\t\tif !kindsToExcludeSet.Has(api.Kind.String()) {\n\t\t\tfileredAPIs = append(fileredAPIs, api)\n\t\t}\n\t}\n\treturn fileredAPIs\n}\n\n// checkIfAPIExists checks if referenced apis in trafficsplitter are either defined in yaml or already deployed.\nfunc checkIfAPIExists(trafficSplitterAPIs []*userconfig.TrafficSplit, apis []userconfig.API, deployedRealtimeAPIs strset.Set) error {\n\tvar missingAPIs []string\n\t// check if apis named in trafficsplitter are either defined in same yaml or already deployed\n\tfor _, trafficSplitAPI := range trafficSplitterAPIs {\n\t\t// check if already deployed\n\t\tdeployed := deployedRealtimeAPIs.Has(trafficSplitAPI.Name)\n\n\t\t// check defined apis\n\t\tfor _, definedAPI := range apis {\n\t\t\tif trafficSplitAPI.Name == definedAPI.Name {\n\t\t\t\tdeployed = true\n\t\t\t}\n\t\t}\n\t\tif !deployed {\n\t\t\tmissingAPIs = append(missingAPIs, trafficSplitAPI.Name)\n\t\t}\n\t}\n\tif len(missingAPIs) != 0 {\n\t\treturn ErrorAPIsNotDeployed(missingAPIs)\n\t}\n\treturn nil\n\n}\n"
  },
  {
    "path": "pkg/operator/schema/config_key.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage schema\n\nconst (\n\t// Job Submission\n\tBatchSizeKey          = \"batch_size\"\n\tItemsKey              = \"items\"\n\tItemListKey           = \"item_list\"\n\tFilePathListerKey     = \"file_path_lister\"\n\tDelimitedFilesKey     = \"delimited_files\"\n\tS3PathsKey            = \"s3_paths\"\n\tIncludesKey           = \"includes\"\n\tExcludesKey           = \"excludes\"\n\tWorkersKey            = \"workers\"\n\tTimeoutKey            = \"timeout\"\n\tMaxReceiveCountKey    = \"max_receive_count\"\n\tARNKey                = \"arn\"\n\tSQSDeadLetterQueueKey = \"sqs_dead_letter_queue\"\n)\n"
  },
  {
    "path": "pkg/operator/schema/job_submission.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage schema\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n)\n\ntype ItemList struct {\n\tItems     []json.RawMessage `json:\"items\"`\n\tBatchSize int               `json:\"batch_size\"`\n}\n\ntype S3Lister struct {\n\tS3Paths    []string `json:\"s3_paths\"` // s3://<bucket_name>/key\n\tIncludes   []string `json:\"includes\"`\n\tExcludes   []string `json:\"excludes\"`\n\tMaxResults *int64   `json:\"-\"` // this is not currently exposed to the user (it's used for validations)\n}\n\ntype FilePathLister struct {\n\tS3Lister\n\tBatchSize int `json:\"batch_size\"`\n}\n\ntype DelimitedFiles struct {\n\tS3Lister\n\tBatchSize int `json:\"batch_size\"`\n}\n\ntype BatchJobSubmission struct {\n\tspec.RuntimeBatchJobConfig\n\tItemList       *ItemList       `json:\"item_list\"`\n\tFilePathLister *FilePathLister `json:\"file_path_lister\"`\n\tDelimitedFiles *DelimitedFiles `json:\"delimited_files\"`\n}\n\ntype TaskJobSubmission struct {\n\tspec.RuntimeTaskJobConfig\n}\n"
  },
  {
    "path": "pkg/operator/schema/schema.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage schema\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/metrics\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/status\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\ntype InfoResponse struct {\n\tClusterConfig      clusterconfig.InternalConfig `json:\"cluster_config\" yaml:\"cluster_config\"`\n\tWorkerNodeInfos    []WorkerNodeInfo             `json:\"worker_node_infos\" yaml:\"worker_node_infos\"`\n\tOperatorNodeInfos  []NodeInfo                   `json:\"operator_node_infos\" yaml:\"operator_node_infos\"`\n\tNumPendingReplicas int                          `json:\"num_pending_replicas\" yaml:\"num_pending_replicas\"`\n}\n\ntype WorkerNodeInfo struct {\n\tNodeInfo\n\tName                 string             `json:\"name\" yaml:\"name\"`\n\tNumReplicas          int                `json:\"num_replicas\" yaml:\"num_replicas\"`\n\tNumEnqueuerReplicas  int                `json:\"num_enqueuer_replicas\" yaml:\"num_enqueuer_replicas\"`\n\tComputeUserCapacity  userconfig.Compute `json:\"compute_user_capacity\" yaml:\"compute_user_capacity\"`   // the total resources available to the user on a node\n\tComputeAvailable     userconfig.Compute `json:\"compute_available\" yaml:\"compute_unavailable\"`         // unused resources on a node\n\tComputeUserRequested userconfig.Compute `json:\"compute_user_requested\" yaml:\"compute_user_requested\"` // total resources requested by user on a node\n}\n\ntype NodeInfo struct {\n\tNodeGroupName string  `json:\"nodegroup_name\" yaml:\"nodegroup_name\"`\n\tInstanceType  string  `json:\"instance_type\" yaml:\"instance_type\"`\n\tIsSpot        bool    `json:\"is_spot\" yaml:\"is_spot\"`\n\tPrice         float64 `json:\"price\" yaml:\"price\"`\n}\n\ntype DeployResult struct {\n\tAPI     *APIResponse `json:\"api\" yaml:\"api\"`\n\tMessage string       `json:\"message\" yaml:\"message\"`\n\tError   string       `json:\"error\" yaml:\"error\"`\n}\n\ntype APIResponse struct {\n\tSpec                      *spec.API               `json:\"spec,omitempty\" yaml:\"spec,omitempty\"`\n\tMetadata                  *spec.Metadata          `json:\"metadata,omitempty\"  yaml:\"metadata,omitempty\"`\n\tStatus                    *status.Status          `json:\"status,omitempty\"  yaml:\"status,omitempty\"`\n\tNumTrafficSplitterTargets *int32                  `json:\"num_traffic_splitter_targets,omitempty\" yaml:\"num_traffic_splitter_targets,omitempty\"`\n\tEndpoint                  *string                 `json:\"endpoint,omitempty\"  yaml:\"endpoint,omitempty\"`\n\tDashboardURL              *string                 `json:\"dashboard_url,omitempty\"  yaml:\"dashboard_url,omitempty\"`\n\tBatchJobStatuses          []status.BatchJobStatus `json:\"batch_job_statuses,omitempty\"  yaml:\"batch_job_statuses,omitempty\"`\n\tTaskJobStatuses           []status.TaskJobStatus  `json:\"task_job_statuses,omitempty\"  yaml:\"task_job_statuses,omitempty\"`\n\tAPIVersions               []APIVersion            `json:\"api_versions,omitempty\"  yaml:\"api_versions,omitempty\"`\n}\n\ntype LogResponse struct {\n\tLogURL string `json:\"log_url\"`\n}\n\ntype BatchJobResponse struct {\n\tAPISpec   spec.API              `json:\"api_spec\" yaml:\"api_spec\"`\n\tJobStatus status.BatchJobStatus `json:\"job_status\" yaml:\"job_status\"`\n\tMetrics   *metrics.BatchMetrics `json:\"metrics,omitempty\" yaml:\"metrics,omitempty\"`\n\tEndpoint  string                `json:\"endpoint\" yaml:\"endpoint\"`\n}\n\ntype TaskJobResponse struct {\n\tAPISpec   spec.API             `json:\"api_spec\" yaml:\"api_spec\"`\n\tJobStatus status.TaskJobStatus `json:\"job_status\" yaml:\"job_status\"`\n\tEndpoint  string               `json:\"endpoint\" yaml:\"endpoint\"`\n}\n\ntype DeleteResponse struct {\n\tMessage string `json:\"message\"`\n}\n\ntype RefreshResponse struct {\n\tMessage string `json:\"message\"`\n}\n\ntype ErrorResponse struct {\n\tKind    string `json:\"kind\"`\n\tMessage string `json:\"message\"`\n}\n\ntype APIVersion struct {\n\tAPIID       string `json:\"api_id\" yaml:\"api_id\"`\n\tLastUpdated int64  `json:\"last_updated\" yaml:\"last_updated\"`\n}\n\ntype VerifyCortexResponse struct{}\n\nfunc (ir InfoResponse) GetNodesWithNodeGroupName(ngName string) []WorkerNodeInfo {\n\tnodesInfo := []WorkerNodeInfo{}\n\tfor _, nodeInfo := range ir.WorkerNodeInfos {\n\t\tif nodeInfo.NodeGroupName == ngName {\n\t\t\tnodesInfo = append(nodesInfo, nodeInfo)\n\t\t}\n\t}\n\treturn nodesInfo\n}\n"
  },
  {
    "path": "pkg/probe/handler.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage probe\n\nimport (\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n)\n\nfunc IsRequestKubeletProbe(r *http.Request) bool {\n\treturn strings.HasPrefix(r.Header.Get(consts.UserAgentKey), consts.KubeProbeUserAgentPrefix)\n}\n"
  },
  {
    "path": "pkg/probe/handler_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage probe_test\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/url\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/probe\"\n\t\"github.com/stretchr/testify/require\"\n\tkcore \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nfunc generateHandler(pb *probe.Probe) http.HandlerFunc {\n\treturn func(w http.ResponseWriter, r *http.Request) {\n\t\tif !pb.IsHealthy() {\n\t\t\tw.WriteHeader(http.StatusInternalServerError)\n\t\t\t_, _ = w.Write([]byte(\"unhealthy\"))\n\t\t\treturn\n\t\t}\n\n\t\tw.WriteHeader(http.StatusOK)\n\t\t_, _ = w.Write([]byte(\"healthy\"))\n\t}\n}\n\nfunc TestHandlerSuccessTCP(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\tvar userHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {\n\t\tw.WriteHeader(http.StatusOK)\n\t}\n\tserver := httptest.NewServer(userHandler)\n\n\tpb := probe.NewDefaultProbe(server.URL, log)\n\thandler := generateHandler(pb)\n\n\tr := httptest.NewRequest(http.MethodGet, \"http://fake.cortex.dev/healthz\", nil)\n\tw := httptest.NewRecorder()\n\n\tstopper := pb.StartProbing()\n\tdefer func() {\n\t\tstopper <- struct{}{}\n\t}()\n\n\tfor {\n\t\tif pb.HasRunOnce() {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t}\n\n\thandler(w, r)\n\n\trequire.Equal(t, http.StatusOK, w.Code)\n\trequire.Equal(t, \"healthy\", w.Body.String())\n}\n\nfunc TestHandlerSuccessHTTP(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\theaders := []kcore.HTTPHeader{\n\t\t{\n\t\t\tName:  \"X-Cortex-Blah\",\n\t\t\tValue: \"Blah\",\n\t\t},\n\t}\n\n\tvar userHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {\n\t\trequire.True(t, probe.IsRequestKubeletProbe(r))\n\t\tfor _, header := range headers {\n\t\t\trequire.Equal(t, header.Value, r.Header.Get(header.Name))\n\t\t}\n\n\t\tw.WriteHeader(http.StatusOK)\n\t}\n\tserver := httptest.NewServer(userHandler)\n\ttargetURL, err := url.Parse(server.URL)\n\trequire.NoError(t, err)\n\n\tpb := probe.NewProbe(\n\t\t&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tHTTPGet: &kcore.HTTPGetAction{\n\t\t\t\t\tPath:        \"/\",\n\t\t\t\t\tPort:        intstr.FromString(targetURL.Port()),\n\t\t\t\t\tHost:        targetURL.Hostname(),\n\t\t\t\t\tHTTPHeaders: headers,\n\t\t\t\t},\n\t\t\t},\n\t\t\tTimeoutSeconds:   3,\n\t\t\tPeriodSeconds:    1,\n\t\t\tSuccessThreshold: 1,\n\t\t\tFailureThreshold: 3,\n\t\t}, log,\n\t)\n\thandler := generateHandler(pb)\n\n\tr := httptest.NewRequest(http.MethodGet, \"http://fake.cortex.dev/healthz\", nil)\n\tw := httptest.NewRecorder()\n\n\tstopper := pb.StartProbing()\n\tdefer func() {\n\t\tstopper <- struct{}{}\n\t}()\n\n\tfor {\n\t\tif pb.HasRunOnce() {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t}\n\n\thandler(w, r)\n\n\trequire.Equal(t, http.StatusOK, w.Code)\n\trequire.Equal(t, \"healthy\", w.Body.String())\n}\n"
  },
  {
    "path": "pkg/probe/probe.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage probe\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"go.uber.org/zap\"\n\tkcore \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nconst (\n\t_defaultInitialDelaySeconds = int32(1)\n\t_defaultTimeoutSeconds      = int32(1)\n\t_defaultPeriodSeconds       = int32(1)\n\t_defaultSuccessThreshold    = int32(1)\n\t_defaultFailureThreshold    = int32(1)\n)\n\ntype Probe struct {\n\t*kcore.Probe\n\tsync.RWMutex\n\tlogger     *zap.SugaredLogger\n\thealthy    bool\n\thasRunOnce bool\n}\n\nfunc NewProbe(probe *kcore.Probe, logger *zap.SugaredLogger) *Probe {\n\treturn &Probe{\n\t\tProbe:  probe,\n\t\tlogger: logger,\n\t}\n}\n\nfunc NewDefaultProbe(target string, logger *zap.SugaredLogger) *Probe {\n\ttargetURL, err := url.Parse(target)\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"failed to parse target URL: %v\", err))\n\t}\n\n\treturn &Probe{\n\t\tProbe: &kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tTCPSocket: &kcore.TCPSocketAction{\n\t\t\t\t\tPort: intstr.FromString(targetURL.Port()),\n\t\t\t\t\tHost: targetURL.Hostname(),\n\t\t\t\t},\n\t\t\t},\n\t\t\tInitialDelaySeconds: _defaultInitialDelaySeconds,\n\t\t\tTimeoutSeconds:      _defaultTimeoutSeconds,\n\t\t\tPeriodSeconds:       _defaultPeriodSeconds,\n\t\t\tSuccessThreshold:    _defaultSuccessThreshold,\n\t\t\tFailureThreshold:    _defaultFailureThreshold,\n\t\t},\n\t\tlogger: logger,\n\t}\n}\n\nfunc (p *Probe) StartProbing() chan struct{} {\n\tstop := make(chan struct{})\n\n\ttime.AfterFunc(time.Duration(p.InitialDelaySeconds)*time.Second, func() {\n\t\tticker := time.NewTicker(time.Duration(p.PeriodSeconds) * time.Second)\n\n\t\tsuccessCount := int32(0)\n\t\tfailureCount := int32(0)\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-stop:\n\t\t\t\treturn\n\t\t\tcase <-ticker.C:\n\t\t\t\thealthy := p.probeContainer()\n\t\t\t\tif healthy {\n\t\t\t\t\tsuccessCount++\n\t\t\t\t\tfailureCount = 0\n\t\t\t\t} else {\n\t\t\t\t\tfailureCount++\n\t\t\t\t\tsuccessCount = 0\n\t\t\t\t}\n\n\t\t\t\tp.Lock()\n\n\t\t\t\tif successCount >= p.SuccessThreshold {\n\t\t\t\t\tp.healthy = true\n\t\t\t\t} else if failureCount >= p.FailureThreshold {\n\t\t\t\t\tp.healthy = false\n\t\t\t\t}\n\t\t\t\tp.hasRunOnce = true\n\n\t\t\t\tp.Unlock()\n\t\t\t}\n\t\t}\n\t})\n\n\treturn stop\n}\n\nfunc (p *Probe) IsHealthy() bool {\n\tp.RLock()\n\tdefer p.RUnlock()\n\n\treturn p.healthy\n}\n\nfunc (p *Probe) HasRunOnce() bool {\n\tp.RLock()\n\tdefer p.RUnlock()\n\n\treturn p.hasRunOnce\n}\n\nfunc AreProbesHealthy(probes []*Probe) bool {\n\tfor _, probe := range probes {\n\t\tif probe == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif !probe.IsHealthy() {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc (p *Probe) probeContainer() bool {\n\tvar err error\n\tvar probeType string\n\n\tswitch {\n\tcase p.HTTPGet != nil:\n\t\terr = p.httpProbe()\n\t\tprobeType = \"http\"\n\tcase p.TCPSocket != nil:\n\t\terr = p.tcpProbe()\n\t\tprobeType = \"tcp\"\n\tcase p.Exec != nil:\n\t\t// Should never be reachable.\n\t\tp.logger.Error(\"exec probe not supported\")\n\t\treturn false\n\tdefault:\n\t\tp.logger.Warn(\"no probe found\")\n\t\treturn false\n\t}\n\n\tif err != nil {\n\t\tp.logger.Warn(errors.Wrapf(err, \"%s probe to user container failed\", probeType))\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc (p *Probe) httpProbe() error {\n\t// to mimic k8s probe functionality\n\ttargetHost := p.HTTPGet.Host\n\tif p.HTTPGet.Host == \"\" {\n\t\ttargetHost = \"localhost\"\n\t}\n\n\ttargetURL := s.EnsurePrefix(\n\t\tnet.JoinHostPort(targetHost, p.HTTPGet.Port.String())+s.EnsurePrefix(p.HTTPGet.Path, \"/\"),\n\t\t\"http://\",\n\t)\n\n\thttpClient := &http.Client{\n\t\tTimeout: time.Duration(p.TimeoutSeconds) * time.Second,\n\t}\n\treq, err := http.NewRequest(http.MethodGet, targetURL, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(consts.UserAgentKey, consts.KubeProbeUserAgentPrefix)\n\n\tfor _, header := range p.HTTPGet.HTTPHeaders {\n\t\treq.Header.Add(header.Name, header.Value)\n\t}\n\n\tres, err := httpClient.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdefer func() {\n\t\t// Ensure body is both read _and_ closed so it can be reused for keep-alive.\n\t\t// No point handling errors, connection just won't be reused.\n\t\t_, _ = io.Copy(ioutil.Discard, res.Body)\n\t\t_ = res.Body.Close()\n\t}()\n\n\t// response status code between 200-399 indicates success\n\tif !(res.StatusCode >= 200 && res.StatusCode < 400) {\n\t\treturn fmt.Errorf(\"HTTP probe did not respond Ready, got status code: %d\", res.StatusCode)\n\t}\n\n\treturn nil\n}\n\nfunc (p *Probe) tcpProbe() error {\n\t// to mimic k8s probe functionality\n\ttargetHost := p.TCPSocket.Host\n\tif p.TCPSocket.Host == \"\" {\n\t\ttargetHost = \"localhost\"\n\t}\n\n\ttimeout := time.Duration(p.TimeoutSeconds) * time.Second\n\taddress := net.JoinHostPort(targetHost, p.TCPSocket.Port.String())\n\tconn, err := net.DialTimeout(\"tcp\", address, timeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_ = conn.Close()\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/probe/probe_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage probe_test\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/url\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/probe\"\n\t\"github.com/stretchr/testify/require\"\n\t\"go.uber.org/zap\"\n\tkcore \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nfunc newLogger(t *testing.T) *zap.SugaredLogger {\n\tt.Helper()\n\n\tconfig := zap.NewDevelopmentConfig()\n\tconfig.Level = zap.NewAtomicLevelAt(zap.FatalLevel)\n\tlogger, err := config.Build()\n\trequire.NoError(t, err)\n\n\tlog := logger.Sugar()\n\n\treturn log\n}\n\nfunc TestDefaultProbeSuccess(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\tvar handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {\n\t\tw.WriteHeader(http.StatusOK)\n\t}\n\tserver := httptest.NewServer(handler)\n\tpb := probe.NewDefaultProbe(server.URL, log)\n\n\tstopper := pb.StartProbing()\n\tdefer func() {\n\t\tstopper <- struct{}{}\n\t}()\n\n\tfor {\n\t\tif pb.HasRunOnce() {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t}\n\n\trequire.True(t, pb.IsHealthy())\n}\n\nfunc TestDefaultProbeFailure(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\ttarget := \"http://127.0.0.1:12345\"\n\tpb := probe.NewDefaultProbe(target, log)\n\n\tstopper := pb.StartProbing()\n\tdefer func() {\n\t\tstopper <- struct{}{}\n\t}()\n\n\tfor {\n\t\tif pb.HasRunOnce() {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t}\n\n\trequire.False(t, pb.IsHealthy())\n}\n\nfunc TestProbeHTTPFailure(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\tpb := probe.NewProbe(\n\t\t&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tHTTPGet: &kcore.HTTPGetAction{\n\t\t\t\t\tPath: \"/healthz\",\n\t\t\t\t\tPort: intstr.FromString(\"12345\"),\n\t\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tInitialDelaySeconds: 1,\n\t\t\tTimeoutSeconds:      3,\n\t\t\tPeriodSeconds:       1,\n\t\t\tSuccessThreshold:    1,\n\t\t\tFailureThreshold:    1,\n\t\t}, log,\n\t)\n\n\tstopper := pb.StartProbing()\n\tdefer func() {\n\t\tstopper <- struct{}{}\n\t}()\n\n\tfor {\n\t\tif pb.HasRunOnce() {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t}\n\n\trequire.False(t, pb.IsHealthy())\n}\n\nfunc TestProbeHTTPSuccess(t *testing.T) {\n\tt.Parallel()\n\n\tlog := newLogger(t)\n\tdefer func() { _ = log.Sync() }()\n\n\tvar handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {\n\t\tw.WriteHeader(http.StatusOK)\n\t}\n\tserver := httptest.NewServer(handler)\n\ttargetURL, err := url.Parse(server.URL)\n\trequire.NoError(t, err)\n\n\tpb := probe.NewProbe(\n\t\t&kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tHTTPGet: &kcore.HTTPGetAction{\n\t\t\t\t\tPath: \"/healthz\",\n\t\t\t\t\tPort: intstr.FromString(targetURL.Port()),\n\t\t\t\t\tHost: targetURL.Hostname(),\n\t\t\t\t},\n\t\t\t},\n\t\t\tInitialDelaySeconds: 1,\n\t\t\tTimeoutSeconds:      3,\n\t\t\tPeriodSeconds:       1,\n\t\t\tSuccessThreshold:    1,\n\t\t\tFailureThreshold:    1,\n\t\t}, log,\n\t)\n\n\tstopper := pb.StartProbing()\n\tdefer func() {\n\t\tstopper <- struct{}{}\n\t}()\n\n\tfor {\n\t\tif pb.HasRunOnce() {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(time.Second)\n\t}\n\n\trequire.True(t, pb.IsHealthy())\n}\n"
  },
  {
    "path": "pkg/proxy/breaker.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage proxy\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\n\t\"go.uber.org/atomic\"\n)\n\nvar (\n\t// ErrRequestQueueFull indicates the breaker queue depth was exceeded.\n\tErrRequestQueueFull = errors.New(\"pending request queue full\")\n)\n\n// BreakerParams defines the parameters of the breaker.\ntype BreakerParams struct {\n\tQueueDepth      int\n\tMaxConcurrency  int\n\tInitialCapacity int\n}\n\n// Breaker is a component that enforces a concurrency limit on the\n// execution of a function. It also maintains a queue of function\n// executions in excess of the concurrency limit. Function call attempts\n// beyond the limit of the queue are failed immediately.\ntype Breaker struct {\n\tinFlight   atomic.Int64\n\ttotalSlots int64\n\tsem        *semaphore\n\n\t// release is the callback function returned to callers by Reserve to\n\t// allow the reservation made by Reserve to be released.\n\trelease func()\n}\n\n// NewBreaker creates a Breaker with the desired queue depth,\n// concurrency limit and initial capacity.\nfunc NewBreaker(params BreakerParams) *Breaker {\n\tif params.QueueDepth <= 0 {\n\t\tpanic(fmt.Sprintf(\"Queue depth must be greater than 0. Got %v.\", params.QueueDepth))\n\t}\n\tif params.MaxConcurrency < 0 {\n\t\tpanic(fmt.Sprintf(\"Max concurrency must be 0 or greater. Got %v.\", params.MaxConcurrency))\n\t}\n\tif params.InitialCapacity < 0 || params.InitialCapacity > params.MaxConcurrency {\n\t\tpanic(fmt.Sprintf(\"Initial capacity must be between 0 and max concurrency. Got %v.\", params.InitialCapacity))\n\t}\n\n\tb := &Breaker{\n\t\ttotalSlots: int64(params.QueueDepth + params.MaxConcurrency),\n\t\tsem:        newSemaphore(params.MaxConcurrency, params.InitialCapacity),\n\t}\n\n\t// Allocating the closure returned by Reserve here avoids an allocation in Reserve.\n\tb.release = func() {\n\t\tb.sem.release()\n\t\tb.releasePending()\n\t}\n\n\treturn b\n}\n\n// tryAcquirePending tries to acquire a slot on the pending \"queue\".\nfunc (b *Breaker) tryAcquirePending() bool {\n\t// This is an atomic version of:\n\t//\n\t// if inFlight == totalSlots {\n\t//   return false\n\t// } else {\n\t//   inFlight++\n\t//   return true\n\t// }\n\t//\n\t// We can't just use an atomic increment as we need to check if we're\n\t// \"allowed\" to increment first. Since a Load and a CompareAndSwap are\n\t// not done atomically, we need to retry until the CompareAndSwap succeeds\n\t// (it fails if we're raced to it) or if we don't fulfill the condition\n\t// anymore.\n\tfor {\n\t\tcur := b.inFlight.Load()\n\t\tif cur == b.totalSlots {\n\t\t\treturn false\n\t\t}\n\t\tif b.inFlight.CAS(cur, cur+1) {\n\t\t\treturn true\n\t\t}\n\t}\n}\n\n// releasePending releases a slot on the pending \"queue\".\nfunc (b *Breaker) releasePending() {\n\tb.inFlight.Dec()\n}\n\n// Reserve reserves an execution slot in the breaker, to permit\n// richer semantics in the caller.\n// The caller on success must execute the callback when done with work.\nfunc (b *Breaker) Reserve(_ context.Context) (func(), bool) {\n\tif !b.tryAcquirePending() {\n\t\treturn nil, false\n\t}\n\n\tif !b.sem.tryAcquire() {\n\t\tb.releasePending()\n\t\treturn nil, false\n\t}\n\n\treturn b.release, true\n}\n\n// Maybe conditionally executes thunk based on the Breaker concurrency\n// and queue parameters. If the concurrency limit and queue capacity are\n// already consumed, Maybe returns immediately without calling thunk. If\n// the thunk was executed, Maybe returns true, else false.\nfunc (b *Breaker) Maybe(ctx context.Context, thunk func()) error {\n\tif !b.tryAcquirePending() {\n\t\treturn ErrRequestQueueFull\n\t}\n\n\tdefer b.releasePending()\n\n\t// Wait for capacity in the active queue.\n\tif err := b.sem.acquire(ctx); err != nil {\n\t\treturn err\n\t}\n\t// Defer releasing capacity in the active.\n\t// It's safe to ignore the error returned by release since we\n\t// make sure the semaphore is only manipulated here and acquire\n\t// + release calls are equally paired.\n\tdefer b.sem.release()\n\n\t// Do the thing.\n\tthunk()\n\t// Report success\n\treturn nil\n}\n\n// InFlight returns the number of requests currently in flight in this breaker.\nfunc (b *Breaker) InFlight() int64 {\n\treturn b.inFlight.Load()\n}\n\n// UpdateConcurrency updates the maximum number of in-flight requests.\nfunc (b *Breaker) UpdateConcurrency(size int) {\n\tb.sem.updateCapacity(size)\n}\n\n// UpdateQueueLength updates the number of allowed requests in-queue\nfunc (b *Breaker) UpdateQueueLength(size int) {\n\tb.totalSlots = int64(b.sem.Capacity() + size)\n}\n\n// Capacity returns the number of allowed in-flight requests on this breaker.\nfunc (b *Breaker) Capacity() int {\n\treturn b.sem.Capacity()\n}\n\nfunc (b *Breaker) QueueLength() int64 {\n\treturn b.totalSlots - int64(b.sem.Capacity())\n}\n\n// newSemaphore creates a semaphore with the desired initial capacity.\nfunc newSemaphore(maxCapacity, initialCapacity int) *semaphore {\n\tqueue := make(chan struct{}, maxCapacity)\n\tsem := &semaphore{queue: queue}\n\tsem.updateCapacity(initialCapacity)\n\treturn sem\n}\n\n// semaphore is an implementation of a semaphore based on packed integers and a channel.\n// state is an uint64 that has two uint32s packed into it: capacity and inFlight. The\n// former specifies how many request are allowed at any given time into the semaphore\n// while the latter refers to the currently in-flight requests.\n// Packing them both into one uint64 allows us to optimize access semantics using atomic\n// operations, which can't be guaranteed on 2 individual values.\n// The channel is merely used as a vehicle to be able to \"wake up\" individual goroutines\n// if capacity becomes free. It's not consistently used in accordance to actual capacity\n// but is rather a communication vehicle to ensure waiting routines are properly woken\n// up.\ntype semaphore struct {\n\tstate atomic.Uint64\n\tqueue chan struct{}\n}\n\n// tryAcquire receives a token from the semaphore if there is one otherwise returns false.\nfunc (s *semaphore) tryAcquire() bool {\n\tfor {\n\t\told := s.state.Load()\n\t\tcapacity, in := unpack(old)\n\t\tif in >= capacity {\n\t\t\treturn false\n\t\t}\n\t\tin++\n\t\tif s.state.CAS(old, pack(capacity, in)) {\n\t\t\treturn true\n\t\t}\n\t}\n}\n\n// acquire acquires capacity from the semaphore.\nfunc (s *semaphore) acquire(ctx context.Context) error {\n\tfor {\n\t\told := s.state.Load()\n\t\tcapacity, in := unpack(old)\n\n\t\tif in >= capacity {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn ctx.Err()\n\t\t\tcase <-s.queue:\n\t\t\t}\n\t\t\t// Force reload state.\n\t\t\tcontinue\n\t\t}\n\n\t\tin++\n\t\tif s.state.CAS(old, pack(capacity, in)) {\n\t\t\treturn nil\n\t\t}\n\t}\n}\n\n// release releases capacity in the semaphore.\n// If the semaphore capacity was reduced in between and as a result inFlight is greater\n// than capacity, we don't wake up goroutines as they'd not get any capacity anyway.\nfunc (s *semaphore) release() {\n\tfor {\n\t\told := s.state.Load()\n\t\tcapacity, in := unpack(old)\n\n\t\tif in == 0 {\n\t\t\tpanic(\"release and acquire are not paired\")\n\t\t}\n\n\t\tin--\n\t\tif s.state.CAS(old, pack(capacity, in)) {\n\t\t\tif in < capacity {\n\t\t\t\tselect {\n\t\t\t\tcase s.queue <- struct{}{}:\n\t\t\t\tdefault:\n\t\t\t\t\t// We generate more wakeups than we might need as we don't know\n\t\t\t\t\t// how many goroutines are waiting here. It is therefore okay\n\t\t\t\t\t// to drop the poke on the floor here as this case would mean we\n\t\t\t\t\t// have enough wakeups to wake up as many goroutines as this semaphore\n\t\t\t\t\t// can take, which is guaranteed to be enough.\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// updateCapacity updates the capacity of the semaphore to the desired size.\nfunc (s *semaphore) updateCapacity(size int) {\n\ts64 := uint64(size)\n\tfor {\n\t\told := s.state.Load()\n\t\tcapacity, in := unpack(old)\n\n\t\tif capacity == s64 {\n\t\t\t// Nothing to do, exit early.\n\t\t\treturn\n\t\t}\n\n\t\tif s.state.CAS(old, pack(s64, in)) {\n\t\t\tif s64 > capacity {\n\t\t\t\tfor i := uint64(0); i < s64-capacity; i++ {\n\t\t\t\t\tselect {\n\t\t\t\t\tcase s.queue <- struct{}{}:\n\t\t\t\t\tdefault:\n\t\t\t\t\t\t// See comment in `release` for explanation of this case.\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Capacity is the capacity of the semaphore.\nfunc (s *semaphore) Capacity() int {\n\tcapacity, _ := unpack(s.state.Load())\n\treturn int(capacity)\n}\n\n// unpack takes an uint64 and returns two uint32 (as uint64) comprised of the leftmost\n// and the rightmost bits respectively.\nfunc unpack(in uint64) (uint64, uint64) {\n\treturn in >> 32, in & 0xffffffff\n}\n\n// pack takes two uint32 (as uint64 to avoid casting) and packs them into a single uint64\n// at the leftmost and the rightmost bits respectively.\n// It's up to the caller to ensure that left and right actually fit into 32 bit.\nfunc pack(left, right uint64) uint64 {\n\treturn left<<32 | right\n}\n"
  },
  {
    "path": "pkg/proxy/breaker_test.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage proxy\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/require\"\n\t\"go.uber.org/atomic\"\n)\n\nconst (\n\t// semAcquireTimeout is a timeout for tests that try to acquire\n\t// a token of a semaphore.\n\tsemAcquireTimeout = 10 * time.Second\n\n\t// semNoChangeTimeout is some additional wait time after a number\n\t// of acquires is reached to assert that no more acquires get through.\n\tsemNoChangeTimeout = 50 * time.Millisecond\n)\n\nfunc TestBreakerInvalidConstructor(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\toptions BreakerParams\n\t}{{\n\t\tname:    \"QueueDepth = 0\",\n\t\toptions: BreakerParams{QueueDepth: 0, MaxConcurrency: 1, InitialCapacity: 1},\n\t}, {\n\t\tname:    \"MaxConcurrency negative\",\n\t\toptions: BreakerParams{QueueDepth: 1, MaxConcurrency: -1, InitialCapacity: 1},\n\t}, {\n\t\tname:    \"InitialCapacity negative\",\n\t\toptions: BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: -1},\n\t}, {\n\t\tname:    \"InitialCapacity out-of-bounds\",\n\t\toptions: BreakerParams{QueueDepth: 1, MaxConcurrency: 5, InitialCapacity: 6},\n\t}}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tdefer func() {\n\t\t\t\tif r := recover(); r == nil {\n\t\t\t\t\tt.Error(\"Expected a panic but the code didn't panic.\")\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\tNewBreaker(test.options)\n\t\t})\n\t}\n}\n\nfunc TestBreakerReserveOverload(t *testing.T) {\n\tparams := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1}\n\tb := NewBreaker(params) // Breaker capacity = 2\n\tcb1, rr := b.Reserve(context.Background())\n\tif !rr {\n\t\tt.Fatal(\"Reserve1 failed\")\n\t}\n\t_, rr = b.Reserve(context.Background())\n\tif rr {\n\t\tt.Fatal(\"Reserve2 was an unexpected success.\")\n\t}\n\t// Release a slot.\n\tcb1()\n\t// And reserve it again.\n\tcb2, rr := b.Reserve(context.Background())\n\tif !rr {\n\t\tt.Fatal(\"Reserve2 failed\")\n\t}\n\tcb2()\n}\n\nfunc TestBreakerOverloadMixed(t *testing.T) {\n\t// This tests when reservation and maybe are intermised.\n\tparams := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1}\n\tb := NewBreaker(params) // Breaker capacity = 2\n\treqs := newRequestor(b)\n\n\t// Bring breaker to capacity.\n\treqs.request()\n\t// This happens in go-routine, so spin.\n\tfor _, in := unpack(b.sem.state.Load()); in != 1; _, in = unpack(b.sem.state.Load()) {\n\t\ttime.Sleep(time.Millisecond * 2)\n\t}\n\t_, rr := b.Reserve(context.Background())\n\tif rr {\n\t\tt.Fatal(\"Reserve was an unexpected success.\")\n\t}\n\t// Open a slot.\n\treqs.processSuccessfully(t)\n\t// Now reservation should work.\n\tcb, rr := b.Reserve(context.Background())\n\tif !rr {\n\t\tt.Fatal(\"Reserve unexpectedly failed\")\n\t}\n\t// Process the reservation.\n\tcb()\n}\n\nfunc TestBreakerOverload(t *testing.T) {\n\tparams := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1}\n\tb := NewBreaker(params) // Breaker capacity = 2\n\treqs := newRequestor(b)\n\n\t// Bring breaker to capacity.\n\treqs.request()\n\treqs.request()\n\n\t// Overshoot by one.\n\treqs.request()\n\treqs.expectFailure(t)\n\n\t// The remainer should succeed.\n\treqs.processSuccessfully(t)\n\treqs.processSuccessfully(t)\n}\n\nfunc TestBreakerQueueing(t *testing.T) {\n\tparams := BreakerParams{QueueDepth: 2, MaxConcurrency: 1, InitialCapacity: 0}\n\tb := NewBreaker(params) // Breaker capacity = 2\n\treqs := newRequestor(b)\n\n\t// Bring breaker to capacity. Doesn't error because queue subsumes these requests.\n\treqs.request()\n\treqs.request()\n\n\t// Update concurrency to allow the requests to be processed.\n\tb.UpdateConcurrency(1)\n\n\t// They should pass just fine.\n\treqs.processSuccessfully(t)\n\treqs.processSuccessfully(t)\n}\n\nfunc TestBreakerInflight(t *testing.T) {\n\tparams := BreakerParams{QueueDepth: 2, MaxConcurrency: 1, InitialCapacity: 1}\n\tb := NewBreaker(params) // Breaker capacity = 2\n\treqs := newRequestor(b)\n\n\t// Bring breaker to capacity. Doesn't error because queue subsumes these requests.\n\treqs.request()\n\treqs.request()\n\treqs.request()\n\n\trequire.Eventually(t, func() bool {\n\t\treturn b.InFlight() == int64(3)\n\t}, time.Second, 10*time.Millisecond)\n\trequire.Equal(t, reqs.InProgress.Load(), int64(1))\n}\n\nfunc TestBreakerNoOverload(t *testing.T) {\n\tparams := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1}\n\tb := NewBreaker(params) // Breaker capacity = 2\n\treqs := newRequestor(b)\n\n\t// Bring request to capacity.\n\treqs.request()\n\treqs.request()\n\n\t// Process one, send a new one in, at capacity again.\n\treqs.processSuccessfully(t)\n\treqs.request()\n\n\t// Process one, send a new one in, at capacity again.\n\treqs.processSuccessfully(t)\n\treqs.request()\n\n\t// Process the remainder successfully.\n\treqs.processSuccessfully(t)\n\treqs.processSuccessfully(t)\n}\n\nfunc TestBreakerCancel(t *testing.T) {\n\tparams := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 0}\n\tb := NewBreaker(params)\n\treqs := newRequestor(b)\n\n\t// Cancel a request which cannot get capacity.\n\tctx1, cancel1 := context.WithCancel(context.Background())\n\treqs.requestWithContext(ctx1)\n\tcancel1()\n\treqs.expectFailure(t)\n\n\t// This request cannot get capacity either. This reproduced a bug we had when\n\t// freeing slots on the pendingRequests channel.\n\tctx2, cancel2 := context.WithCancel(context.Background())\n\treqs.requestWithContext(ctx2)\n\tcancel2()\n\treqs.expectFailure(t)\n\n\t// Let through a request with capacity then timeout following request\n\tb.UpdateConcurrency(1)\n\treqs.request()\n\n\t// Exceed capacity and assert one failure. This makes sure the Breaker is consistently\n\t// at capacity.\n\treqs.request()\n\treqs.request()\n\treqs.expectFailure(t)\n\n\t// This request cannot get capacity.\n\tctx3, cancel3 := context.WithCancel(context.Background())\n\treqs.requestWithContext(ctx3)\n\tcancel3()\n\treqs.expectFailure(t)\n\n\t// The requests that were put in earlier should succeed.\n\treqs.processSuccessfully(t)\n\treqs.processSuccessfully(t)\n}\n\nfunc TestBreakerUpdateConcurrency(t *testing.T) {\n\tparams := BreakerParams{QueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 0}\n\tb := NewBreaker(params)\n\tb.UpdateConcurrency(1)\n\tif got, want := b.Capacity(), 1; got != want {\n\t\tt.Errorf(\"Capacity() = %d, want: %d\", got, want)\n\t}\n\n\tb.UpdateConcurrency(0)\n\tif got, want := b.Capacity(), 0; got != want {\n\t\tt.Errorf(\"Capacity() = %d, want: %d\", got, want)\n\t}\n}\n\n// Test empty semaphore, token cannot be acquired\nfunc TestSemaphoreAcquireHasNoCapacity(t *testing.T) {\n\tgotChan := make(chan struct{}, 1)\n\n\tsem := newSemaphore(1, 0)\n\ttryAcquire(sem, gotChan)\n\n\tselect {\n\tcase <-gotChan:\n\t\tt.Error(\"Token was acquired but shouldn't have been\")\n\tcase <-time.After(semNoChangeTimeout):\n\t\t// Test succeeds, semaphore didn't change in configured time.\n\t}\n}\n\nfunc TestSemaphoreAcquireNonBlockingHasNoCapacity(t *testing.T) {\n\tsem := newSemaphore(1, 0)\n\tif sem.tryAcquire() {\n\t\tt.Error(\"Should have failed immediately\")\n\t}\n}\n\n// Test empty semaphore, add capacity, token can be acquired\nfunc TestSemaphoreAcquireHasCapacity(t *testing.T) {\n\tgotChan := make(chan struct{}, 1)\n\twant := 1\n\n\tsem := newSemaphore(1, 0)\n\ttryAcquire(sem, gotChan)\n\tsem.updateCapacity(1) // Allows 1 acquire\n\n\tfor i := 0; i < want; i++ {\n\t\tselect {\n\t\tcase <-gotChan:\n\t\t\t// Successfully acquired a token.\n\t\tcase <-time.After(semAcquireTimeout):\n\t\t\tt.Error(\"Was not able to acquire token before timeout\")\n\t\t}\n\t}\n\n\tselect {\n\tcase <-gotChan:\n\t\tt.Errorf(\"Got more acquires than wanted, want = %d, got at least %d\", want, want+1)\n\tcase <-time.After(semNoChangeTimeout):\n\t\t// No change happened, success.\n\t}\n}\n\nfunc TestSemaphoreRelease(t *testing.T) {\n\tsem := newSemaphore(1, 1)\n\tsem.acquire(context.Background())\n\tfunc() {\n\t\tdefer func() {\n\t\t\tif e := recover(); e != nil {\n\t\t\t\tt.Error(\"Expected no panic, got message:\", e)\n\t\t\t}\n\t\t\tsem.release()\n\t\t}()\n\t}()\n\tfunc() {\n\t\tdefer func() {\n\t\t\tif e := recover(); e == nil {\n\t\t\t\tt.Error(\"Expected panic, but got none\")\n\t\t\t}\n\t\t}()\n\t\tsem.release()\n\t}()\n}\n\nfunc TestSemaphoreUpdateCapacity(t *testing.T) {\n\tconst initialCapacity = 1\n\tsem := newSemaphore(3, initialCapacity)\n\tif got, want := sem.Capacity(), 1; got != want {\n\t\tt.Errorf(\"Capacity = %d, want: %d\", got, want)\n\t}\n\tsem.acquire(context.Background())\n\tsem.updateCapacity(initialCapacity + 2)\n\tif got, want := sem.Capacity(), 3; got != want {\n\t\tt.Errorf(\"Capacity = %d, want: %d\", got, want)\n\t}\n}\n\nfunc TestPackUnpack(t *testing.T) {\n\twantL := uint64(256)\n\twantR := uint64(513)\n\n\tgotL, gotR := unpack(pack(wantL, wantR))\n\n\tif gotL != wantL || gotR != wantR {\n\t\tt.Fatalf(\"Got %d, %d want %d, %d\", gotL, gotR, wantL, wantR)\n\t}\n}\n\nfunc tryAcquire(sem *semaphore, gotChan chan struct{}) {\n\tgo func() {\n\t\t// blocking until someone puts the token into the semaphore\n\t\tsem.acquire(context.Background())\n\t\tgotChan <- struct{}{}\n\t}()\n}\n\n// requestor is a set of test helpers around breaker testing.\ntype requestor struct {\n\tbreaker    *Breaker\n\tacceptedCh chan bool\n\tbarrierCh  chan struct{}\n\tInProgress atomic.Int64\n}\n\nfunc newRequestor(breaker *Breaker) *requestor {\n\treturn &requestor{\n\t\tbreaker:    breaker,\n\t\tacceptedCh: make(chan bool),\n\t\tbarrierCh:  make(chan struct{}),\n\t}\n}\n\n// request is the same as requestWithContext but with a default context.\nfunc (r *requestor) request() {\n\tr.requestWithContext(context.Background())\n}\n\n// requestWithContext simulates a request in a separate goroutine. The\n// request will either fail immediately (as observable via expectFailure)\n// or block until processSuccessfully is called.\nfunc (r *requestor) requestWithContext(ctx context.Context) {\n\tgo func() {\n\t\terr := r.breaker.Maybe(ctx, func() {\n\t\t\tr.InProgress.Inc()\n\t\t\t<-r.barrierCh\n\t\t})\n\t\tr.acceptedCh <- err == nil\n\t}()\n}\n\n// expectFailure waits for a request to finish and asserts it to be failed.\nfunc (r *requestor) expectFailure(t *testing.T) {\n\tt.Helper()\n\tif <-r.acceptedCh {\n\t\tt.Error(\"expected request to fail but it succeeded\")\n\t}\n}\n\n// processSuccessfully allows a request to pass the barrier, waits for it to\n// be finished and asserts it to succeed.\nfunc (r *requestor) processSuccessfully(t *testing.T) {\n\tt.Helper()\n\tr.barrierCh <- struct{}{}\n\tif !<-r.acceptedCh {\n\t\tt.Error(\"expected request to succeed but it failed\")\n\t}\n}\n\nfunc BenchmarkBreakerMaybe(b *testing.B) {\n\top := func() {}\n\n\tfor _, c := range []int{1, 10, 100, 1000} {\n\t\tbreaker := NewBreaker(BreakerParams{QueueDepth: 10000000, MaxConcurrency: c, InitialCapacity: c})\n\n\t\tb.Run(fmt.Sprintf(\"%d-sequential\", c), func(b *testing.B) {\n\t\t\tfor j := 0; j < b.N; j++ {\n\t\t\t\tbreaker.Maybe(context.Background(), op)\n\t\t\t}\n\t\t})\n\n\t\tb.Run(fmt.Sprintf(\"%d-parallel\", c), func(b *testing.B) {\n\t\t\tb.RunParallel(func(pb *testing.PB) {\n\t\t\t\tfor pb.Next() {\n\t\t\t\t\tbreaker.Maybe(context.Background(), op)\n\t\t\t\t}\n\t\t\t})\n\t\t})\n\t}\n}\n\nfunc BenchmarkBreakerReserve(b *testing.B) {\n\top := func() {}\n\tbreaker := NewBreaker(BreakerParams{QueueDepth: 1, MaxConcurrency: 10000000, InitialCapacity: 10000000})\n\n\tb.Run(\"sequential\", func(b *testing.B) {\n\t\tfor j := 0; j < b.N; j++ {\n\t\t\tfree, got := breaker.Reserve(context.Background())\n\t\t\top()\n\t\t\tif got {\n\t\t\t\tfree()\n\t\t\t}\n\t\t}\n\t})\n\n\tb.Run(\"parallel\", func(b *testing.B) {\n\t\tb.RunParallel(func(pb *testing.PB) {\n\t\t\tfor pb.Next() {\n\t\t\t\tfree, got := breaker.Reserve(context.Background())\n\t\t\t\top()\n\t\t\t\tif got {\n\t\t\t\t\tfree()\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t})\n}\n"
  },
  {
    "path": "pkg/proxy/handler.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage proxy\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net/http\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/telemetry\"\n\t\"github.com/cortexlabs/cortex/pkg/probe\"\n)\n\nfunc Handler(breaker *Breaker, next http.Handler) http.HandlerFunc {\n\treturn func(w http.ResponseWriter, r *http.Request) {\n\t\tif probe.IsRequestKubeletProbe(r) || breaker == nil {\n\t\t\tnext.ServeHTTP(w, r)\n\t\t\treturn\n\t\t}\n\n\t\tif err := breaker.Maybe(r.Context(), func() {\n\t\t\tnext.ServeHTTP(w, r)\n\t\t}); err != nil {\n\t\t\tif errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRequestQueueFull) {\n\t\t\t\thttp.Error(w, err.Error(), http.StatusServiceUnavailable)\n\t\t\t} else {\n\t\t\t\tw.WriteHeader(http.StatusInternalServerError)\n\t\t\t\ttelemetry.Error(err)\n\t\t\t}\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "pkg/proxy/handler_test.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage proxy_test\n\nimport (\n\t\"context\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/proxy\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nconst (\n\tuserContainerHost = \"http://user-container.cortex.dev\"\n)\n\nfunc TestProxyHandlerQueueFull(t *testing.T) {\n\t// This test sends three requests of which one should fail immediately as the queue\n\t// saturates.\n\tresp := make(chan struct{})\n\tblockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t<-resp\n\t})\n\n\tbreaker := proxy.NewBreaker(\n\t\tproxy.BreakerParams{\n\t\t\tQueueDepth:      1,\n\t\t\tMaxConcurrency:  1,\n\t\t\tInitialCapacity: 1,\n\t\t},\n\t)\n\n\th := proxy.Handler(breaker, blockHandler)\n\n\treq := httptest.NewRequest(http.MethodGet, userContainerHost, nil)\n\tresps := make(chan *httptest.ResponseRecorder)\n\tfor i := 0; i < 3; i++ {\n\t\tgo func() {\n\t\t\trec := httptest.NewRecorder()\n\t\t\th(rec, req)\n\t\t\tresps <- rec\n\t\t}()\n\t}\n\n\t// One of the three requests fails and it should be the first we see since the others\n\t// are still held by the resp channel.\n\tfailure := <-resps\n\trequire.Equal(t, http.StatusServiceUnavailable, failure.Code)\n\trequire.True(t, strings.Contains(failure.Body.String(), \"pending request queue full\"))\n\n\t// Allow the remaining requests to pass.\n\tclose(resp)\n\tfor i := 0; i < 2; i++ {\n\t\tres := <-resps\n\t\trequire.Equal(t, http.StatusOK, res.Code)\n\t}\n}\n\nfunc TestProxyHandlerBreakerTimeout(t *testing.T) {\n\t// This test sends a request which will take a long time to complete.\n\t// Then another one with a very short context timeout.\n\t// Verifies that the second one fails with timeout.\n\tseen := make(chan struct{})\n\tresp := make(chan struct{})\n\tdefer close(resp) // Allow all requests to pass through.\n\tblockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tseen <- struct{}{}\n\t\t<-resp\n\t})\n\tbreaker := proxy.NewBreaker(proxy.BreakerParams{\n\t\tQueueDepth: 1, MaxConcurrency: 1, InitialCapacity: 1,\n\t})\n\th := proxy.Handler(breaker, blockHandler)\n\n\tgo func() {\n\t\th(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, userContainerHost, nil))\n\t}()\n\n\t// Wait until the first request has entered the handler.\n\t<-seen\n\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)\n\tdefer cancel()\n\n\trec := httptest.NewRecorder()\n\th(rec, httptest.NewRequest(http.MethodGet, userContainerHost, nil).WithContext(ctx))\n\n\trequire.Equal(t, http.StatusServiceUnavailable, rec.Code)\n\trequire.True(t, strings.Contains(rec.Body.String(), context.DeadlineExceeded.Error()))\n}\n"
  },
  {
    "path": "pkg/proxy/proxy.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage proxy\n\nimport (\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"net/url\"\n)\n\n// NewReverseProxy creates a new cortex base reverse proxy\nfunc NewReverseProxy(target string, maxIdle, maxIdlePerHost int) *httputil.ReverseProxy {\n\ttargetURL, err := url.Parse(target)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\thttpProxy := httputil.NewSingleHostReverseProxy(targetURL)\n\thttpProxy.Transport = buildHTTPTransport(maxIdle, maxIdlePerHost)\n\n\treturn httpProxy\n}\n\nfunc buildHTTPTransport(maxIdle, maxIdlePerHost int) http.RoundTripper {\n\ttransport := http.DefaultTransport.(*http.Transport).Clone()\n\ttransport.DisableKeepAlives = false\n\ttransport.MaxIdleConns = maxIdle\n\ttransport.MaxIdleConnsPerHost = maxIdlePerHost\n\ttransport.ForceAttemptHTTP2 = false\n\ttransport.DisableCompression = true\n\treturn transport\n}\n"
  },
  {
    "path": "pkg/proxy/proxy_test.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage proxy_test\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/proxy\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNewReverseProxy(t *testing.T) {\n\tvar isHandlerCalled bool\n\tvar handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {\n\t\tisHandlerCalled = true\n\t}\n\n\tserver := httptest.NewServer(handler)\n\thttpProxy := proxy.NewReverseProxy(server.URL, 1000, 1000)\n\n\tresp := httptest.NewRecorder()\n\treq := httptest.NewRequest(http.MethodPost, \"http://user-container.cortex.dev\", nil)\n\thttpProxy.ServeHTTP(resp, req)\n\n\trequire.True(t, isHandlerCalled)\n}\n"
  },
  {
    "path": "pkg/proxy/request_stats.go",
    "content": "/*\nCopyright 2018 The Knative Authors\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nModifications Copyright 2022 Cortex Labs, Inc.\n*/\n\npackage proxy\n\nimport (\n\t\"net/http\"\n\t\"sync\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promauto\"\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n)\n\ntype RequestStats struct {\n\tsync.Mutex\n\tcounts []int64\n}\n\nfunc (s *RequestStats) Append(val int64) {\n\ts.Lock()\n\tdefer s.Unlock()\n\ts.counts = append(s.counts, val)\n}\n\nfunc (s *RequestStats) GetAllAndDelete() []int64 {\n\tvar output []int64\n\ts.Lock()\n\tdefer s.Unlock()\n\toutput = s.counts\n\ts.counts = []int64{}\n\treturn output\n}\n\nfunc (s *RequestStats) Report() RequestStatsReport {\n\trequestCounts := s.GetAllAndDelete()\n\n\ttotal := 0.0\n\tif len(requestCounts) > 0 {\n\t\tfor _, val := range requestCounts {\n\t\t\ttotal += float64(val)\n\t\t}\n\n\t\ttotal /= float64(len(requestCounts))\n\t}\n\n\treturn RequestStatsReport{AvgInFlight: total}\n}\n\ntype RequestStatsReport struct {\n\tAvgInFlight float64\n}\n\ntype PrometheusStatsReporter struct {\n\thandler          http.Handler\n\tinFlightRequests prometheus.Gauge\n}\n\nfunc NewPrometheusStatsReporter() *PrometheusStatsReporter {\n\tinFlightRequestsGauge := promauto.NewGauge(prometheus.GaugeOpts{\n\t\tName: \"cortex_in_flight_requests\",\n\t\tHelp: \"The number of in-flight requests for a cortex API\",\n\t})\n\n\treturn &PrometheusStatsReporter{\n\t\thandler:          promhttp.Handler(),\n\t\tinFlightRequests: inFlightRequestsGauge,\n\t}\n}\n\nfunc (r *PrometheusStatsReporter) Report(stats RequestStatsReport) {\n\tr.inFlightRequests.Set(stats.AvgInFlight)\n}\n\nfunc (r *PrometheusStatsReporter) ServeHTTP(w http.ResponseWriter, req *http.Request) {\n\tr.handler.ServeHTTP(w, req)\n}\n"
  },
  {
    "path": "pkg/types/async/s3_paths.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage async\n\nimport (\n\t\"fmt\"\n)\n\nfunc StoragePath(clusterUID, apiName string) string {\n\treturn fmt.Sprintf(\"%s/workloads/%s\", clusterUID, apiName)\n}\n\nfunc PayloadPath(storagePath string, requestID string) string {\n\treturn fmt.Sprintf(\"%s/%s/payload\", storagePath, requestID)\n}\n\nfunc HeadersPath(storagePath string, requestID string) string {\n\treturn fmt.Sprintf(\"%s/%s/headers.json\", storagePath, requestID)\n}\n\nfunc ResultPath(storagePath string, requestID string) string {\n\treturn fmt.Sprintf(\"%s/%s/result.json\", storagePath, requestID)\n}\n\nfunc StatusPrefixPath(storagePath string, requestID string) string {\n\treturn fmt.Sprintf(\"%s/%s/status\", storagePath, requestID)\n}\n\nfunc StatusPath(storagePath string, requestID string, status Status) string {\n\treturn fmt.Sprintf(\"%s/%s\", StatusPrefixPath(storagePath, requestID), status)\n}\n"
  },
  {
    "path": "pkg/types/async/status.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage async\n\n// Status is an enum type for workload status\ntype Status string\n\n// Different possible workload status\nconst (\n\tStatusNotFound   Status = \"not_found\"\n\tStatusFailed     Status = \"failed\"\n\tStatusInProgress Status = \"in_progress\"\n\tStatusInQueue    Status = \"in_queue\"\n\tStatusCompleted  Status = \"completed\"\n)\n\nfunc (status Status) String() string {\n\treturn string(status)\n}\n\nfunc (status Status) Valid() bool {\n\tswitch status {\n\tcase StatusNotFound, StatusFailed, StatusInProgress, StatusInQueue, StatusCompleted:\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/availability_zones.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nvar _azBlacklist = strset.New(\"us-east-1e\")\n\nfunc (cc *Config) setAvailabilityZones(awsClient *aws.Client) error {\n\tif len(cc.AvailabilityZones) == 0 {\n\t\tif err := cc.setDefaultAvailabilityZones(awsClient); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\n\tif err := cc.validateUserAvailabilityZones(awsClient); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (cc *Config) setDefaultAvailabilityZones(awsClient *aws.Client) error {\n\tinstanceTypes := strset.New()\n\tfor _, ng := range cc.NodeGroups {\n\t\tinstanceTypes.Add(ng.InstanceType)\n\t}\n\tinstanceTypesSlice := instanceTypes.Slice()\n\n\tvar zones strset.Set\n\tvar err error\n\tif len(instanceTypesSlice) > 0 {\n\t\tzones, err = awsClient.ListSupportedAvailabilityZones(instanceTypesSlice[0], instanceTypesSlice[1:]...)\n\t}\n\tif len(zones) == 0 || err != nil {\n\t\t// Try without checking instance types\n\t\tzones, err = awsClient.ListAvailabilityZonesInRegion()\n\t\tif err != nil {\n\t\t\treturn nil // Let eksctl choose the availability zones\n\t\t}\n\t}\n\n\tzones.Subtract(_azBlacklist)\n\n\tif len(zones) < 2 {\n\t\treturn ErrorNotEnoughDefaultSupportedZones(awsClient.Region, zones, instanceTypesSlice[0], instanceTypesSlice[1:]...)\n\t}\n\n\t// See https://github.com/weaveworks/eksctl/blob/master/pkg/eks/api.go\n\tif awsClient.Region == \"us-east-1\" {\n\t\tzones.ShrinkSorted(2)\n\t} else {\n\t\tzones.ShrinkSorted(3)\n\t}\n\n\tcc.AvailabilityZones = zones.SliceSorted()\n\n\treturn nil\n}\n\nfunc (cc *Config) validateUserAvailabilityZones(awsClient *aws.Client) error {\n\tallZones, err := awsClient.ListAvailabilityZonesInRegion()\n\tif err != nil {\n\t\treturn nil // Skip validation\n\t}\n\n\tfor _, userZone := range cc.AvailabilityZones {\n\t\tif !allZones.Has(userZone) {\n\t\t\treturn ErrorInvalidAvailabilityZone(userZone, allZones, awsClient.Region)\n\t\t}\n\t}\n\n\tif len(cc.NodeGroups) > 0 {\n\t\tinstanceTypes := strset.New()\n\t\tfor _, ng := range cc.NodeGroups {\n\t\t\tinstanceTypes.Add(ng.InstanceType)\n\t\t}\n\t\tinstanceTypesSlice := instanceTypes.Slice()\n\n\t\tsupportedZones, err := awsClient.ListSupportedAvailabilityZones(instanceTypesSlice[0], instanceTypesSlice[1:]...)\n\t\tif err != nil {\n\t\t\t// Skip validation instance-based validation\n\t\t\tsupportedZones = strset.Difference(allZones, _azBlacklist)\n\t\t}\n\n\t\tfor _, userZone := range cc.AvailabilityZones {\n\t\t\tif !supportedZones.Has(userZone) {\n\t\t\t\treturn ErrorUnsupportedAvailabilityZone(userZone, instanceTypesSlice[0], instanceTypesSlice[1:]...)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (cc *Config) validateSubnets(awsClient *aws.Client) error {\n\tif len(cc.Subnets) == 0 {\n\t\treturn nil\n\t}\n\n\tallZones, err := awsClient.ListAvailabilityZonesInRegion()\n\tif err != nil {\n\t\treturn nil // Skip validation\n\t}\n\n\tuserZones := strset.New()\n\n\tfor i, subnetConfig := range cc.Subnets {\n\t\tif !allZones.Has(subnetConfig.AvailabilityZone) {\n\t\t\treturn errors.Wrap(ErrorInvalidAvailabilityZone(subnetConfig.AvailabilityZone, allZones, cc.Region), s.Index(i), AvailabilityZoneKey)\n\t\t}\n\t\tif userZones.Has(subnetConfig.AvailabilityZone) {\n\t\t\treturn ErrorAvailabilityZoneSpecifiedTwice(subnetConfig.AvailabilityZone)\n\t\t}\n\t\tuserZones.Add(subnetConfig.AvailabilityZone)\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/aws_policy.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"text/template\"\n\n\t\"github.com/aws/aws-sdk-go/service/iam\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n)\n\nfunc DefaultPolicyName(clusterName string, region string) string {\n\treturn fmt.Sprintf(\"cortex-%s-%s\", clusterName, region)\n}\n\nfunc DefaultPolicyARN(accountID string, clusterName string, region string) string {\n\treturn fmt.Sprintf(\"arn:%s:iam::%s:policy/%s\", aws.PartitionFromRegion(region), accountID, DefaultPolicyName(clusterName, region))\n}\n\nvar _cortexPolicy = `\n{\n\t\"Version\": \"2012-10-17\",\n\t\"Statement\": [\n\t\t{\n\t\t\t\"Action\": [\n\t\t\t\t\"sts:GetCallerIdentity\",\n\t\t\t\t\"ecr:GetAuthorizationToken\",\n\t\t\t\t\"ecr:BatchGetImage\",\n\t\t\t\t\"sqs:ListQueues\",\n\t\t\t\t\"ec2:DescribeSpotPriceHistory\"\n\t\t\t],\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Resource\": \"*\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": \"sqs:*\",\n\t\t\t\"Resource\": \"arn:*:sqs:{{ .Region }}:{{ .AccountID }}:cx_*\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": \"s3:*\",\n\t\t\t\"Resource\": \"arn:*:s3:::{{ .Bucket }}\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": \"s3:*\",\n\t\t\t\"Resource\": \"arn:*:s3:::{{ .Bucket }}/*\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": [\n\t\t\t\t\"logs:CreateLogStream\",\n\t\t\t\t\"logs:DescribeLogStreams\",\n\t\t\t\t\"logs:PutLogEvents\",\n\t\t\t\t\"logs:CreateLogGroup\"\n\t\t\t],\n\t\t\t\"Resource\": \"arn:*:logs:{{ .Region }}:{{ .AccountID }}:log-group:{{ .LogGroup }}:*\"\n\t\t},\n\t\t{\n\t\t\t\"Effect\": \"Allow\",\n\t\t\t\"Action\": \"logs:CreateLogGroup\",\n\t\t\t\"Resource\": \"arn:*:logs:{{ .Region }}:{{ .AccountID }}:log-group:{{ .LogGroup }}\"\n\t\t}\n\t]\n}\n`\n\ntype CortexPolicyTemplateArgs struct {\n\tClusterName string\n\tLogGroup    string\n\tRegion      string\n\tBucket      string\n\tAccountID   string\n}\n\nfunc CreateDefaultPolicy(awsClient *aws.Client, args CortexPolicyTemplateArgs) error {\n\tpolicyName := DefaultPolicyName(args.ClusterName, args.Region)\n\taccountID, _, err := awsClient.GetCachedAccountID()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpolicyARN := DefaultPolicyARN(accountID, args.ClusterName, args.Region)\n\tpolicyTemplate, err := template.New(\"policy\").Parse(_cortexPolicy)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to parse aws policy template\")\n\t}\n\n\tbuf := &bytes.Buffer{}\n\terr = policyTemplate.Execute(buf, args)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to execute aws policy template\")\n\t}\n\n\tcompactBuf := &bytes.Buffer{}\n\n\terr = json.Compact(compactBuf, buf.Bytes())\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to parse and remove whitespace from aws policy json\")\n\t}\n\n\tpolicyDocument := compactBuf.String()\n\n\t_, err = awsClient.IAM().CreatePolicy(&iam.CreatePolicyInput{\n\t\tPolicyDocument: &policyDocument,\n\t\tPolicyName:     &policyName,\n\t})\n\tif err != nil {\n\t\tif aws.IsErrCode(err, iam.ErrCodeEntityAlreadyExistsException) {\n\t\t\terr := AddNewPolicyVersion(awsClient, policyARN, policyDocument)\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, \"failed to create iam policy for cortex\")\n\t\t\t}\n\t\t} else {\n\t\t\treturn errors.Wrap(err, \"failed to create iam policy for cortex\")\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc AddNewPolicyVersion(awsClient *aws.Client, policyARN string, policyDocument string) error {\n\tpolicies, err := awsClient.IAM().ListPolicyVersions(&iam.ListPolicyVersionsInput{\n\t\tPolicyArn: &policyARN,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif len(policies.Versions) == 0 {\n\t\treturn errors.ErrorUnexpected(\"encountered a policy without any policy versions\")\n\t}\n\n\tnumPolicies := len(policies.Versions)\n\toldestPolicy := *policies.Versions[0]\n\n\tfor _, policyPtr := range policies.Versions {\n\t\tpolicy := *policyPtr\n\n\t\tif policy.CreateDate.Before(*oldestPolicy.CreateDate) && !*policy.IsDefaultVersion {\n\t\t\toldestPolicy = policy\n\t\t}\n\t}\n\n\t// can only have a max of 5 versions, so delete the oldest non-default version before adding a new policy version\n\tif numPolicies > 4 {\n\t\t_, err := awsClient.IAM().DeletePolicyVersion(&iam.DeletePolicyVersionInput{\n\t\t\tPolicyArn: &policyARN,\n\t\t\tVersionId: oldestPolicy.VersionId,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn errors.WithStack(err)\n\t\t}\n\t}\n\n\t_, err = awsClient.IAM().CreatePolicyVersion(&iam.CreatePolicyVersionInput{\n\t\tSetAsDefault:   pointer.Bool(true),\n\t\tPolicyDocument: &policyDocument,\n\t\tPolicyArn:      &policyARN,\n\t})\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\treturn nil\n\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/cluster_config.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\nimport (\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"math\"\n\t\"net\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils\"\n\t\"github.com/aws/aws-sdk-go/service/cloudformation\"\n\t\"github.com/aws/aws-sdk-go/service/iam\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\tcr \"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tlibhash \"github.com/cortexlabs/cortex/pkg/lib/hash\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\tlibmath \"github.com/cortexlabs/cortex/pkg/lib/math\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\tlibstr \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/structs\"\n\t\"github.com/cortexlabs/yaml\"\n)\n\nconst (\n\t// MaxNodeGroups represents the max number of node groups in a cluster\n\tMaxNodeGroups = 100\n\n\t// MaxNodesToAddOnClusterUp represents the max number of nodes to add on cluster up\n\t// Limited to 200 nodes (rounded down from 248 nodes) for two reasons:\n\t//\n\t// * To prevent overloading the API servers when the nodes are being added.\n\t//\n\t// * To prevent hitting the 500 targets per LB (when the cross-load balancing is enabled) limit (quota code L-B211E961);\n\t//   500 divided by 2 target listeners - 1 operator node - 1 prometheus node => 248\n\tMaxNodesToAddOnClusterUp = 200\n\n\t// MaxNodesToAddOnClusterConfigure represents the max number of nodes to add on cluster up/configure\n\tMaxNodesToAddOnClusterConfigure = 100\n\t// ClusterNameTag is the tag used for storing a cluster's name in AWS resources\n\tClusterNameTag = \"cortex.dev/cluster-name\"\n\t// SQSQueueDelimiter is the delimiter character used for naming cortex SQS queues (e.g. cx_<cluster_hash>_b_<api_name>_<jon_id>)\n\t// In this case, _ was chosen to simplify the retrieval of information for the queue's name,\n\t// since the api naming scheme does not allow this character.\n\tSQSQueueDelimiter = \"_\"\n)\n\nvar (\n\t_operatorNodeGroupInstanceType = \"t3.medium\"\n\n\t_maxNodeGroupLengthWithPrefix = 32\n\t_maxNodeGroupLength           = _maxNodeGroupLengthWithPrefix - len(\"cx-wd-\") // or cx-ws-\n\t_maxInstancePools             = 20\n\t_defaultIAMPolicies           = []string{\"arn:aws:iam::aws:policy/AmazonS3FullAccess\"}\n\t_invalidTagPrefixes           = []string{\"kubernetes.io/\", \"k8s.io/\", \"eksctl.\", \"alpha.eksctl.\", \"beta.eksctl.\", \"aws:\", \"Aws:\", \"aWs:\", \"awS:\", \"aWS:\", \"AwS:\", \"aWS:\", \"AWS:\"}\n\n\t_smallestIOPSForIO1VolumeType = int64(100)\n\t_highestIOPSForIO1VolumeType  = int64(64000)\n\t_smallestIOPSForGP3VolumeType = int64(3000)\n\t_highestIOPSForGP3VolumeType  = int64(16000)\n\n\t_maxIOPSToVolumeSizeRatioForIO1 = int64(50)\n\t_maxIOPSToVolumeSizeRatioForGP3 = int64(500)\n\t_minIOPSToThroughputRatioForGP3 = int64(4)\n\n\t_minSubnetMask = 16\n\t_maxSubnetMask = 24\n\n\t// This regex is stricter than the actual S3 rules\n\t_strictS3BucketRegex = regexp.MustCompile(`^([a-z0-9])+(-[a-z0-9]+)*$`)\n)\n\ntype CoreConfig struct {\n\tClusterName            string `json:\"cluster_name\" yaml:\"cluster_name\"`\n\tRegion                 string `json:\"region\" yaml:\"region\"`\n\tPrometheusInstanceType string `json:\"prometheus_instance_type\" yaml:\"prometheus_instance_type\"`\n\n\tImageOperator                   string `json:\"image_operator\" yaml:\"image_operator\"`\n\tImageControllerManager          string `json:\"image_controller_manager\" yaml:\"image_controller_manager\"`\n\tImageManager                    string `json:\"image_manager\" yaml:\"image_manager\"`\n\tImageKubexit                    string `json:\"image_kubexit\" yaml:\"image_kubexit\"`\n\tImageProxy                      string `json:\"image_proxy\" yaml:\"image_proxy\"`\n\tImageActivator                  string `json:\"image_activator\" yaml:\"image_activator\"`\n\tImageAutoscaler                 string `json:\"image_autoscaler\" yaml:\"image_autoscaler\"`\n\tImageAsyncGateway               string `json:\"image_async_gateway\" yaml:\"image_async_gateway\"`\n\tImageEnqueuer                   string `json:\"image_enqueuer\" yaml:\"image_enqueuer\"`\n\tImageDequeuer                   string `json:\"image_dequeuer\" yaml:\"image_dequeuer\"`\n\tImageClusterAutoscaler          string `json:\"image_cluster_autoscaler\" yaml:\"image_cluster_autoscaler\"`\n\tImageMetricsServer              string `json:\"image_metrics_server\" yaml:\"image_metrics_server\"`\n\tImageNvidiaDevicePlugin         string `json:\"image_nvidia_device_plugin\" yaml:\"image_nvidia_device_plugin\"`\n\tImageNeuronDevicePlugin         string `json:\"image_neuron_device_plugin\" yaml:\"image_neuron_device_plugin\"`\n\tImageNeuronScheduler            string `json:\"image_neuron_scheduler\" yaml:\"image_neuron_scheduler\"`\n\tImageFluentBit                  string `json:\"image_fluent_bit\" yaml:\"image_fluent_bit\"`\n\tImageIstioProxy                 string `json:\"image_istio_proxy\" yaml:\"image_istio_proxy\"`\n\tImageIstioPilot                 string `json:\"image_istio_pilot\" yaml:\"image_istio_pilot\"`\n\tImagePrometheus                 string `json:\"image_prometheus\" yaml:\"image_prometheus\"`\n\tImagePrometheusConfigReloader   string `json:\"image_prometheus_config_reloader\" yaml:\"image_prometheus_config_reloader\"`\n\tImagePrometheusOperator         string `json:\"image_prometheus_operator\" yaml:\"image_prometheus_operator\"`\n\tImagePrometheusStatsDExporter   string `json:\"image_prometheus_statsd_exporter\" yaml:\"image_prometheus_statsd_exporter\"`\n\tImagePrometheusDCGMExporter     string `json:\"image_prometheus_dcgm_exporter\" yaml:\"image_prometheus_dcgm_exporter\"`\n\tImagePrometheusKubeStateMetrics string `json:\"image_prometheus_kube_state_metrics\" yaml:\"image_prometheus_kube_state_metrics\"`\n\tImagePrometheusNodeExporter     string `json:\"image_prometheus_node_exporter\" yaml:\"image_prometheus_node_exporter\"`\n\tImageKubeRBACProxy              string `json:\"image_kube_rbac_proxy\" yaml:\"image_kube_rbac_proxy\"`\n\tImageGrafana                    string `json:\"image_grafana\" yaml:\"image_grafana\"`\n\tImageEventExporter              string `json:\"image_event_exporter\" yaml:\"image_event_exporter\"`\n\n\tNodeGroups                        []*NodeGroup       `json:\"node_groups\" yaml:\"node_groups\"`\n\tTags                              map[string]string  `json:\"tags\" yaml:\"tags\"`\n\tAvailabilityZones                 []string           `json:\"availability_zones\" yaml:\"availability_zones\"`\n\tSSLCertificateARN                 *string            `json:\"ssl_certificate_arn,omitempty\" yaml:\"ssl_certificate_arn,omitempty\"`\n\tIAMPolicyARNs                     []string           `json:\"iam_policy_arns\" yaml:\"iam_policy_arns\"`\n\tSubnetVisibility                  SubnetVisibility   `json:\"subnet_visibility\" yaml:\"subnet_visibility\"`\n\tSubnets                           []*Subnet          `json:\"subnets,omitempty\" yaml:\"subnets,omitempty\"`\n\tNATGateway                        NATGateway         `json:\"nat_gateway\" yaml:\"nat_gateway\"`\n\tAPILoadBalancerType               LoadBalancerType   `json:\"api_load_balancer_type\" yaml:\"api_load_balancer_type\"`\n\tAPILoadBalancerScheme             LoadBalancerScheme `json:\"api_load_balancer_scheme\" yaml:\"api_load_balancer_scheme\"`\n\tOperatorLoadBalancerScheme        LoadBalancerScheme `json:\"operator_load_balancer_scheme\" yaml:\"operator_load_balancer_scheme\"`\n\tAPILoadBalancerCIDRWhiteList      []string           `json:\"api_load_balancer_cidr_white_list,omitempty\" yaml:\"api_load_balancer_cidr_white_list,omitempty\"`\n\tOperatorLoadBalancerCIDRWhiteList []string           `json:\"operator_load_balancer_cidr_white_list,omitempty\" yaml:\"operator_load_balancer_cidr_white_list,omitempty\"`\n\tVPCCIDR                           *string            `json:\"vpc_cidr,omitempty\" yaml:\"vpc_cidr,omitempty\"`\n\tTelemetry                         bool               `json:\"telemetry\" yaml:\"telemetry\"`\n}\n\ntype ManagedConfig struct {\n\t// fields that must be set by Cortex\n\tCortexPolicyARN string `json:\"cortex_policy_arn\" yaml:\"cortex_policy_arn\"`\n\tAccountID       string `json:\"account_id\" yaml:\"account_id\"`\n\tClusterUID      string `json:\"cluster_uid\" yaml:\"cluster_uid\"`\n\tBucket          string `json:\"bucket\" yaml:\"bucket\"`\n}\n\ntype NodeGroup struct {\n\tName                     string      `json:\"name\" yaml:\"name\"`\n\tInstanceType             string      `json:\"instance_type\" yaml:\"instance_type\"`\n\tMinInstances             int64       `json:\"min_instances\" yaml:\"min_instances\"`\n\tMaxInstances             int64       `json:\"max_instances\" yaml:\"max_instances\"`\n\tPriority                 int64       `json:\"priority\" yaml:\"priority\"`\n\tInstanceVolumeSize       int64       `json:\"instance_volume_size\" yaml:\"instance_volume_size\"`\n\tInstanceVolumeType       VolumeType  `json:\"instance_volume_type\" yaml:\"instance_volume_type\"`\n\tInstanceVolumeIOPS       *int64      `json:\"instance_volume_iops\" yaml:\"instance_volume_iops\"`\n\tInstanceVolumeThroughput *int64      `json:\"instance_volume_throughput\" yaml:\"instance_volume_throughput\"`\n\tSpot                     bool        `json:\"spot\" yaml:\"spot\"`\n\tSpotConfig               *SpotConfig `json:\"spot_config\" yaml:\"spot_config\"`\n}\n\n// compares the supported updatable fields of a nodegroup\nfunc (ng *NodeGroup) HasChanged(old *NodeGroup) bool {\n\treturn ng.MaxInstances != old.MaxInstances || ng.MinInstances != old.MinInstances || ng.Priority != old.Priority\n}\n\nfunc (ng *NodeGroup) UpdatePlan(old *NodeGroup) string {\n\tvar changes []string\n\n\tif old.MinInstances != ng.MinInstances {\n\t\tchanges = append(changes, fmt.Sprintf(\"%s %d->%d\", MinInstancesKey, old.MinInstances, ng.MinInstances))\n\t}\n\tif old.MaxInstances != ng.MaxInstances {\n\t\tchanges = append(changes, fmt.Sprintf(\"%s %d->%d\", MaxInstancesKey, old.MaxInstances, ng.MaxInstances))\n\t}\n\tif old.Priority != ng.Priority {\n\t\tchanges = append(changes, fmt.Sprintf(\"%s %d->%d\", PriorityKey, old.Priority, ng.Priority))\n\t}\n\n\treturn fmt.Sprintf(\"nodegroup %s will be updated with the following changes: %s\", ng.Name, s.StrsAnd(changes))\n}\n\ntype SpotConfig struct {\n\tInstanceDistribution                []string `json:\"instance_distribution\" yaml:\"instance_distribution\"`\n\tOnDemandBaseCapacity                *int64   `json:\"on_demand_base_capacity\" yaml:\"on_demand_base_capacity\"`\n\tOnDemandPercentageAboveBaseCapacity *int64   `json:\"on_demand_percentage_above_base_capacity\" yaml:\"on_demand_percentage_above_base_capacity\"`\n\tMaxPrice                            *float64 `json:\"max_price\" yaml:\"max_price\"`\n\tInstancePools                       *int64   `json:\"instance_pools\" yaml:\"instance_pools\"`\n}\n\ntype Subnet struct {\n\tAvailabilityZone string `json:\"availability_zone\" yaml:\"availability_zone\"`\n\tSubnetID         string `json:\"subnet_id\" yaml:\"subnet_id\"`\n}\n\ntype Config struct {\n\tCoreConfig    `yaml:\",inline\"`\n\tManagedConfig `yaml:\",inline\"`\n}\n\ntype OperatorMetadata struct {\n\tAPIVersion          string `json:\"api_version\" yaml:\"api_version\"`\n\tOperatorID          string `json:\"operator_id\" yaml:\"operator_id\"`\n\tClusterID           string `json:\"cluster_id\" yaml:\"cluster_id\"`\n\tIsOperatorInCluster bool   `json:\"is_operator_in_cluster\" yaml:\"is_operator_in_cluster\"`\n}\n\ntype InternalConfig struct {\n\tConfig\n\n\t// Populated by operator\n\tOperatorMetadata\n}\n\n// The bare minimum to identify a cluster\ntype AccessConfig struct {\n\tClusterName  string `json:\"cluster_name\" yaml:\"cluster_name\"`\n\tRegion       string `json:\"region\" yaml:\"region\"`\n\tImageManager string `json:\"image_manager\" yaml:\"image_manager\"`\n}\n\ntype ConfigureChanges struct {\n\tNodeGroupsToAdd       []string\n\tNodeGroupsToRemove    []string\n\tNodeGroupsToUpdate    []string\n\tEKSNodeGroupsToRemove []string // EKS node group names of (NodeGroupsToRemove ∩ Cortex-converted EKS node groups) ∪ (Cortex-converted EKS node groups - the new cluster config's nodegroups)\n\tFieldsToUpdate        []string\n}\n\nfunc (c *ConfigureChanges) HasChanges() bool {\n\treturn len(c.NodeGroupsToAdd)+len(c.NodeGroupsToRemove)+len(c.NodeGroupsToUpdate)+len(c.EKSNodeGroupsToRemove)+len(c.FieldsToUpdate) != 0\n}\n\n// GetGhostEKSNodeGroups returns the set difference between EKSNodeGroupsToRemove and the EKS-converted NodeGroupsToRemove\nfunc (c *ConfigureChanges) GetGhostEKSNodeGroups() []string {\n\tif len(c.EKSNodeGroupsToRemove) <= len(c.NodeGroupsToRemove) {\n\t\treturn nil\n\t}\n\n\teksNodeGroupPrefix := \"cx-wx-\"\n\tvar ghostEKSNodeGroups []string\n\tfor _, eksNgToRemove := range c.EKSNodeGroupsToRemove {\n\t\tif !slices.HasString(c.NodeGroupsToRemove, eksNgToRemove[len(eksNodeGroupPrefix):]) {\n\t\t\tghostEKSNodeGroups = append(ghostEKSNodeGroups, eksNgToRemove)\n\t\t}\n\t}\n\treturn ghostEKSNodeGroups\n}\n\n// NewForFile initializes and validates the cluster config from the YAML config file\nfunc NewForFile(clusterConfigPath string) (*Config, error) {\n\tconfig := Config{}\n\terrs := cr.ParseYAMLFile(&config, FullConfigValidation, clusterConfigPath)\n\tif errors.HasError(errs) {\n\t\treturn nil, errors.FirstError(errs...)\n\t}\n\n\treturn &config, nil\n}\n\nfunc ValidateRegion(region string) error {\n\tif !aws.EKSSupportedRegions.Has(region) {\n\t\treturn ErrorInvalidRegion(region)\n\t}\n\treturn nil\n}\n\nfunc RegionValidator(region string) (string, error) {\n\tif err := ValidateRegion(region); err != nil {\n\t\treturn \"\", err\n\t}\n\treturn region, nil\n}\n\nfunc (cc *Config) DeepCopy() (Config, error) {\n\tdeepCopied := Config{}\n\terr := structs.DeepCopy(&deepCopied, cc)\n\tif err != nil {\n\t\treturn Config{}, err\n\t}\n\n\treturn deepCopied, nil\n}\n\nfunc (cc *Config) Hash() (string, error) {\n\tbytes, err := yaml.Marshal(cc)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tconfigHash := sha256.New()\n\tconfigHash.Write(bytes)\n\treturn hex.EncodeToString(configHash.Sum(nil)), nil\n}\n\nvar CoreConfigStructFieldValidations = []*cr.StructFieldValidation{\n\t{\n\t\tStructField: \"ClusterName\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   \"cortex\",\n\t\t\tMaxLength: 54, // leaves room for 8 char uniqueness string (and \"-\") for bucket name (63 chars max)\n\t\t\tMinLength: 3,\n\t\t\tValidator: validateClusterName,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"Region\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tRequired:  true,\n\t\t\tMinLength: 1,\n\t\t\tValidator: RegionValidator,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"PrometheusInstanceType\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tMinLength: 1,\n\t\t\tDefault:   \"t3.medium\",\n\t\t\tValidator: validatePrometheusInstanceType,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"Telemetry\",\n\t\tBoolValidation: &cr.BoolValidation{\n\t\t\tDefault: true,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageOperator\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/operator:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageControllerManager\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/controller-manager:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageManager\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/manager:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageKubexit\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/kubexit:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageProxy\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/proxy:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageActivator\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/activator:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageAutoscaler\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/autoscaler:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageAsyncGateway\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/async-gateway:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageEnqueuer\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/enqueuer:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageDequeuer\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/dequeuer:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageClusterAutoscaler\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/cluster-autoscaler:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageMetricsServer\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/metrics-server:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageNvidiaDevicePlugin\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/nvidia-device-plugin:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageNeuronDevicePlugin\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/neuron-device-plugin:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageNeuronScheduler\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/neuron-scheduler:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageFluentBit\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/fluent-bit:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageIstioProxy\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/istio-proxy:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageIstioPilot\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/istio-pilot:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImagePrometheus\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/prometheus:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImagePrometheusConfigReloader\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/prometheus-config-reloader:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImagePrometheusOperator\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/prometheus-operator:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImagePrometheusStatsDExporter\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/prometheus-statsd-exporter:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImagePrometheusDCGMExporter\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/prometheus-dcgm-exporter:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImagePrometheusKubeStateMetrics\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/prometheus-kube-state-metrics:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImagePrometheusNodeExporter\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/prometheus-node-exporter:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageKubeRBACProxy\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/kube-rbac-proxy:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageGrafana\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/grafana:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"ImageEventExporter\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:   consts.DefaultRegistry() + \"/event-exporter:\" + consts.CortexVersion,\n\t\t\tValidator: validateImageVersion,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"NodeGroups\",\n\t\tStructListValidation: &cr.StructListValidation{\n\t\t\tAllowExplicitNull: true,\n\t\t\tTreatNullAsEmpty:  true,\n\t\t\tStructValidation:  nodeGroupsFieldValidation,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"Tags\",\n\t\tStringMapValidation: &cr.StringMapValidation{\n\t\t\tAllowExplicitNull:  true,\n\t\t\tAllowEmpty:         true,\n\t\t\tConvertNullToEmpty: true,\n\t\t\tKeyStringValidator: &cr.StringValidation{\n\t\t\t\tMinLength:                  1,\n\t\t\t\tMaxLength:                  127,\n\t\t\t\tDisallowLeadingWhitespace:  true,\n\t\t\t\tDisallowTrailingWhitespace: true,\n\t\t\t\tInvalidPrefixes:            _invalidTagPrefixes,\n\t\t\t\tAWSTag:                     true,\n\t\t\t},\n\t\t\tValueStringValidator: &cr.StringValidation{\n\t\t\t\tMinLength:                  1,\n\t\t\t\tMaxLength:                  255,\n\t\t\t\tDisallowLeadingWhitespace:  true,\n\t\t\t\tDisallowTrailingWhitespace: true,\n\t\t\t\tInvalidPrefixes:            _invalidTagPrefixes,\n\t\t\t\tAWSTag:                     true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\tStructField: \"SSLCertificateARN\",\n\t\tStringPtrValidation: &cr.StringPtrValidation{\n\t\t\tAllowExplicitNull: true,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"IAMPolicyARNs\",\n\t\tStringListValidation: &cr.StringListValidation{\n\t\t\tDefault:           _defaultIAMPolicies,\n\t\t\tAllowEmpty:        true,\n\t\t\tAllowExplicitNull: true,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"AvailabilityZones\",\n\t\tStringListValidation: &cr.StringListValidation{\n\t\t\tAllowEmpty:        true,\n\t\t\tAllowExplicitNull: true,\n\t\t\tDisallowDups:      true,\n\t\t\tInvalidLengths:    []int{1},\n\t\t},\n\t},\n\t{\n\t\tStructField: \"SubnetVisibility\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tAllowedValues: SubnetVisibilityStrings(),\n\t\t\tDefault:       PublicSubnetVisibility.String(),\n\t\t},\n\t\tParser: func(str string) (interface{}, error) {\n\t\t\treturn SubnetVisibilityFromString(str), nil\n\t\t},\n\t},\n\t{\n\t\tStructField: \"Subnets\",\n\t\tStructListValidation: &cr.StructListValidation{\n\t\t\tAllowExplicitNull: true,\n\t\t\tMinLength:         2,\n\t\t\tStructValidation: &cr.StructValidation{\n\t\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField:      \"AvailabilityZone\",\n\t\t\t\t\t\tStringValidation: &cr.StringValidation{},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField:      \"SubnetID\",\n\t\t\t\t\t\tStringValidation: &cr.StringValidation{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\tStructField: \"NATGateway\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tAllowedValues: NATGatewayStrings(),\n\t\t},\n\t\tParser: func(str string) (interface{}, error) {\n\t\t\treturn NATGatewayFromString(str), nil\n\t\t},\n\t\tDefaultDependentFields: []string{\"SubnetVisibility\", \"Subnets\"},\n\t\tDefaultDependentFieldsFunc: func(vals []interface{}) interface{} {\n\t\t\tsubnetVisibility := vals[0].(SubnetVisibility)\n\t\t\tsubnets := vals[1].([]*Subnet)\n\n\t\t\tif len(subnets) > 0 {\n\t\t\t\treturn NoneNATGateway.String()\n\t\t\t}\n\t\t\tif subnetVisibility == PublicSubnetVisibility {\n\t\t\t\treturn NoneNATGateway.String()\n\t\t\t}\n\t\t\treturn SingleNATGateway.String()\n\t\t},\n\t},\n\t{\n\t\tStructField: \"APILoadBalancerType\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tAllowedValues: LoadBalancerTypeStrings(),\n\t\t\tDefault:       NLBLoadBalancerType.String(),\n\t\t},\n\t\tParser: func(str string) (interface{}, error) {\n\t\t\treturn LoadBalancerTypeFromString(str), nil\n\t\t},\n\t},\n\t{\n\t\tStructField: \"APILoadBalancerScheme\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tAllowedValues: LoadBalancerSchemeStrings(),\n\t\t\tDefault:       InternetFacingLoadBalancerScheme.String(),\n\t\t},\n\t\tParser: func(str string) (interface{}, error) {\n\t\t\treturn LoadBalancerSchemeFromString(str), nil\n\t\t},\n\t},\n\t{\n\t\tStructField: \"APILoadBalancerCIDRWhiteList\",\n\t\tStringListValidation: &cr.StringListValidation{\n\t\t\tValidator: func(addresses []string) ([]string, error) {\n\t\t\t\tfor i, address := range addresses {\n\t\t\t\t\t_, err := validateCIDR(address)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"index %d\", i))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn addresses, nil\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\tStructField: \"OperatorLoadBalancerCIDRWhiteList\",\n\t\tStringListValidation: &cr.StringListValidation{\n\t\t\tValidator: func(addresses []string) ([]string, error) {\n\t\t\t\tfor i, address := range addresses {\n\t\t\t\t\t_, err := validateCIDR(address)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, errors.Wrap(err, fmt.Sprintf(\"index %d\", i))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn addresses, nil\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\tStructField: \"OperatorLoadBalancerScheme\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tAllowedValues: LoadBalancerSchemeStrings(),\n\t\t\tDefault:       InternetFacingLoadBalancerScheme.String(),\n\t\t},\n\t\tParser: func(str string) (interface{}, error) {\n\t\t\treturn LoadBalancerSchemeFromString(str), nil\n\t\t},\n\t},\n\t{\n\t\tStructField: \"VPCCIDR\",\n\t\tStringPtrValidation: &cr.StringPtrValidation{\n\t\t\tValidator: validateVPCCIDR,\n\t\t},\n\t},\n}\n\nvar ManagedConfigStructFieldValidations = []*cr.StructFieldValidation{\n\t{\n\t\tStructField: \"ClusterUID\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:          \"\",\n\t\t\tAllowEmpty:       true,\n\t\t\tTreatNullAsEmpty: true,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"Bucket\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tDefault:          \"\",\n\t\t\tAllowEmpty:       true,\n\t\t\tTreatNullAsEmpty: true,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"CortexPolicyARN\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tRequired:         false,\n\t\t\tAllowEmpty:       true,\n\t\t\tTreatNullAsEmpty: true,\n\t\t},\n\t},\n\t{\n\t\tStructField: \"AccountID\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tRequired:         false,\n\t\t\tAllowEmpty:       true,\n\t\t\tTreatNullAsEmpty: true,\n\t\t},\n\t},\n}\n\nvar nodeGroupsFieldValidation *cr.StructValidation = &cr.StructValidation{\n\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t{\n\t\t\tStructField: \"Name\",\n\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\tRequired:         true,\n\t\t\t\tAlphaNumericDash: true,\n\t\t\t\tMaxLength:        _maxNodeGroupLength,\n\t\t\t\tInvalidSuffixes:  []string{\"-\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"InstanceType\",\n\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\tRequired:  true,\n\t\t\t\tMinLength: 1,\n\t\t\t\tValidator: validateInstanceType,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"MinInstances\",\n\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\tDefault:              int64(1),\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(0),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"MaxInstances\",\n\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\tDefault:              int64(5),\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(0), // this will be validated to be > 0 during cluster up (can be scaled down later)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Priority\",\n\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\tDefault:              int64(1),\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(1),\n\t\t\t\tLessThanOrEqualTo:    pointer.Int64(100),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"InstanceVolumeSize\",\n\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\tDefault:              50,\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(20), // large enough to fit docker images and any other overhead\n\t\t\t\tLessThanOrEqualTo:    pointer.Int64(16384),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"InstanceVolumeType\",\n\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\tAllowedValues: VolumeTypesStrings(),\n\t\t\t\tDefault:       GP3VolumeType.String(),\n\t\t\t},\n\t\t\tParser: func(str string) (interface{}, error) {\n\t\t\t\treturn VolumeTypeFromString(str), nil\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"InstanceVolumeIOPS\",\n\t\t\tInt64PtrValidation: &cr.Int64PtrValidation{\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"InstanceVolumeThroughput\",\n\t\t\tInt64PtrValidation: &cr.Int64PtrValidation{\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(125),\n\t\t\t\tLessThanOrEqualTo:    pointer.Int64(1000),\n\t\t\t\tAllowExplicitNull:    true,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Spot\",\n\t\t\tBoolValidation: &cr.BoolValidation{\n\t\t\t\tDefault: false,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"SpotConfig\",\n\t\t\tStructValidation: &cr.StructValidation{\n\t\t\t\tDefaultNil:        true,\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField: \"InstanceDistribution\",\n\t\t\t\t\t\tStringListValidation: &cr.StringListValidation{\n\t\t\t\t\t\t\tDisallowDups:      true,\n\t\t\t\t\t\t\tValidator:         validateInstanceDistribution,\n\t\t\t\t\t\t\tAllowExplicitNull: true,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField: \"OnDemandBaseCapacity\",\n\t\t\t\t\t\tInt64PtrValidation: &cr.Int64PtrValidation{\n\t\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(0),\n\t\t\t\t\t\t\tAllowExplicitNull:    true,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField: \"OnDemandPercentageAboveBaseCapacity\",\n\t\t\t\t\t\tInt64PtrValidation: &cr.Int64PtrValidation{\n\t\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(0),\n\t\t\t\t\t\t\tLessThanOrEqualTo:    pointer.Int64(100),\n\t\t\t\t\t\t\tAllowExplicitNull:    true,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField: \"MaxPrice\",\n\t\t\t\t\t\tFloat64PtrValidation: &cr.Float64PtrValidation{\n\t\t\t\t\t\t\tGreaterThan:       pointer.Float64(0),\n\t\t\t\t\t\t\tAllowExplicitNull: true,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField: \"InstancePools\",\n\t\t\t\t\t\tInt64PtrValidation: &cr.Int64PtrValidation{\n\t\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(1),\n\t\t\t\t\t\t\tLessThanOrEqualTo:    pointer.Int64(int64(_maxInstancePools)),\n\t\t\t\t\t\t\tAllowExplicitNull:    true,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n}\n\nvar FullConfigValidation = &cr.StructValidation{\n\tRequired:               true,\n\tStructFieldValidations: append(CoreConfigStructFieldValidations, ManagedConfigStructFieldValidations...),\n\tAllowExtraFields:       false,\n}\n\nvar AccessValidation = &cr.StructValidation{\n\tAllowExtraFields: true,\n\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t{\n\t\t\tStructField: \"ClusterName\",\n\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\tDefault:   \"cortex\",\n\t\t\t\tMaxLength: 54, // leaves room for 8 char uniqueness string (and \"-\") for bucket name (63 chars max)\n\t\t\t\tMinLength: 3,\n\t\t\t\tValidator: validateClusterName,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Region\",\n\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\tRequired:  true,\n\t\t\t\tMinLength: 1,\n\t\t\t\tValidator: RegionValidator,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"ImageManager\",\n\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\tDefault:   consts.DefaultRegistry() + \"/manager:\" + consts.CortexVersion,\n\t\t\t\tValidator: validateImageVersion,\n\t\t\t},\n\t\t},\n\t},\n}\n\nfunc (cc *Config) ToAccessConfig() AccessConfig {\n\treturn AccessConfig{\n\t\tClusterName:  cc.ClusterName,\n\t\tRegion:       cc.Region,\n\t\tImageManager: cc.ImageManager,\n\t}\n}\n\nfunc SQSNamePrefix(clusterName string) string {\n\t// 8 was chosen to make sure that other identifiers can be added to the full queue name before reaching the 80 char SQS name limit\n\treturn \"cx\" + SQSQueueDelimiter + libhash.String(clusterName)[:8] + SQSQueueDelimiter\n}\n\n// SQSNamePrefix returns a string with the hash of cluster name and adds trailing \"_\" e.g. cx_abcd1234_\nfunc (cc *CoreConfig) SQSNamePrefix() string {\n\treturn SQSNamePrefix(cc.ClusterName)\n}\n\nfunc (cc *Config) validate(awsClient *aws.Client) error {\n\tif cc.APILoadBalancerType == NLBLoadBalancerType {\n\t\tisSupportedByNLB, err := aws.IsInstanceSupportedByNLB(cc.PrometheusInstanceType)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !isSupportedByNLB {\n\t\t\treturn errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(cc.PrometheusInstanceType), PrometheusInstanceTypeKey)\n\t\t}\n\t}\n\n\tnumNodeGroups := len(cc.NodeGroups)\n\tif numNodeGroups > MaxNodeGroups {\n\t\treturn ErrorMaxNumOfNodeGroupsReached(MaxNodeGroups)\n\t}\n\n\tngNames := []string{}\n\tinstances := []aws.InstanceTypeRequests{\n\t\t{\n\t\t\tInstanceType:              _operatorNodeGroupInstanceType,\n\t\t\tRequiredOnDemandInstances: 1,\n\t\t},\n\t\t{\n\t\t\tInstanceType:              cc.PrometheusInstanceType,\n\t\t\tRequiredOnDemandInstances: 1,\n\t\t},\n\t}\n\tfor _, nodeGroup := range cc.NodeGroups {\n\t\tif !slices.HasString(ngNames, nodeGroup.Name) {\n\t\t\tngNames = append(ngNames, nodeGroup.Name)\n\t\t} else {\n\t\t\treturn errors.Wrap(ErrorDuplicateNodeGroupName(nodeGroup.Name), NodeGroupsKey)\n\t\t}\n\n\t\terr := nodeGroup.validateNodeGroup(awsClient, cc.Region, cc.APILoadBalancerType)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, NodeGroupsKey, nodeGroup.Name)\n\t\t}\n\n\t\tinstances = append(instances, aws.InstanceTypeRequests{\n\t\t\tInstanceType:              nodeGroup.InstanceType,\n\t\t\tRequiredOnDemandInstances: nodeGroup.MaxPossibleOnDemandInstances(),\n\t\t\tRequiredSpotInstances:     nodeGroup.MaxPossibleSpotInstances(),\n\t\t})\n\t}\n\n\tif err := awsClient.VerifyInstanceQuota(instances); err != nil {\n\t\t// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API\n\t\tif !aws.IsAWSError(err) {\n\t\t\treturn errors.Wrap(err, NodeGroupsKey)\n\t\t}\n\t}\n\n\tif len(cc.AvailabilityZones) > 0 && len(cc.Subnets) > 0 {\n\t\treturn ErrorSpecifyOneOrNone(AvailabilityZonesKey, SubnetsKey)\n\t}\n\n\tif len(cc.Subnets) > 0 && cc.NATGateway != NoneNATGateway {\n\t\treturn ErrorNoNATGatewayWithSubnets()\n\t}\n\n\tif cc.SubnetVisibility == PrivateSubnetVisibility && cc.NATGateway == NoneNATGateway && len(cc.Subnets) == 0 {\n\t\treturn ErrorNATRequiredWithPrivateSubnetVisibility()\n\t}\n\n\taccountID, _, err := awsClient.GetCachedAccountID()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif cc.AccountID != \"\" {\n\t\treturn ErrorDisallowedField(AccountIDKey)\n\t}\n\tcc.AccountID = accountID\n\n\tif cc.Bucket != \"\" {\n\t\treturn ErrorDisallowedField(BucketKey)\n\t}\n\tcc.Bucket = BucketName(accountID, cc.ClusterName, cc.Region)\n\t// check if the bucket already exists in a different region for some reason\n\tbucketRegion, _ := aws.GetBucketRegion(cc.Bucket)\n\tif bucketRegion != \"\" && bucketRegion != cc.Region { // if the bucket didn't exist, we will create it in the correct region, so there is no error\n\t\treturn ErrorS3RegionDiffersFromCluster(cc.Bucket, bucketRegion, cc.Region)\n\t}\n\n\tif cc.CortexPolicyARN != \"\" {\n\t\treturn ErrorDisallowedField(CortexPolicyARNKey)\n\t}\n\tcc.CortexPolicyARN = DefaultPolicyARN(accountID, cc.ClusterName, cc.Region)\n\n\tdefaultPoliciesSet := strset.New(_defaultIAMPolicies...)\n\tfor i := range cc.IAMPolicyARNs {\n\t\tpolicyARN := cc.IAMPolicyARNs[i]\n\n\t\tif defaultPoliciesSet.Has(policyARN) {\n\t\t\tpartition := aws.PartitionFromRegion(cc.Region)\n\t\t\tadjustedPolicyARN := strings.Replace(policyARN, \"arn:aws:\", fmt.Sprintf(\"arn:%s:\", partition), 1)\n\t\t\tcc.IAMPolicyARNs[i] = adjustedPolicyARN\n\t\t\tpolicyARN = adjustedPolicyARN\n\t\t}\n\t\t_, err := awsClient.IAM().GetPolicy(&iam.GetPolicyInput{\n\t\t\tPolicyArn: pointer.String(policyARN),\n\t\t})\n\t\tif err != nil {\n\t\t\tif aws.IsErrCode(err, iam.ErrCodeNoSuchEntityException) {\n\t\t\t\treturn errors.Wrap(ErrorIAMPolicyARNNotFound(policyARN), IAMPolicyARNsKey)\n\t\t\t}\n\t\t\treturn errors.Wrap(err, IAMPolicyARNsKey)\n\t\t}\n\t}\n\n\tif cc.SSLCertificateARN != nil {\n\t\texists, err := awsClient.DoesCertificateExist(*cc.SSLCertificateARN)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, SSLCertificateARNKey)\n\t\t}\n\n\t\tif !exists {\n\t\t\treturn errors.Wrap(ErrorSSLCertificateARNNotFound(*cc.SSLCertificateARN, cc.Region), SSLCertificateARNKey)\n\t\t}\n\t}\n\n\tfor tagName, tagValue := range cc.Tags {\n\t\tif strings.HasPrefix(tagName, \"cortex.dev/\") {\n\t\t\tif tagName != ClusterNameTag {\n\t\t\t\treturn errors.Wrap(cr.ErrorCantHavePrefix(tagName, \"cortex.dev/\"), TagsKey)\n\t\t\t}\n\t\t\tif tagValue != cc.ClusterName {\n\t\t\t\treturn errors.Wrap(ErrorCantOverrideDefaultTag(), TagsKey)\n\t\t\t}\n\t\t}\n\t}\n\tcc.Tags[ClusterNameTag] = cc.ClusterName\n\n\tif len(cc.Subnets) > 0 {\n\t\tif err := cc.validateSubnets(awsClient); err != nil {\n\t\t\treturn errors.Wrap(err, SubnetsKey)\n\t\t}\n\t} else {\n\t\tif err := cc.setAvailabilityZones(awsClient); err != nil {\n\t\t\treturn errors.Wrap(err, AvailabilityZonesKey)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (cc *Config) validateTopLevelSectionDiff(oldConfig Config) ([]string, error) {\n\tvar fieldsToUpdate []string\n\t// validate actionable changes\n\tnewClusterConfigCopy, err := cc.DeepCopy()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\toldClusterConfigCopy, err := oldConfig.DeepCopy()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif libstr.Obj(newClusterConfigCopy.SSLCertificateARN) != libstr.Obj(oldClusterConfigCopy.SSLCertificateARN) {\n\t\tfieldsToUpdate = append(fieldsToUpdate, SSLCertificateARNKey)\n\t}\n\n\tif libstr.Obj(newClusterConfigCopy.APILoadBalancerCIDRWhiteList) != libstr.Obj(oldClusterConfigCopy.APILoadBalancerCIDRWhiteList) {\n\t\tfieldsToUpdate = append(fieldsToUpdate, APILoadBalancerCIDRWhiteListKey)\n\t}\n\n\tif libstr.Obj(newClusterConfigCopy.OperatorLoadBalancerCIDRWhiteList) != libstr.Obj(oldClusterConfigCopy.OperatorLoadBalancerCIDRWhiteList) {\n\t\tfieldsToUpdate = append(fieldsToUpdate, OperatorLoadBalancerCIDRWhiteListKey)\n\t}\n\n\tclearUpdatableFields(&newClusterConfigCopy)\n\tclearUpdatableFields(&oldClusterConfigCopy)\n\n\th1, err := newClusterConfigCopy.Hash()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\th2, err := oldClusterConfigCopy.Hash()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif h1 != h2 {\n\t\treturn nil, ErrorConfigCannotBeChangedOnConfigure()\n\t}\n\n\treturn fieldsToUpdate, nil\n}\n\nfunc clearUpdatableFields(clusterConfig *Config) {\n\tclusterConfig.SSLCertificateARN = nil\n\tclusterConfig.APILoadBalancerCIDRWhiteList = nil\n\tclusterConfig.OperatorLoadBalancerCIDRWhiteList = nil\n\tclusterConfig.NodeGroups = []*NodeGroup{}\n}\n\nfunc (cc *Config) validateSharedNodeGroupsDiff(oldConfig Config) error {\n\tsharedNgsFromNewConfig, sharedNgsFromOldConfig := cc.getCommonNodeGroups(oldConfig)\n\tfor i := range sharedNgsFromNewConfig {\n\t\tnewNgCopy, err := sharedNgsFromNewConfig[i].DeepCopy()\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, NodeGroupsKey)\n\t\t}\n\t\toldNgCopy, err := sharedNgsFromOldConfig[i].DeepCopy()\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, NodeGroupsKey)\n\t\t}\n\n\t\tnewNgCopy.MinInstances = 0\n\t\tnewNgCopy.MaxInstances = 0\n\t\tnewNgCopy.Priority = 0\n\t\toldNgCopy.MinInstances = 0\n\t\toldNgCopy.MaxInstances = 0\n\t\toldNgCopy.Priority = 0\n\n\t\tnewHash, err := newNgCopy.Hash()\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, NodeGroupsKey)\n\t\t}\n\t\toldHash, err := oldNgCopy.Hash()\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, NodeGroupsKey)\n\t\t}\n\n\t\tif newHash != oldHash {\n\t\t\treturn errors.Wrap(ErrorNodeGroupCanOnlyBeScaled(), NodeGroupsKey, newNgCopy.Name)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (cc *Config) validateNodeAdditionRate(k8sClient *k8s.Client) error {\n\tworkloadNodes, err := k8sClient.ListNodesByLabel(\"workload\", \"true\")\n\tif err != nil {\n\t\treturn err\n\t}\n\ttotalCurrentNodes := int64(len(workloadNodes))\n\ttotalRequestedNodes := getTotalMinInstances(cc.NodeGroups)\n\n\tif totalRequestedNodes-totalCurrentNodes > MaxNodesToAddOnClusterConfigure {\n\t\treturn ErrorMaxNodesToAddOnClusterConfigure(totalRequestedNodes, totalCurrentNodes, MaxNodesToAddOnClusterConfigure)\n\t}\n\n\treturn nil\n}\n\n// this validates the user-provided cluster config\nfunc (cc *Config) ValidateOnInstall(awsClient *aws.Client) error {\n\tfmt.Print(\"verifying your configuration ...\\n\\n\")\n\n\terr := cc.validate(awsClient)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\trequestedTotalMinInstances := getTotalMinInstances(cc.NodeGroups)\n\tif requestedTotalMinInstances > MaxNodesToAddOnClusterUp {\n\t\treturn errors.Wrap(ErrorMaxNodesToAddOnClusterUp(requestedTotalMinInstances, MaxNodesToAddOnClusterUp), NodeGroupsKey)\n\t}\n\n\t// setting max_instances to 0 during cluster creation is not permitted (but scaling max_instances to 0 afterwards is allowed)\n\tfor _, nodeGroup := range cc.NodeGroups {\n\t\tif nodeGroup != nil && nodeGroup.MaxInstances == 0 {\n\t\t\treturn errors.Wrap(ErrorNodeGroupMaxInstancesIsZero(), NodeGroupsKey, nodeGroup.Name)\n\t\t}\n\t}\n\n\tif cc.ClusterUID != \"\" {\n\t\treturn ErrorDisallowedField(ClusterUIDKey)\n\t}\n\tcc.ClusterUID = strconv.FormatInt(time.Now().Unix(), 10)\n\n\tvar requiredVPCs int\n\tif len(cc.Subnets) == 0 {\n\t\trequiredVPCs = 1\n\t}\n\tlongestCIDRWhiteList := libmath.MaxInt(len(cc.APILoadBalancerCIDRWhiteList), len(cc.OperatorLoadBalancerCIDRWhiteList))\n\tif err := VerifyNetworkQuotas(awsClient, 1, cc.NATGateway != NoneNATGateway, cc.NATGateway == HighlyAvailableNATGateway, requiredVPCs, strset.FromSlice(cc.AvailabilityZones), len(cc.NodeGroups), len(cc.NodeGroups), longestCIDRWhiteList, false); err != nil {\n\t\t// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API\n\t\tif !aws.IsAWSError(err) {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (cc *Config) ValidateOnConfigure(awsClient *aws.Client, k8sClient *k8s.Client, oldConfig Config, eksNodeGroupStacks []*cloudformation.StackSummary) (ConfigureChanges, error) {\n\tfmt.Print(\"verifying your configuration ...\\n\\n\")\n\n\tcc.ClusterUID = oldConfig.ClusterUID\n\terr := cc.validate(awsClient)\n\tif err != nil {\n\t\treturn ConfigureChanges{}, err\n\t}\n\n\tfieldsToUpdate, err := cc.validateTopLevelSectionDiff(oldConfig)\n\tif err != nil {\n\t\treturn ConfigureChanges{}, err\n\t}\n\n\terr = cc.validateSharedNodeGroupsDiff(oldConfig)\n\tif err != nil {\n\t\treturn ConfigureChanges{}, err\n\t}\n\n\terr = cc.validateNodeAdditionRate(k8sClient)\n\tif err != nil {\n\t\treturn ConfigureChanges{}, errors.Wrap(err, NodeGroupsKey)\n\t}\n\n\tngsToBeAdded := cc.getNewNodeGroups(oldConfig)\n\tngsToBeRemoved := cc.getRemovedNodeGroups(oldConfig)\n\n\ttempMaxNodeGroupCount := len(cc.NodeGroups) + len(ngsToBeRemoved)\n\ttempNetAdditionOfNodeGroupCount := tempMaxNodeGroupCount - len(oldConfig.NodeGroups)\n\tlongestCIDRWhiteList := libmath.MaxInt(len(cc.APILoadBalancerCIDRWhiteList), len(cc.OperatorLoadBalancerCIDRWhiteList))\n\tif err := VerifyNetworkQuotasOnConfigure(awsClient, strset.FromSlice(cc.AvailabilityZones), tempMaxNodeGroupCount, tempNetAdditionOfNodeGroupCount, longestCIDRWhiteList); err != nil {\n\t\t// Skip AWS errors, since some regions (e.g. eu-north-1) do not support this API\n\t\tif !aws.IsAWSError(err) {\n\t\t\treturn ConfigureChanges{}, errors.Wrap(err, NodeGroupsKey)\n\t\t}\n\t}\n\n\tsharedNgsFromNewConfig, sharedNgsFromOldConfig := cc.getCommonNodeGroups(oldConfig)\n\tngsToBeUpdated := []*NodeGroup{}\n\tfor i := range sharedNgsFromNewConfig {\n\t\tif sharedNgsFromNewConfig[i].HasChanged(sharedNgsFromOldConfig[i]) {\n\t\t\tngsToBeUpdated = append(ngsToBeUpdated, sharedNgsFromNewConfig[i])\n\t\t}\n\t}\n\n\treturn ConfigureChanges{\n\t\tNodeGroupsToAdd:       GetNodeGroupNames(ngsToBeAdded),\n\t\tNodeGroupsToRemove:    GetNodeGroupNames(ngsToBeRemoved),\n\t\tNodeGroupsToUpdate:    GetNodeGroupNames(ngsToBeUpdated),\n\t\tEKSNodeGroupsToRemove: getStaleEksNodeGroups(cc.ClusterName, eksNodeGroupStacks, cc.NodeGroups, ngsToBeRemoved),\n\t\tFieldsToUpdate:        fieldsToUpdate,\n\t}, nil\n}\n\nfunc (ng *NodeGroup) validateNodeGroup(awsClient *aws.Client, region string, loadBalancerType LoadBalancerType) error {\n\tif ng.MinInstances > ng.MaxInstances {\n\t\treturn ErrorMinInstancesGreaterThanMax(ng.MinInstances, ng.MaxInstances)\n\t}\n\n\tprimaryInstanceType := ng.InstanceType\n\n\tif loadBalancerType == NLBLoadBalancerType {\n\t\tisPrimaryInstanceSupportedByNLB, err := aws.IsInstanceSupportedByNLB(primaryInstanceType)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !isPrimaryInstanceSupportedByNLB {\n\t\t\treturn errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(primaryInstanceType), InstanceTypeKey)\n\t\t}\n\t}\n\n\tif !aws.InstanceTypes[region].Has(primaryInstanceType) {\n\t\treturn errors.Wrap(ErrorInstanceTypeNotSupportedInRegion(primaryInstanceType, region), InstanceTypeKey)\n\t}\n\n\tif _, ok := aws.InstanceMetadatas[region][primaryInstanceType]; !ok {\n\t\treturn errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(primaryInstanceType), InstanceTypeKey)\n\t}\n\n\t// throw error if IOPS defined for other storage than io1/gp3\n\tif ng.InstanceVolumeType != IO1VolumeType && ng.InstanceVolumeType != GP3VolumeType && ng.InstanceVolumeIOPS != nil {\n\t\treturn ErrorIOPSNotSupported(ng.InstanceVolumeType)\n\t}\n\n\t// throw error if throughput defined for other storage than gp3\n\tif ng.InstanceVolumeType != GP3VolumeType && ng.InstanceVolumeThroughput != nil {\n\t\treturn ErrorThroughputNotSupported(ng.InstanceVolumeType)\n\t}\n\n\tif ng.InstanceVolumeType == GP3VolumeType && ((ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput == nil) || (ng.InstanceVolumeIOPS == nil && ng.InstanceVolumeThroughput != nil)) {\n\t\treturn ErrorSpecifyTwoOrNone(InstanceVolumeIOPSKey, InstanceVolumeThroughputKey)\n\t}\n\n\tif ng.InstanceVolumeIOPS != nil {\n\t\tif ng.InstanceVolumeType == IO1VolumeType {\n\t\t\tif *ng.InstanceVolumeIOPS < _smallestIOPSForIO1VolumeType {\n\t\t\t\treturn ErrorIOPSTooSmall(ng.InstanceVolumeType, *ng.InstanceVolumeIOPS, _smallestIOPSForIO1VolumeType)\n\t\t\t}\n\t\t\tif *ng.InstanceVolumeIOPS > _highestIOPSForIO1VolumeType {\n\t\t\t\treturn ErrorIOPSTooLarge(ng.InstanceVolumeType, *ng.InstanceVolumeIOPS, _highestIOPSForIO1VolumeType)\n\t\t\t}\n\t\t\tif *ng.InstanceVolumeIOPS > ng.InstanceVolumeSize*_maxIOPSToVolumeSizeRatioForIO1 {\n\t\t\t\treturn ErrorIOPSToVolumeSizeRatio(ng.InstanceVolumeType, _maxIOPSToVolumeSizeRatioForIO1, *ng.InstanceVolumeIOPS, ng.InstanceVolumeSize)\n\t\t\t}\n\t\t} else {\n\t\t\tif *ng.InstanceVolumeIOPS < _smallestIOPSForGP3VolumeType {\n\t\t\t\treturn ErrorIOPSTooSmall(ng.InstanceVolumeType, *ng.InstanceVolumeIOPS, _smallestIOPSForGP3VolumeType)\n\t\t\t}\n\t\t\tif *ng.InstanceVolumeIOPS > _highestIOPSForGP3VolumeType {\n\t\t\t\treturn ErrorIOPSTooLarge(ng.InstanceVolumeType, *ng.InstanceVolumeIOPS, _highestIOPSForGP3VolumeType)\n\t\t\t}\n\t\t\tif *ng.InstanceVolumeIOPS > ng.InstanceVolumeSize*_maxIOPSToVolumeSizeRatioForGP3 {\n\t\t\t\treturn ErrorIOPSToVolumeSizeRatio(ng.InstanceVolumeType, _maxIOPSToVolumeSizeRatioForGP3, *ng.InstanceVolumeIOPS, ng.InstanceVolumeSize)\n\t\t\t}\n\t\t\tiopsToThroughputRatio := float64(*ng.InstanceVolumeIOPS) / float64(*ng.InstanceVolumeThroughput)\n\t\t\tif iopsToThroughputRatio < float64(_minIOPSToThroughputRatioForGP3) {\n\t\t\t\treturn ErrorIOPSToThroughputRatio(ng.InstanceVolumeType, _minIOPSToThroughputRatioForGP3, *ng.InstanceVolumeIOPS, *ng.InstanceVolumeThroughput)\n\t\t\t}\n\t\t}\n\t} else if ng.InstanceVolumeType == GP3VolumeType {\n\t\tng.InstanceVolumeIOPS = pointer.Int64(3000)\n\t\tng.InstanceVolumeThroughput = pointer.Int64(125)\n\t} else if ng.InstanceVolumeType == IO1VolumeType {\n\t\tng.InstanceVolumeIOPS = pointer.Int64(libmath.MinInt64(ng.InstanceVolumeSize*_maxIOPSToVolumeSizeRatioForIO1, 3000))\n\t}\n\n\tif ng.Spot {\n\t\tng.FillEmptySpotFields(region)\n\n\t\tprimaryInstance := aws.InstanceMetadatas[region][primaryInstanceType]\n\n\t\tfor _, instanceType := range ng.SpotConfig.InstanceDistribution {\n\t\t\tif instanceType == primaryInstanceType {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif !aws.InstanceTypes[region].Has(instanceType) {\n\t\t\t\treturn errors.Wrap(ErrorInstanceTypeNotSupportedInRegion(instanceType, region), SpotConfigKey, InstanceDistributionKey)\n\t\t\t}\n\n\t\t\tif loadBalancerType == NLBLoadBalancerType {\n\t\t\t\tisSecondaryInstanceSupportedByNLB, err := aws.IsInstanceSupportedByNLB(primaryInstanceType)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif !isSecondaryInstanceSupportedByNLB {\n\t\t\t\t\treturn errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(primaryInstanceType), SpotConfigKey, InstanceDistributionKey)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif _, ok := aws.InstanceMetadatas[region][instanceType]; !ok {\n\t\t\t\treturn errors.Wrap(ErrorInstanceTypeNotSupportedByCortex(instanceType), SpotConfigKey, InstanceDistributionKey)\n\t\t\t}\n\n\t\t\tinstanceMetadata := aws.InstanceMetadatas[region][instanceType]\n\t\t\terr := CheckSpotInstanceCompatibility(primaryInstance, instanceMetadata)\n\t\t\tif err != nil {\n\t\t\t\treturn errors.Wrap(err, SpotConfigKey, InstanceDistributionKey)\n\t\t\t}\n\n\t\t\tspotInstancePrice, awsErr := awsClient.SpotInstancePrice(instanceMetadata.Type)\n\t\t\tif awsErr == nil {\n\t\t\t\tif err := CheckSpotInstancePriceCompatibility(primaryInstance, instanceMetadata, ng.SpotConfig.MaxPrice, spotInstancePrice); err != nil {\n\t\t\t\t\treturn errors.Wrap(err, SpotConfigKey, InstanceDistributionKey)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif ng.SpotConfig.OnDemandBaseCapacity != nil && *ng.SpotConfig.OnDemandBaseCapacity > ng.MaxInstances {\n\t\t\treturn ErrorOnDemandBaseCapacityGreaterThanMax(*ng.SpotConfig.OnDemandBaseCapacity, ng.MaxInstances)\n\t\t}\n\t} else {\n\t\tif ng.SpotConfig != nil {\n\t\t\treturn ErrorConfiguredWhenSpotIsNotEnabled(SpotConfigKey)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (cc *Config) GetNodeGroupByName(name string) *NodeGroup {\n\tfor _, ng := range cc.NodeGroups {\n\t\tif ng.Name == name {\n\t\t\treturn ng\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (cc *Config) getNewNodeGroups(oldConfig Config) []*NodeGroup {\n\tvar newNodeGroups []*NodeGroup\n\tfor _, updatingNg := range cc.NodeGroups {\n\t\tisNewNg := true\n\t\tfor _, previousNg := range oldConfig.NodeGroups {\n\t\t\tif previousNg.Name == updatingNg.Name {\n\t\t\t\tisNewNg = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif isNewNg {\n\t\t\tngCopy := *updatingNg\n\t\t\tnewNodeGroups = append(newNodeGroups, &ngCopy)\n\t\t}\n\t}\n\treturn newNodeGroups\n}\n\nfunc (cc *Config) getRemovedNodeGroups(oldConfig Config) []*NodeGroup {\n\tvar removedNodeGroups []*NodeGroup\n\tfor _, previousNg := range oldConfig.NodeGroups {\n\t\tisRemovedNg := true\n\t\tfor _, updatingNg := range cc.NodeGroups {\n\t\t\tif previousNg.Name == updatingNg.Name {\n\t\t\t\tisRemovedNg = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif isRemovedNg {\n\t\t\tngCopy := *previousNg\n\t\t\tremovedNodeGroups = append(removedNodeGroups, &ngCopy)\n\t\t}\n\t}\n\treturn removedNodeGroups\n}\n\nfunc (cc *Config) getCommonNodeGroups(oldConfig Config) ([]*NodeGroup, []*NodeGroup) {\n\tvar commonNewNodeGroups []*NodeGroup\n\tvar commonOldNodeGroups []*NodeGroup\n\tfor _, previousNg := range oldConfig.NodeGroups {\n\t\tfor _, updatingNg := range cc.NodeGroups {\n\t\t\tif previousNg.Name == updatingNg.Name {\n\t\t\t\tngNewCopy := *updatingNg\n\t\t\t\tngOldCopy := *previousNg\n\t\t\t\tcommonNewNodeGroups = append(commonNewNodeGroups, &ngNewCopy)\n\t\t\t\tcommonOldNodeGroups = append(commonOldNodeGroups, &ngOldCopy)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn commonNewNodeGroups, commonOldNodeGroups\n}\n\nfunc getTotalMinInstances(nodeGroups []*NodeGroup) int64 {\n\ttotalMinInstances := int64(0)\n\tfor _, ng := range nodeGroups {\n\t\ttotalMinInstances += ng.MinInstances\n\t}\n\treturn totalMinInstances\n}\n\nfunc GetNodeGroupNames(nodeGroups []*NodeGroup) []string {\n\tngNames := make([]string, len(nodeGroups))\n\tfor i := range nodeGroups {\n\t\tngNames[i] = nodeGroups[i].Name\n\t}\n\treturn ngNames\n}\n\nfunc CheckSpotInstanceCompatibility(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error {\n\tif target.Inf > 0 && suggested.Inf == 0 {\n\t\treturn ErrorIncompatibleSpotInstanceTypeInf(suggested)\n\t}\n\n\tif target.GPU > suggested.GPU {\n\t\treturn ErrorIncompatibleSpotInstanceTypeGPU(target, suggested)\n\t}\n\n\tif target.Memory.Cmp(suggested.Memory) > 0 {\n\t\treturn ErrorIncompatibleSpotInstanceTypeMemory(target, suggested)\n\t}\n\n\tif target.CPU.Cmp(suggested.CPU) > 0 {\n\t\treturn ErrorIncompatibleSpotInstanceTypeCPU(target, suggested)\n\t}\n\n\treturn nil\n}\n\nfunc CheckSpotInstancePriceCompatibility(target aws.InstanceMetadata, suggested aws.InstanceMetadata, maxPrice *float64, spotInstancePrice float64) error {\n\tif (maxPrice == nil || *maxPrice == target.Price) && target.Price < spotInstancePrice {\n\t\treturn ErrorSpotPriceGreaterThanTargetOnDemand(spotInstancePrice, target, suggested)\n\t}\n\n\tif maxPrice != nil && *maxPrice < spotInstancePrice {\n\t\treturn ErrorSpotPriceGreaterThanMaxPrice(spotInstancePrice, *maxPrice, suggested)\n\t}\n\treturn nil\n}\n\nfunc AutoGenerateSpotConfig(spotConfig *SpotConfig, region string, instanceType string) {\n\tprimaryInstance := aws.InstanceMetadatas[region][instanceType]\n\tcleanedDistribution := []string{instanceType}\n\tfor _, spotInstance := range spotConfig.InstanceDistribution {\n\t\tif spotInstance != instanceType {\n\t\t\tcleanedDistribution = append(cleanedDistribution, spotInstance)\n\t\t}\n\t}\n\tspotConfig.InstanceDistribution = cleanedDistribution\n\n\tif spotConfig.MaxPrice == nil {\n\t\tspotConfig.MaxPrice = &primaryInstance.Price\n\t}\n\n\tif spotConfig.OnDemandBaseCapacity == nil {\n\t\tspotConfig.OnDemandBaseCapacity = pointer.Int64(0)\n\t}\n\n\tif spotConfig.OnDemandPercentageAboveBaseCapacity == nil {\n\t\tspotConfig.OnDemandPercentageAboveBaseCapacity = pointer.Int64(0)\n\t}\n\n\tif spotConfig.InstancePools == nil {\n\t\tif len(spotConfig.InstanceDistribution) < _maxInstancePools {\n\t\t\tspotConfig.InstancePools = pointer.Int64(int64(len(spotConfig.InstanceDistribution)))\n\t\t} else {\n\t\t\tspotConfig.InstancePools = pointer.Int64(int64(_maxInstancePools))\n\t\t}\n\t}\n}\n\nfunc (ng *NodeGroup) FillEmptySpotFields(region string) {\n\tif ng.SpotConfig == nil {\n\t\tng.SpotConfig = &SpotConfig{}\n\t}\n\tAutoGenerateSpotConfig(ng.SpotConfig, region, ng.InstanceType)\n}\n\nfunc validateCIDR(cidr string) (string, error) {\n\t_, _, err := net.ParseCIDR(cidr)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(err)\n\t}\n\n\treturn cidr, nil\n}\n\nfunc validateVPCCIDR(cidr string) (string, error) {\n\t_, network, err := net.ParseCIDR(cidr)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(err)\n\t}\n\n\tif network != nil {\n\t\tmaskSize, _ := network.Mask.Size()\n\t\tif maskSize < _minSubnetMask || maskSize > _maxSubnetMask {\n\t\t\treturn \"\", ErrorSubnetMaskOutOfRange(maskSize, _minSubnetMask, _maxSubnetMask)\n\t\t}\n\t}\n\n\treturn cidr, nil\n}\n\nfunc validateInstanceType(instanceType string) (string, error) {\n\tif err := aws.CheckValidInstanceType(instanceType); err != nil {\n\t\treturn \"\", err\n\t}\n\n\tparsedType, err := aws.ParseInstanceType(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif parsedType.Size == \"nano\" || parsedType.Size == \"micro\" {\n\t\treturn \"\", ErrorInstanceTypeTooSmall(instanceType)\n\t}\n\n\tisAMDGPU, err := aws.IsAMDGPUInstance(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif isAMDGPU {\n\t\treturn \"\", ErrorAMDGPUInstancesNotSupported(instanceType)\n\t}\n\n\tisMac, err := aws.IsMacInstance(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif isMac {\n\t\treturn \"\", ErrorMacInstancesNotSupported(instanceType)\n\t}\n\n\tisFPGA, err := aws.IsFPGAInstance(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif isFPGA {\n\t\treturn \"\", ErrorFPGAInstancesNotSupported(instanceType)\n\t}\n\n\tisAlevo, err := aws.IsAlevoInstance(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif isAlevo {\n\t\treturn \"\", ErrorAlevoInstancesNotSupported(instanceType)\n\t}\n\n\tisGaudi, err := aws.IsGaudiInstance(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif isGaudi {\n\t\treturn \"\", ErrorGaudiInstancesNotSupported(instanceType)\n\t}\n\n\tisTrainium, err := aws.IsTrainiumInstance(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif isTrainium {\n\t\treturn \"\", ErrorTrainiumInstancesNotSupported(instanceType)\n\t}\n\n\tif _, ok := awsutils.InstanceNetworkingLimits[instanceType]; !ok {\n\t\treturn \"\", ErrorInstanceTypeNotSupportedByCortex(instanceType)\n\t}\n\n\treturn instanceType, nil\n}\n\nfunc validatePrometheusInstanceType(instanceType string) (string, error) {\n\t_, err := validateInstanceType(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tisGPU, err := aws.IsGPUInstance(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif isGPU {\n\t\treturn \"\", ErrorGPUInstancesNotSupported(instanceType)\n\t}\n\n\tisInf, err := aws.IsInferentiaInstance(instanceType)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif isInf {\n\t\treturn \"\", ErrorInferentiaInstancesNotSupported(instanceType)\n\t}\n\n\treturn instanceType, nil\n}\n\nfunc validateInstanceDistribution(instances []string) ([]string, error) {\n\tfor _, instance := range instances {\n\t\t_, err := validateInstanceType(instance)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn instances, nil\n}\n\nfunc (ng *NodeGroup) DeepCopy() (NodeGroup, error) {\n\tdeepCopied := NodeGroup{}\n\terr := structs.DeepCopy(&deepCopied, ng)\n\tif err != nil {\n\t\treturn NodeGroup{}, err\n\t}\n\n\treturn deepCopied, nil\n}\n\nfunc (ng *NodeGroup) Hash() (string, error) {\n\tbytes, err := yaml.Marshal(ng)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\thash := sha256.New()\n\thash.Write(bytes)\n\treturn hex.EncodeToString(hash.Sum(nil)), nil\n}\n\nfunc (ng *NodeGroup) MaxPossibleOnDemandInstances() int64 {\n\tif !ng.Spot || ng.SpotConfig == nil {\n\t\treturn ng.MaxInstances\n\t}\n\n\tonDemandBaseCap, onDemandPctAboveBaseCap := ng.SpotConfigOnDemandValues()\n\treturn onDemandBaseCap + int64(math.Ceil(float64(onDemandPctAboveBaseCap)/100*float64(ng.MaxInstances-onDemandBaseCap)))\n}\n\nfunc (ng *NodeGroup) MaxPossibleSpotInstances() int64 {\n\tif !ng.Spot {\n\t\treturn 0\n\t}\n\n\tif ng.SpotConfig == nil {\n\t\treturn ng.MaxInstances\n\t}\n\n\tonDemandBaseCap, onDemandPctAboveBaseCap := ng.SpotConfigOnDemandValues()\n\treturn ng.MaxInstances - onDemandBaseCap - int64(math.Floor(float64(onDemandPctAboveBaseCap)/100*float64(ng.MaxInstances-onDemandBaseCap)))\n}\n\nfunc (ng *NodeGroup) SpotConfigOnDemandValues() (int64, int64) {\n\t// default OnDemandBaseCapacity is 0\n\tvar onDemandBaseCapacity int64 = 0\n\tif ng.SpotConfig.OnDemandBaseCapacity != nil {\n\t\tonDemandBaseCapacity = *ng.SpotConfig.OnDemandBaseCapacity\n\t}\n\n\t// default OnDemandPercentageAboveBaseCapacity is 0\n\tvar onDemandPercentageAboveBaseCapacity int64 = 0\n\tif ng.SpotConfig.OnDemandPercentageAboveBaseCapacity != nil {\n\t\tonDemandPercentageAboveBaseCapacity = *ng.SpotConfig.OnDemandPercentageAboveBaseCapacity\n\t}\n\n\treturn onDemandBaseCapacity, onDemandPercentageAboveBaseCapacity\n}\n\nfunc doesStackExist(stack *cloudformation.StackSummary) bool {\n\tif stack == nil || stack.StackName == nil || slices.HasString([]string{\n\t\tcloudformation.StackStatusDeleteComplete,\n\t\tcloudformation.StackStatusDeleteInProgress,\n\t}, *stack.StackStatus) {\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc getStaleEksNodeGroups(clusterName string, eksNodeGroupStacks []*cloudformation.StackSummary, ngsToExist, ngsMarkedForRemoval []*NodeGroup) []string {\n\teksNodeGroupsToRemove := strset.New()\n\tfor _, ng := range ngsMarkedForRemoval {\n\t\tlifecycle := \"d\"\n\t\tif ng.Spot {\n\t\t\tlifecycle = \"s\"\n\t\t}\n\n\t\teksNgName := fmt.Sprintf(\"cx-w%s-%s\", lifecycle, ng.Name)\n\t\teksStackName := fmt.Sprintf(\"eksctl-%s-nodegroup-cx-w%s-%s\", clusterName, lifecycle, ng.Name)\n\t\tfor _, eksNgStack := range eksNodeGroupStacks {\n\t\t\tif eksNgStack == nil || eksNgStack.StackName == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif *eksNgStack.StackName == eksStackName {\n\t\t\t\teksNodeGroupsToRemove.Add(eksNgName)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, eksNgStack := range eksNodeGroupStacks {\n\t\tif !doesStackExist(eksNgStack) {\n\t\t\tcontinue\n\t\t}\n\n\t\tfoundNg := false\n\t\tfor _, ng := range ngsToExist {\n\t\t\tlifecycle := \"d\"\n\t\t\tif ng.Spot {\n\t\t\t\tlifecycle = \"s\"\n\t\t\t}\n\t\t\teksStackName := fmt.Sprintf(\"eksctl-%s-nodegroup-cx-w%s-%s\", clusterName, lifecycle, ng.Name)\n\t\t\tif *eksNgStack.StackName == eksStackName {\n\t\t\t\tfoundNg = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !foundNg {\n\t\t\teksNgName := (*eksNgStack.StackName)[len(fmt.Sprintf(\"eksctl-%s-nodegroup-\", clusterName)):]\n\t\t\teksNodeGroupsToRemove.Add(eksNgName)\n\t\t}\n\t}\n\n\treturn eksNodeGroupsToRemove.Slice()\n}\n\nfunc (cc *CoreConfig) TelemetryEvent() map[string]interface{} {\n\tevent := make(map[string]interface{})\n\n\tif cc.ClusterName != \"cortex\" {\n\t\tevent[\"cluster_name._is_custom\"] = true\n\t}\n\n\tevent[\"region\"] = cc.Region\n\tevent[\"prometheus_instance_type\"] = cc.PrometheusInstanceType\n\n\tif !strings.HasPrefix(cc.ImageOperator, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_operator._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageControllerManager, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_operator_controller_manager._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageManager, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_manager._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageKubexit, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_kubexit._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageProxy, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_proxy._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageActivator, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_activator._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageAutoscaler, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_autoscaler._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageAsyncGateway, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_async_gateway._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageEnqueuer, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_enqueuer._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageDequeuer, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_dequeuer._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageClusterAutoscaler, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_cluster_autoscaler._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageMetricsServer, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_metrics_server._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageNvidiaDevicePlugin, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_nvidia_device_plugin._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageNeuronDevicePlugin, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_neuron_device_plugin._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageNeuronScheduler, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_neuron_scheduler._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageFluentBit, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_fluent_bit._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageIstioProxy, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_istio_proxy._is_custom\"] = true\n\t}\n\tif !strings.HasPrefix(cc.ImageIstioPilot, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_istio_pilot._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImagePrometheus, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_prometheus._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImagePrometheusConfigReloader, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_prometheus_config_reloader._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImagePrometheusOperator, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_prometheus_operator._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImagePrometheusStatsDExporter, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_prometheus_statsd_exporter._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImagePrometheusDCGMExporter, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_prometheus_dcgm_exporter._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImagePrometheusKubeStateMetrics, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_prometheus_kube_state_metrics._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImagePrometheusNodeExporter, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_prometheus_node_exporter._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImageKubeRBACProxy, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_kube_rbac_proxy._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImageGrafana, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_grafana._is_custom\"] = true\n\t}\n\tif strings.HasPrefix(cc.ImageEventExporter, \"quay.io/cortexlabs/\") {\n\t\tevent[\"image_event_exporter._is_custom\"] = true\n\t}\n\n\tif len(cc.Tags) > 0 {\n\t\tevent[\"tags._is_defined\"] = true\n\t\tevent[\"tags._len\"] = len(cc.Tags)\n\t}\n\tif len(cc.AvailabilityZones) > 0 {\n\t\tevent[\"availability_zones._is_defined\"] = true\n\t\tevent[\"availability_zones._len\"] = len(cc.AvailabilityZones)\n\t\tevent[\"availability_zones\"] = cc.AvailabilityZones\n\t}\n\tif len(cc.Subnets) > 0 {\n\t\tevent[\"subnets._is_defined\"] = true\n\t\tevent[\"subnets._len\"] = len(cc.Subnets)\n\t\tevent[\"subnets\"] = cc.Subnets\n\t}\n\tif cc.SSLCertificateARN != nil {\n\t\tevent[\"ssl_certificate_arn._is_defined\"] = true\n\t}\n\n\t// CortexPolicyARN should be managed by cortex\n\tif !strset.New(_defaultIAMPolicies...).IsEqual(strset.New(cc.IAMPolicyARNs...)) {\n\t\tevent[\"iam_policy_arns._is_custom\"] = true\n\t}\n\tevent[\"iam_policy_arns._len\"] = len(cc.IAMPolicyARNs)\n\n\tevent[\"subnet_visibility\"] = cc.SubnetVisibility\n\tevent[\"nat_gateway\"] = cc.NATGateway\n\tevent[\"api_load_balancer_type\"] = cc.APILoadBalancerType\n\tevent[\"api_load_balancer_scheme\"] = cc.APILoadBalancerScheme\n\tevent[\"operator_load_balancer_scheme\"] = cc.OperatorLoadBalancerScheme\n\tif cc.VPCCIDR != nil {\n\t\tevent[\"vpc_cidr._is_defined\"] = true\n\t}\n\n\tonDemandInstanceTypes := strset.New()\n\tspotInstanceTypes := strset.New()\n\tvar totalMinSize, totalMaxSize int\n\n\tevent[\"node_groups._len\"] = len(cc.NodeGroups)\n\tfor i, ng := range cc.NodeGroups {\n\t\tnodeGroupKey := func(field string) string {\n\t\t\treturn fmt.Sprintf(\"node_groups.%d.%s\", i, field)\n\t\t}\n\n\t\tevent[nodeGroupKey(\"_is_defined\")] = true\n\t\tevent[nodeGroupKey(\"name\")] = ng.Name\n\t\tevent[nodeGroupKey(\"instance_type\")] = ng.InstanceType\n\t\tevent[nodeGroupKey(\"min_instances\")] = ng.MinInstances\n\t\tevent[nodeGroupKey(\"max_instances\")] = ng.MaxInstances\n\t\tevent[nodeGroupKey(\"priority\")] = ng.Priority\n\t\tevent[nodeGroupKey(\"instance_volume_size\")] = ng.InstanceVolumeSize\n\t\tevent[nodeGroupKey(\"instance_volume_type\")] = ng.InstanceVolumeType\n\t\tif ng.InstanceVolumeIOPS != nil {\n\t\t\tevent[nodeGroupKey(\"instance_volume_iops.is_defined\")] = true\n\t\t\tevent[nodeGroupKey(\"instance_volume_iops\")] = *ng.InstanceVolumeIOPS\n\t\t}\n\t\tif ng.InstanceVolumeThroughput != nil {\n\t\t\tevent[nodeGroupKey(\"instance_volume_throughput.is_defined\")] = true\n\t\t\tevent[nodeGroupKey(\"instance_volume_throughput\")] = *ng.InstanceVolumeThroughput\n\t\t}\n\n\t\tevent[nodeGroupKey(\"spot\")] = ng.Spot\n\t\tif !ng.Spot {\n\t\t\tonDemandInstanceTypes.Add(ng.InstanceType)\n\t\t} else {\n\t\t\tspotInstanceTypes.Add(ng.InstanceType)\n\t\t}\n\t\tif ng.SpotConfig != nil {\n\t\t\tevent[nodeGroupKey(\"spot_config._is_defined\")] = true\n\t\t\tif len(ng.SpotConfig.InstanceDistribution) > 0 {\n\t\t\t\tevent[nodeGroupKey(\"spot_config.instance_distribution._is_defined\")] = true\n\t\t\t\tevent[nodeGroupKey(\"spot_config.instance_distribution._len\")] = len(ng.SpotConfig.InstanceDistribution)\n\t\t\t\tevent[nodeGroupKey(\"spot_config.instance_distribution\")] = ng.SpotConfig.InstanceDistribution\n\t\t\t\tspotInstanceTypes.Add(ng.SpotConfig.InstanceDistribution...)\n\t\t\t}\n\t\t\tif ng.SpotConfig.OnDemandBaseCapacity != nil {\n\t\t\t\tevent[nodeGroupKey(\"spot_config.on_demand_base_capacity._is_defined\")] = true\n\t\t\t\tevent[nodeGroupKey(\"spot_config.on_demand_base_capacity\")] = *ng.SpotConfig.OnDemandBaseCapacity\n\t\t\t}\n\t\t\tif ng.SpotConfig.OnDemandPercentageAboveBaseCapacity != nil {\n\t\t\t\tevent[nodeGroupKey(\"spot_config.on_demand_percentage_above_base_capacity._is_defined\")] = true\n\t\t\t\tevent[nodeGroupKey(\"spot_config.on_demand_percentage_above_base_capacity\")] = *ng.SpotConfig.OnDemandPercentageAboveBaseCapacity\n\t\t\t}\n\t\t\tif ng.SpotConfig.MaxPrice != nil {\n\t\t\t\tevent[nodeGroupKey(\"spot_config.max_price._is_defined\")] = true\n\t\t\t\tevent[nodeGroupKey(\"spot_config.max_price\")] = *ng.SpotConfig.MaxPrice\n\t\t\t}\n\t\t\tif ng.SpotConfig.InstancePools != nil {\n\t\t\t\tevent[nodeGroupKey(\"spot_config.instance_pools._is_defined\")] = true\n\t\t\t\tevent[nodeGroupKey(\"spot_config.instance_pools\")] = *ng.SpotConfig.InstancePools\n\t\t\t}\n\t\t}\n\n\t\ttotalMinSize += int(ng.MinInstances)\n\t\ttotalMaxSize += int(ng.MaxInstances)\n\t}\n\n\tevent[\"node_groups._total_min_size\"] = totalMinSize\n\tevent[\"node_groups._total_max_size\"] = totalMaxSize\n\tevent[\"node_groups._on_demand_instances\"] = onDemandInstanceTypes.Slice()\n\tevent[\"node_groups._spot_instances\"] = spotInstanceTypes.Slice()\n\tevent[\"node_groups._instances\"] = strset.Union(onDemandInstanceTypes, spotInstanceTypes).Slice()\n\n\treturn event\n}\n\nfunc (cc *CoreConfig) GetNodeGroupByName(name string) *NodeGroup {\n\tfor _, ng := range cc.NodeGroups {\n\t\tif ng.Name == name {\n\t\t\tmatchedNodeGroup := *ng\n\t\t\treturn &matchedNodeGroup\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (cc *CoreConfig) GetNodeGroupNames() []string {\n\tallNodeGroupNames := make([]string, len(cc.NodeGroups))\n\tfor i := range cc.NodeGroups {\n\t\tallNodeGroupNames[i] = cc.NodeGroups[i].Name\n\t}\n\n\treturn allNodeGroupNames\n}\n\nfunc BucketName(accountID, clusterName, region string) string {\n\tbucketID := libhash.String(accountID + region)[:8] // this is to \"guarantee\" a globally unique name\n\treturn clusterName + \"-\" + bucketID\n}\n\nfunc validateClusterName(clusterName string) (string, error) {\n\tif !_strictS3BucketRegex.MatchString(clusterName) {\n\t\treturn \"\", errors.Wrap(ErrorDidNotMatchStrictS3Regex(), clusterName)\n\t}\n\treturn clusterName, nil\n}\n\nfunc validateImageVersion(image string) (string, error) {\n\treturn cr.ValidateImageVersion(image, consts.CortexVersion)\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/config_key.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\nconst (\n\tBucketKey     = \"bucket\"\n\tClusterUIDKey = \"cluster_uid\"\n\n\tClusterNameKey                         = \"cluster_name\"\n\tRegionKey                              = \"region\"\n\tPrometheusInstanceTypeKey              = \"prometheus_instance_type\"\n\tNodeGroupsKey                          = \"node_groups\"\n\tInstanceTypeKey                        = \"instance_type\"\n\tAcceleratorTypeKey                     = \"accelerator_type\"\n\tAcceleratorsPerInstanceKey             = \"accelerators_per_instance\"\n\tMinInstancesKey                        = \"min_instances\"\n\tMaxInstancesKey                        = \"max_instances\"\n\tPriorityKey                            = \"priority\"\n\tSpotKey                                = \"spot\"\n\tSpotConfigKey                          = \"spot_config\"\n\tInstanceDistributionKey                = \"instance_distribution\"\n\tOnDemandBaseCapacityKey                = \"on_demand_base_capacity\"\n\tOnDemandPercentageAboveBaseCapacityKey = \"on_demand_percentage_above_base_capacity\"\n\tInstanceVolumeSizeKey                  = \"instance_volume_size\"\n\tInstanceVolumeTypeKey                  = \"instance_volume_type\"\n\tInstanceVolumeIOPSKey                  = \"instance_volume_iops\"\n\tInstanceVolumeThroughputKey            = \"instance_volume_throughput\"\n\tInstancePoolsKey                       = \"instance_pools\"\n\tMaxPriceKey                            = \"max_price\"\n\tNetworkKey                             = \"network\"\n\tSubnetKey                              = \"subnet\"\n\tTagsKey                                = \"tags\"\n\tAvailabilityZonesKey                   = \"availability_zones\"\n\tSubnetsKey                             = \"subnets\"\n\tAvailabilityZoneKey                    = \"availability_zone\"\n\tSubnetIDKey                            = \"subnet_id\"\n\tSSLCertificateARNKey                   = \"ssl_certificate_arn\"\n\tCortexPolicyARNKey                     = \"cortex_policy_arn\"\n\tIAMPolicyARNsKey                       = \"iam_policy_arns\"\n\tSubnetVisibilityKey                    = \"subnet_visibility\"\n\tNATGatewayKey                          = \"nat_gateway\"\n\tAPILoadBalancerSchemeKey               = \"api_load_balancer_scheme\"\n\tOperatorLoadBalancerSchemeKey          = \"operator_load_balancer_scheme\"\n\tAPILoadBalancerCIDRWhiteListKey        = \"api_load_balancer_cidr_white_list\"\n\tOperatorLoadBalancerCIDRWhiteListKey   = \"operator_load_balancer_cidr_white_list\"\n\tVPCCIDRKey                             = \"vpc_cidr\"\n\tAccountIDKey                           = \"account_id\"\n\tTelemetryKey                           = \"telemetry\"\n)\n"
  },
  {
    "path": "pkg/types/clusterconfig/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n)\n\nconst (\n\tErrInvalidProvider                        = \"clusterconfig.invalid_provider\"\n\tErrInvalidLegacyProvider                  = \"clusterconfig.invalid_legacy_provider\"\n\tErrDisallowedField                        = \"clusterconfig.disallowed_field\"\n\tErrInvalidRegion                          = \"clusterconfig.invalid_region\"\n\tErrNodeGroupMaxInstancesIsZero            = \"clusterconfig.node_group_max_instances_is_zero\"\n\tErrMaxNumOfNodeGroupsReached              = \"clusterconfig.max_num_of_nodegroups_reached\"\n\tErrDuplicateNodeGroupName                 = \"clusterconfig.duplicate_nodegroup_name\"\n\tErrMaxNodesToAddOnClusterUp               = \"clusterconfig.max_nodes_to_add_on_cluster_up\"\n\tErrMaxNodesToAddOnClusterConfigure        = \"clusterconfig.max_nodes_to_add_on_cluster_configure\"\n\tErrInstanceTypeTooSmall                   = \"clusterconfig.instance_type_too_small\"\n\tErrMinInstancesGreaterThanMax             = \"clusterconfig.min_instances_greater_than_max\"\n\tErrInstanceTypeNotSupportedInRegion       = \"clusterconfig.instance_type_not_supported_in_region\"\n\tErrIncompatibleSpotInstanceTypeMemory     = \"clusterconfig.incompatible_spot_instance_type_memory\"\n\tErrIncompatibleSpotInstanceTypeCPU        = \"clusterconfig.incompatible_spot_instance_type_cpu\"\n\tErrIncompatibleSpotInstanceTypeGPU        = \"clusterconfig.incompatible_spot_instance_type_gpu\"\n\tErrIncompatibleSpotInstanceTypeInf        = \"clusterconfig.incompatible_spot_instance_type_inf\"\n\tErrSpotPriceGreaterThanTargetOnDemand     = \"clusterconfig.spot_price_greater_than_target_on_demand\"\n\tErrSpotPriceGreaterThanMaxPrice           = \"clusterconfig.spot_price_greater_than_max_price\"\n\tErrInstanceTypeNotSupportedByCortex       = \"clusterconfig.instance_type_not_supported_by_cortex\"\n\tErrAMDGPUInstancesNotSupported            = \"clusterconfig.amd_gpu_instances_not_supported\"\n\tErrGPUInstancesNotSupported               = \"clusterconfig.gpu_instance_not_supported\"\n\tErrInferentiaInstancesNotSupported        = \"clusterconfig.inferentia_instances_not_supported\"\n\tErrMacInstancesNotSupported               = \"clusterconfig.mac_instances_not_supported\"\n\tErrFPGAInstancesNotSupported              = \"clusterconfig.fpga_instances_not_supported\"\n\tErrAlevoInstancesNotSupported             = \"clusterconfig.alevo_instances_not_supported\"\n\tErrGaudiInstancesNotSupported             = \"clusterconfig.gaudi_instances_not_supported\"\n\tErrTrainiumInstancesNotSupported          = \"clusterconfig.trainium_instances_not_supported\"\n\tErrAtLeastOneInstanceDistribution         = \"clusterconfig.at_least_one_instance_distribution\"\n\tErrNoCompatibleSpotInstanceFound          = \"clusterconfig.no_compatible_spot_instance_found\"\n\tErrConfiguredWhenSpotIsNotEnabled         = \"clusterconfig.configured_when_spot_is_not_enabled\"\n\tErrOnDemandBaseCapacityGreaterThanMax     = \"clusterconfig.on_demand_base_capacity_greater_than_max\"\n\tErrInvalidAvailabilityZone                = \"clusterconfig.invalid_availability_zone\"\n\tErrAvailabilityZoneSpecifiedTwice         = \"clusterconfig.availability_zone_specified_twice\"\n\tErrUnsupportedAvailabilityZone            = \"clusterconfig.unsupported_availability_zone\"\n\tErrNotEnoughValidDefaultAvailibilityZones = \"clusterconfig.not_enough_valid_default_availability_zones\"\n\tErrNoNATGatewayWithSubnets                = \"clusterconfig.no_nat_gateway_with_subnets\"\n\tErrSubnetMaskOutOfRange                   = \"clusterconfig.subnet_mask_out_of_range\"\n\tErrConfigCannotBeChangedOnConfigure       = \"clusterconfig.config_cannot_be_changed_on_configure\"\n\tErrNodeGroupCanOnlyBeScaled               = \"clusterconfig.node_group_can_only_be_scaled\"\n\tErrSpecifyOneOrNone                       = \"clusterconfig.specify_one_or_none\"\n\tErrSpecifyTwoOrNone                       = \"clusterconfig.specify_two_or_none\"\n\tErrDependentFieldMustBeSpecified          = \"clusterconfig.dependent_field_must_be_specified\"\n\tErrFieldConfigurationDependentOnCondition = \"clusterconfig.field_configuration_dependent_on_condition\"\n\tErrDidNotMatchStrictS3Regex               = \"clusterconfig.did_not_match_strict_s3_regex\"\n\tErrNATRequiredWithPrivateSubnetVisibility = \"clusterconfig.nat_required_with_private_subnet_visibility\"\n\tErrS3RegionDiffersFromCluster             = \"clusterconfig.s3_region_differs_from_cluster\"\n\tErrIOPSNotSupported                       = \"clusterconfig.iops_not_supported\"\n\tErrThroughputNotSupported                 = \"clusterconfig.throughput_not_supported\"\n\tErrIOPSTooSmall                           = \"clusterconfig.iops_too_small\"\n\tErrIOPSTooLarge                           = \"clusterconfig.iops_too_large\"\n\tErrIOPSToVolumeSizeRatio                  = \"clusterconfig.iops_to_volume_size_ratio\"\n\tErrIOPSToThroughputRatio                  = \"clusterconfig.iops_to_throughput_ratio\"\n\tErrCantOverrideDefaultTag                 = \"clusterconfig.cant_override_default_tag\"\n\tErrSSLCertificateARNNotFound              = \"clusterconfig.ssl_certificate_arn_not_found\"\n\tErrIAMPolicyARNNotFound                   = \"clusterconfig.iam_policy_arn_not_found\"\n)\n\nfunc ErrorInvalidProvider(providerStr string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidProvider,\n\t\tMessage: fmt.Sprintf(\"\\\"%s\\\" is not a supported provider; only aws is supported, so the provider field may be removed from your cluster configuration file\", providerStr),\n\t})\n}\n\nfunc ErrorInvalidLegacyProvider(providerStr string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidLegacyProvider,\n\t\tMessage: fmt.Sprintf(\"the %s provider is no longer supported on cortex v%s; only aws is supported, so the provider field may be removed from your cluster configuration file\", providerStr, consts.CortexVersionMinor),\n\t})\n}\n\nfunc ErrorDisallowedField(field string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDisallowedField,\n\t\tMessage: fmt.Sprintf(\"the %s field cannot be configured by the user\", field),\n\t})\n}\n\nfunc ErrorInvalidRegion(region string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidRegion,\n\t\tMessage: fmt.Sprintf(\"%s is not a valid AWS region, or is an AWS region which is not supported by AWS EKS; please choose one of the following regions: %s\", s.UserStr(region), strings.Join(aws.EKSSupportedRegions.SliceSorted(), \", \")),\n\t})\n}\n\nfunc ErrorNodeGroupMaxInstancesIsZero() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNodeGroupMaxInstancesIsZero,\n\t\tMessage: fmt.Sprintf(\"nodegroups cannot be created with `%s` set to 0 (but `%s` can be scaled to 0 after the nodegroup has been created)\", MaxInstancesKey, MaxInstancesKey),\n\t})\n}\n\nfunc ErrorMaxNumOfNodeGroupsReached(maxNodeGroups int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMaxNumOfNodeGroupsReached,\n\t\tMessage: fmt.Sprintf(\"cannot have more than %d nodegroups\", maxNodeGroups),\n\t})\n}\n\nfunc ErrorDuplicateNodeGroupName(duplicateNgName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDuplicateNodeGroupName,\n\t\tMessage: fmt.Sprintf(\"cannot have multiple nodegroups with the same name (%s)\", duplicateNgName),\n\t})\n}\n\nfunc ErrorMaxNodesToAddOnClusterUp(requestedNodes, maxNodes int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMaxNodesToAddOnClusterUp,\n\t\tMessage: fmt.Sprintf(\"cannot create a cluster with %d instances (at most %d instances can be created initially); reduce %s for your nodegroups (you may add additional instances via the `cortex cluster configure` command after your cluster has been created)\", requestedNodes, maxNodes, MinInstancesKey),\n\t})\n}\n\nfunc ErrorMaxNodesToAddOnClusterConfigure(requestedNodes, currentNodes, maxNodes int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMaxNodesToAddOnClusterConfigure,\n\t\tMessage: fmt.Sprintf(\"cannot add %d instances to your cluster (you requested %d total instances, but your cluster currently has %d instances); only %d instances can be added at time, so reduce the sum of %s across all nodegroups by %d\", requestedNodes-currentNodes, requestedNodes, currentNodes, maxNodes, MinInstancesKey, requestedNodes-currentNodes-maxNodes),\n\t})\n}\n\nfunc ErrorInstanceTypeTooSmall(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInstanceTypeTooSmall,\n\t\tMessage: fmt.Sprintf(\"%s: cortex does not support nano or micro instances - please specify a larger instance type\", instanceType),\n\t})\n}\n\nfunc ErrorMinInstancesGreaterThanMax(min int64, max int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMinInstancesGreaterThanMax,\n\t\tMessage: fmt.Sprintf(\"%s cannot be greater than %s (%d > %d)\", MinInstancesKey, MaxInstancesKey, min, max),\n\t})\n}\n\nfunc ErrorInstanceTypeNotSupportedInRegion(instanceType string, region string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInstanceTypeNotSupportedInRegion,\n\t\tMessage: fmt.Sprintf(\"%s instances are not available in %s\", instanceType, region),\n\t})\n}\n\nfunc ErrorIncompatibleSpotInstanceTypeMemory(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIncompatibleSpotInstanceTypeMemory,\n\t\tMessage: fmt.Sprintf(\"all instances must have at least as much memory as %s (%s has %s memory, but %s only has %s memory)\", target.Type, target.Type, target.Memory.String(), suggested.Type, suggested.Memory.String()),\n\t})\n}\n\nfunc ErrorIncompatibleSpotInstanceTypeCPU(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIncompatibleSpotInstanceTypeCPU,\n\t\tMessage: fmt.Sprintf(\"all instances must have at least as much CPU as %s (%s has %s CPU, but %s only has %s CPU)\", target.Type, target.Type, target.CPU.String(), suggested.Type, suggested.CPU.String()),\n\t})\n}\n\nfunc ErrorIncompatibleSpotInstanceTypeGPU(target aws.InstanceMetadata, suggested aws.InstanceMetadata) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIncompatibleSpotInstanceTypeGPU,\n\t\tMessage: fmt.Sprintf(\"all instances must have at least as much GPU as %s (%s has %d GPU, but %s only has %d GPU)\", target.Type, target.Type, target.GPU, suggested.Type, suggested.GPU),\n\t})\n}\n\nfunc ErrorIncompatibleSpotInstanceTypeInf(suggested aws.InstanceMetadata) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIncompatibleSpotInstanceTypeInf,\n\t\tMessage: fmt.Sprintf(\"all instances must have at least 1 Inferentia chip, but %s doesn't have any\", suggested.Type),\n\t})\n}\n\nfunc ErrorSpotPriceGreaterThanTargetOnDemand(spotPrice float64, target aws.InstanceMetadata, suggested aws.InstanceMetadata) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSpotPriceGreaterThanTargetOnDemand,\n\t\tMessage: fmt.Sprintf(\"%s will not be allocated because its current spot price is $%g which is greater than %s's on-demand price of $%g\", suggested.Type, spotPrice, target.Type, target.Price),\n\t})\n}\n\nfunc ErrorSpotPriceGreaterThanMaxPrice(suggestedSpotPrice float64, maxPrice float64, suggested aws.InstanceMetadata) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSpotPriceGreaterThanMaxPrice,\n\t\tMessage: fmt.Sprintf(\"%s will not be allocated because its current spot price is $%g which is greater than the configured max price $%g\", suggested.Type, suggestedSpotPrice, maxPrice),\n\t})\n}\n\nfunc ErrorInstanceTypeNotSupportedByCortex(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInstanceTypeNotSupportedByCortex,\n\t\tMessage: fmt.Sprintf(\"instance type %s is not supported by cortex\", instanceType),\n\t})\n}\n\nfunc ErrorAMDGPUInstancesNotSupported(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAMDGPUInstancesNotSupported,\n\t\tMessage: fmt.Sprintf(\"AMD GPU instances (including %s) are not supported by cortex\", instanceType),\n\t})\n}\n\nfunc ErrorGPUInstancesNotSupported(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrGPUInstancesNotSupported,\n\t\tMessage: fmt.Sprintf(\"GPU instances (including %s) are not supported\", instanceType),\n\t})\n}\n\nfunc ErrorInferentiaInstancesNotSupported(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInferentiaInstancesNotSupported,\n\t\tMessage: fmt.Sprintf(\"Inferentia instances (including %s) are not supported\", instanceType),\n\t})\n}\n\nfunc ErrorMacInstancesNotSupported(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMacInstancesNotSupported,\n\t\tMessage: fmt.Sprintf(\"mac instances (including %s) are not supported by cortex\", instanceType),\n\t})\n}\n\nfunc ErrorFPGAInstancesNotSupported(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFPGAInstancesNotSupported,\n\t\tMessage: fmt.Sprintf(\"FPGA instances (including %s) are not supported by cortex\", instanceType),\n\t})\n}\n\nfunc ErrorAlevoInstancesNotSupported(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAlevoInstancesNotSupported,\n\t\tMessage: fmt.Sprintf(\"Alevo instances (including %s) are not supported by cortex\", instanceType),\n\t})\n}\n\nfunc ErrorGaudiInstancesNotSupported(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrGaudiInstancesNotSupported,\n\t\tMessage: fmt.Sprintf(\"Gaudi instances (including %s) are not supported by cortex\", instanceType),\n\t})\n}\n\nfunc ErrorTrainiumInstancesNotSupported(instanceType string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrTrainiumInstancesNotSupported,\n\t\tMessage: fmt.Sprintf(\"Trainium instances (including %s) are not supported by cortex\", instanceType),\n\t})\n}\n\nfunc ErrorConfiguredWhenSpotIsNotEnabled(configKey string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrConfiguredWhenSpotIsNotEnabled,\n\t\tMessage: fmt.Sprintf(\"%s cannot be specified unless spot is enabled (to enable spot instances, set `%s: true` in your cluster configuration file)\", configKey, SpotKey),\n\t})\n}\n\nfunc ErrorOnDemandBaseCapacityGreaterThanMax(onDemandBaseCapacity int64, max int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrOnDemandBaseCapacityGreaterThanMax,\n\t\tMessage: fmt.Sprintf(\"%s cannot be greater than %s (%d > %d)\", OnDemandBaseCapacityKey, MaxInstancesKey, onDemandBaseCapacity, max),\n\t})\n}\n\nfunc ErrorInvalidAvailabilityZone(userZone string, allZones strset.Set, region string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidAvailabilityZone,\n\t\tMessage: fmt.Sprintf(\"%s is not an availability zone in %s; please choose from the following availability zones: %s\", userZone, region, s.StrsOr(allZones.SliceSorted())),\n\t})\n}\n\nfunc ErrorAvailabilityZoneSpecifiedTwice(zone string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrAvailabilityZoneSpecifiedTwice,\n\t\tMessage: fmt.Sprintf(\"availability zone \\\"%s\\\" is specified twice\", zone),\n\t})\n}\n\nfunc ErrorUnsupportedAvailabilityZone(userZone string, instanceType string, instanceTypes ...string) error {\n\tmsg := fmt.Sprintf(\"the %s availability zone does not support EKS and the %s instance type; please choose a different availability zone, instance type, or region\", userZone, instanceType)\n\n\tif len(instanceTypes) > 0 {\n\t\tallInstanceTypes := append([]string{instanceType}, instanceTypes...)\n\t\tmsg = fmt.Sprintf(\"the %s availability zone does not support EKS and %s instance types; please choose a different availability zone, instance types, or region\", userZone, s.StrsAnd(allInstanceTypes))\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUnsupportedAvailabilityZone,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorNotEnoughDefaultSupportedZones(region string, validZones strset.Set, instanceType string, instanceTypes ...string) error {\n\tareNoStr := \"are no\"\n\tif len(validZones) > 0 {\n\t\tareNoStr = \"aren't enough\"\n\t}\n\n\tmsg := fmt.Sprintf(\"there %s availability zones in %s which support EKS and the %s instance type; please choose a different instance type or a different region\", areNoStr, region, instanceType)\n\n\tif len(instanceTypes) > 0 {\n\t\tallInstanceTypes := append([]string{instanceType}, instanceTypes...)\n\t\tmsg = fmt.Sprintf(\"there %s availability zones in %s which support EKS and the %s instance types; please choose different instance types or a different region\", areNoStr, region, s.StrsAnd(allInstanceTypes))\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNotEnoughValidDefaultAvailibilityZones,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorNoNATGatewayWithSubnets() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoNATGatewayWithSubnets,\n\t\tMessage: fmt.Sprintf(\"nat gateway cannot be automatically created when specifying subnets for your cluster; please unset %s or %s\", NATGatewayKey, SubnetsKey),\n\t})\n}\n\nfunc ErrorSubnetMaskOutOfRange(requestedMaskSize, minMaskSize, maxMaskSize int) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSubnetMaskOutOfRange,\n\t\tMessage: fmt.Sprintf(\"invalid network size /%d; the network size must be between /%d and /%d\", requestedMaskSize, minMaskSize, maxMaskSize),\n\t})\n}\n\nfunc ErrorConfigCannotBeChangedOnConfigure() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrConfigCannotBeChangedOnConfigure,\n\t\tMessage: fmt.Sprintf(\"in a running cluster, only %s can be modified\", s.StrsAnd([]string{NodeGroupsKey, SSLCertificateARNKey, OperatorLoadBalancerCIDRWhiteListKey, APILoadBalancerCIDRWhiteListKey})),\n\t})\n}\n\nfunc ErrorNodeGroupCanOnlyBeScaled() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNodeGroupCanOnlyBeScaled,\n\t\tMessage: \"in a running cluster, nodegroups can only be scaled, added, or deleted\",\n\t})\n}\n\nfunc ErrorSpecifyOneOrNone(fieldName1 string, fieldName2 string, fieldNames ...string) error {\n\tfieldNames = append([]string{fieldName1, fieldName2}, fieldNames...)\n\n\tmessage := fmt.Sprintf(\"specify exactly one or none of the following fields: %s\", s.StrsAnd(fieldNames))\n\tif len(fieldNames) == 2 {\n\t\tmessage = fmt.Sprintf(\"cannot specify both %s and %s; specify only one (or neither)\", fieldNames[0], fieldNames[1])\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSpecifyOneOrNone,\n\t\tMessage: message,\n\t})\n}\n\nfunc ErrorSpecifyTwoOrNone(fieldName1 string, fieldName2 string, fieldNames ...string) error {\n\tfieldNames = append([]string{fieldName1, fieldName2}, fieldNames...)\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSpecifyTwoOrNone,\n\t\tMessage: fmt.Sprintf(\"specify exactly two or none of the following fields: %s\", s.StrsAnd(fieldNames)),\n\t})\n}\n\nfunc ErrorDependentFieldMustBeSpecified(configuredField string, dependencyField string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDependentFieldMustBeSpecified,\n\t\tMessage: fmt.Sprintf(\"%s must be specified when configuring %s\", dependencyField, configuredField),\n\t})\n}\n\nfunc ErrorFieldConfigurationDependentOnCondition(configuredField string, configuredFieldValue string, dependencyField string, dependencyFieldValue string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFieldConfigurationDependentOnCondition,\n\t\tMessage: fmt.Sprintf(\"cannot set %s = %s when %s = %s\", configuredField, configuredFieldValue, dependencyField, dependencyFieldValue),\n\t})\n}\n\nfunc ErrorDidNotMatchStrictS3Regex() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDidNotMatchStrictS3Regex,\n\t\tMessage: \"only lowercase alphanumeric characters and dashes are allowed, with no consecutive dashes and no leading or trailing dashes\",\n\t})\n}\n\nfunc ErrorNATRequiredWithPrivateSubnetVisibility() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNATRequiredWithPrivateSubnetVisibility,\n\t\tMessage: fmt.Sprintf(\"a nat gateway is required when `%s: %s` is specified; either set %s to %s or %s, or set %s to %s\", SubnetVisibilityKey, PrivateSubnetVisibility, NATGatewayKey, s.UserStr(SingleNATGateway), s.UserStr(HighlyAvailableNATGateway), SubnetVisibilityKey, s.UserStr(PublicSubnetVisibility)),\n\t})\n}\n\nfunc ErrorS3RegionDiffersFromCluster(bucketName string, bucketRegion string, clusterRegion string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrS3RegionDiffersFromCluster,\n\t\tMessage: fmt.Sprintf(\"the %s bucket already exists but is in %s (your cluster is in %s); either change the region of your cluster to %s or delete your bucket to allow cortex to create the bucket for you in %s\", bucketName, bucketRegion, clusterRegion, bucketRegion, clusterRegion),\n\t})\n}\n\nfunc ErrorIOPSNotSupported(volumeType VolumeType) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIOPSNotSupported,\n\t\tMessage: fmt.Sprintf(\"IOPS cannot be configured for volume type %s; set `%s: %s`, `%s: %s`, or remove `%s` from your cluster configuration file\", volumeType, InstanceVolumeTypeKey, IO1VolumeType, InstanceVolumeTypeKey, GP3VolumeType, InstanceVolumeIOPSKey),\n\t})\n}\n\nfunc ErrorThroughputNotSupported(volumeType VolumeType) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrThroughputNotSupported,\n\t\tMessage: fmt.Sprintf(\"throughput cannot be configured for volume type %s; set `%s: %s` or remove `%s` from your cluster configuration file\", volumeType, InstanceVolumeTypeKey, GP3VolumeType, InstanceVolumeThroughputKey),\n\t})\n}\n\nfunc ErrorIOPSTooSmall(volumeType VolumeType, iops, minIOPS int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIOPSTooSmall,\n\t\tMessage: fmt.Sprintf(\"for %s volume type, %s (%d) cannot be smaller than %d\", volumeType, InstanceVolumeIOPSKey, iops, minIOPS),\n\t})\n}\n\nfunc ErrorIOPSTooLarge(volumeType VolumeType, iops, maxIOPS int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIOPSTooLarge,\n\t\tMessage: fmt.Sprintf(\"for %s volume type, %s (%d) cannot be larger than %d\", volumeType, InstanceVolumeIOPSKey, iops, maxIOPS),\n\t})\n}\n\nfunc ErrorIOPSToVolumeSizeRatio(volumeType VolumeType, ratio, iops int64, volumeSize int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIOPSToVolumeSizeRatio,\n\t\tMessage: fmt.Sprintf(\"for %s volume type, %s (%d) cannot be more than %d times larger than %s (%d); increase `%s` or decrease `%s` in your cluster configuration file\", volumeType, InstanceVolumeIOPSKey, iops, ratio, InstanceVolumeSizeKey, volumeSize, InstanceVolumeSizeKey, InstanceVolumeIOPSKey),\n\t})\n}\n\nfunc ErrorIOPSToThroughputRatio(volumeType VolumeType, ratio, iops, throughput int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIOPSToThroughputRatio,\n\t\tMessage: fmt.Sprintf(\"for %s volume type, %s (%d) must be at least %d times larger than %s (%d); decrease `%s` or increase `%s` in your cluster configuration file\", volumeType, InstanceVolumeIOPSKey, iops, ratio, InstanceVolumeThroughputKey, throughput, InstanceVolumeThroughputKey, InstanceVolumeIOPSKey),\n\t})\n}\n\nfunc ErrorCantOverrideDefaultTag() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCantOverrideDefaultTag,\n\t\tMessage: fmt.Sprintf(\"the \\\"%s\\\" tag cannot be overridden (it is set by default, and will always be equal to your cluster name)\", ClusterNameTag),\n\t})\n}\n\nfunc ErrorSSLCertificateARNNotFound(sslCertificateARN string, region string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSSLCertificateARNNotFound,\n\t\tMessage: fmt.Sprintf(\"unable to find the specified ssl certificate in %s: %s\", region, sslCertificateARN),\n\t})\n}\n\nfunc ErrorIAMPolicyARNNotFound(policyARN string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIAMPolicyARNNotFound,\n\t\tMessage: fmt.Sprintf(\"unable to find iam policy %s\", policyARN),\n\t})\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/load_balancer_scheme.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\ntype LoadBalancerScheme int\n\nconst (\n\tUnknownLoadBalancerScheme LoadBalancerScheme = iota\n\tInternetFacingLoadBalancerScheme\n\tInternalLoadBalancerScheme\n)\n\nvar _loadBalancerSchemes = []string{\n\t\"unknown\",\n\t\"internet-facing\",\n\t\"internal\",\n}\n\nfunc LoadBalancerSchemeFromString(s string) LoadBalancerScheme {\n\tfor i := 0; i < len(_loadBalancerSchemes); i++ {\n\t\tif s == _loadBalancerSchemes[i] {\n\t\t\treturn LoadBalancerScheme(i)\n\t\t}\n\t}\n\treturn UnknownLoadBalancerScheme\n}\n\nfunc LoadBalancerSchemeStrings() []string {\n\treturn _loadBalancerSchemes[1:]\n}\n\nfunc (t LoadBalancerScheme) String() string {\n\treturn _loadBalancerSchemes[t]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (t LoadBalancerScheme) MarshalText() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (t *LoadBalancerScheme) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_loadBalancerSchemes); i++ {\n\t\tif enum == _loadBalancerSchemes[i] {\n\t\t\t*t = LoadBalancerScheme(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*t = UnknownLoadBalancerScheme\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (t *LoadBalancerScheme) UnmarshalBinary(data []byte) error {\n\treturn t.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (t LoadBalancerScheme) MarshalBinary() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/load_balancer_type.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\ntype LoadBalancerType int\n\nconst (\n\tUnknownLoadBalancerType LoadBalancerType = iota\n\tNLBLoadBalancerType\n\tELBLoadBalancerType\n)\n\nvar _loadBalancerTypes = []string{\n\t\"unknown\",\n\t\"nlb\",\n\t\"elb\",\n}\n\nfunc LoadBalancerTypeFromString(s string) LoadBalancerType {\n\tfor i := 0; i < len(_loadBalancerTypes); i++ {\n\t\tif s == _loadBalancerTypes[i] {\n\t\t\treturn LoadBalancerType(i)\n\t\t}\n\t}\n\treturn UnknownLoadBalancerType\n}\n\nfunc LoadBalancerTypeStrings() []string {\n\treturn _loadBalancerTypes[1:]\n}\n\nfunc (t LoadBalancerType) String() string {\n\treturn _loadBalancerTypes[t]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (t LoadBalancerType) MarshalText() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (t *LoadBalancerType) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_loadBalancerTypes); i++ {\n\t\tif enum == _loadBalancerTypes[i] {\n\t\t\t*t = LoadBalancerType(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*t = UnknownLoadBalancerType\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (t *LoadBalancerType) UnmarshalBinary(data []byte) error {\n\treturn t.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (t LoadBalancerType) MarshalBinary() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/nat_gateway_type.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\ntype NATGateway int\n\nconst (\n\tUnknownNATGateway NATGateway = iota\n\tNoneNATGateway\n\tSingleNATGateway\n\tHighlyAvailableNATGateway\n)\n\nvar _natGateways = []string{\n\t\"unknown\",\n\t\"none\",\n\t\"single\",\n\t\"highly_available\",\n}\n\nfunc NATGatewayFromString(s string) NATGateway {\n\tfor i := 0; i < len(_natGateways); i++ {\n\t\tif s == _natGateways[i] {\n\t\t\treturn NATGateway(i)\n\t\t}\n\t}\n\treturn UnknownNATGateway\n}\n\nfunc NATGatewayStrings() []string {\n\treturn _natGateways[1:]\n}\n\nfunc (t NATGateway) String() string {\n\treturn _natGateways[t]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (t NATGateway) MarshalText() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (t *NATGateway) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_natGateways); i++ {\n\t\tif enum == _natGateways[i] {\n\t\t\t*t = NATGateway(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*t = UnknownNATGateway\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (t *NATGateway) UnmarshalBinary(data []byte) error {\n\treturn t.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (t NATGateway) MarshalBinary() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/network_validations.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n)\n\nconst (\n\t_elasticIPsQuotaCode         = \"L-0263D0A3\" // from EC2 service\n\t_internetGatewayQuotaCode    = \"L-A4707A72\" // from EC2 service\n\t_natGatewayQuotaCode         = \"L-FE5A380F\" // from EC2 service\n\t_vpcQuotaCode                = \"L-F678F1CE\" // from VPC service\n\t_securityGroupsQuotaCode     = \"L-E79EC296\" // from VPC service\n\t_securityGroupRulesQuotaCode = \"L-0EA8095F\" // from VPC service\n)\n\nfunc VerifyNetworkQuotas(\n\tawsClient *aws.Client,\n\trequiredInternetGateways int,\n\tnatGatewayRequired bool,\n\thighlyAvailableNATGateway bool,\n\trequiredVPCs int,\n\tavailabilityZones strset.Set,\n\tnumNodeGroups int,\n\tnetAdditionOfNodeGroups int,\n\tlongestCIDRWhiteList int,\n\tclusterAlreadyExists bool,\n) error {\n\n\tdesiredQuotaCodes := []string{\n\t\t_vpcQuotaCode,\n\t\t_securityGroupsQuotaCode,\n\t\t_securityGroupRulesQuotaCode,\n\t}\n\tserviceCodes := []string{\"vpc\"}\n\n\tif !clusterAlreadyExists {\n\t\tdesiredQuotaCodes = append(desiredQuotaCodes,\n\t\t\t_elasticIPsQuotaCode,\n\t\t\t_internetGatewayQuotaCode,\n\t\t\t_natGatewayQuotaCode,\n\t\t)\n\t\tserviceCodes = append(serviceCodes, \"ec2\")\n\t}\n\n\tquotaCodeToValueMap, err := awsClient.ListServiceQuotas(desiredQuotaCodes, serviceCodes)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar skippedValidations []string\n\tdefer func() {\n\t\tif len(skippedValidations) > 0 {\n\t\t\tfmt.Println(strings.Join(skippedValidations, \"\\n\"))\n\t\t}\n\t}()\n\n\tif !clusterAlreadyExists {\n\t\t// check internet GW quota\n\t\tif requiredInternetGateways > 0 {\n\t\t\tif internetGatewayQuota, found := quotaCodeToValueMap[_internetGatewayQuotaCode]; found {\n\t\t\t\terr := awsClient.VerifyInternetGatewayQuota(internetGatewayQuota, requiredInternetGateways)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tskippedValidations = append(skippedValidations, fmt.Sprintf(\"skipping internet gateway quota verification: unable to find internet gateway quota (%s)\", _internetGatewayQuotaCode))\n\t\t\t}\n\t\t}\n\n\t\t// check nat GW quota\n\t\tif natGatewayRequired {\n\t\t\tif natGatewayQuota, found := quotaCodeToValueMap[_natGatewayQuotaCode]; found {\n\t\t\t\terr := awsClient.VerifyNATGatewayQuota(natGatewayQuota, availabilityZones, highlyAvailableNATGateway)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tskippedValidations = append(skippedValidations, fmt.Sprintf(\"skipping nat gateway quota verification: unable to find nat gateway quota (%s)\\n\", _natGatewayQuotaCode))\n\t\t\t}\n\t\t}\n\n\t\t// check EIP quota\n\t\tif natGatewayRequired {\n\t\t\tif eipQuota, found := quotaCodeToValueMap[_elasticIPsQuotaCode]; found {\n\t\t\t\terr := awsClient.VerifyEIPQuota(eipQuota, availabilityZones, highlyAvailableNATGateway)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tskippedValidations = append(skippedValidations, fmt.Sprintf(\"skipping elastic ip quota verification: unable to find elastic ip quota (%s)\\n\", _elasticIPsQuotaCode))\n\t\t\t}\n\t\t}\n\n\t\t// check required VPC quota\n\t\tif requiredVPCs > 0 {\n\t\t\tif vpcQuota, found := quotaCodeToValueMap[_vpcQuotaCode]; found {\n\t\t\t\terr := awsClient.VerifyVPCQuota(vpcQuota, requiredVPCs)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tskippedValidations = append(skippedValidations, fmt.Sprintf(\"skipping vpc quota verification: unable to find vpc quota (%s)\\n\", _vpcQuotaCode))\n\t\t\t}\n\t\t}\n\t}\n\n\tif securityGroupRulesQuota, found := quotaCodeToValueMap[_securityGroupRulesQuotaCode]; found {\n\t\terr := awsClient.VerifySecurityGroupRulesQuota(securityGroupRulesQuota, availabilityZones, numNodeGroups, longestCIDRWhiteList)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tskippedValidations = append(skippedValidations, fmt.Sprintf(\"skipping security group rules quota verification: unable to find security group rules quota (%s)\\n\", _securityGroupRulesQuotaCode))\n\t}\n\n\tif securityGroupsQuota, found := quotaCodeToValueMap[_securityGroupsQuotaCode]; found {\n\t\terr := awsClient.VerifySecurityGroupQuota(securityGroupsQuota, netAdditionOfNodeGroups, clusterAlreadyExists)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tskippedValidations = append(skippedValidations, fmt.Sprintf(\"skipping security group quota verification: unable to find security group quota (%s)\\n\", _securityGroupsQuotaCode))\n\t}\n\n\treturn nil\n}\n\nfunc VerifyNetworkQuotasOnConfigure(\n\tawsClient *aws.Client,\n\tavailabilityZones strset.Set,\n\tnumNodeGroups int,\n\tnetAdditionOfNodeGroups int,\n\tlongestCIDRWhiteList int) error {\n\n\treturn VerifyNetworkQuotas(awsClient, 0, false, false, 0, availabilityZones, numNodeGroups, netAdditionOfNodeGroups, longestCIDRWhiteList, true)\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/subnet_visibility.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\ntype SubnetVisibility int\n\nconst (\n\tUnknownSubnetVisibility SubnetVisibility = iota\n\tPublicSubnetVisibility\n\tPrivateSubnetVisibility\n)\n\nvar _subnetVisibilities = []string{\n\t\"unknown\",\n\t\"public\",\n\t\"private\",\n}\n\nfunc SubnetVisibilityFromString(s string) SubnetVisibility {\n\tfor i := 0; i < len(_subnetVisibilities); i++ {\n\t\tif s == _subnetVisibilities[i] {\n\t\t\treturn SubnetVisibility(i)\n\t\t}\n\t}\n\treturn UnknownSubnetVisibility\n}\n\nfunc SubnetVisibilityStrings() []string {\n\treturn _subnetVisibilities[1:]\n}\n\nfunc (t SubnetVisibility) String() string {\n\treturn _subnetVisibilities[t]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (t SubnetVisibility) MarshalText() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (t *SubnetVisibility) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_subnetVisibilities); i++ {\n\t\tif enum == _subnetVisibilities[i] {\n\t\t\t*t = SubnetVisibility(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*t = UnknownSubnetVisibility\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (t *SubnetVisibility) UnmarshalBinary(data []byte) error {\n\treturn t.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (t SubnetVisibility) MarshalBinary() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n"
  },
  {
    "path": "pkg/types/clusterconfig/volume_types.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterconfig\n\ntype VolumeType int\n\nconst (\n\tUnknownVolumeType VolumeType = iota\n\tGP2VolumeType\n\tGP3VolumeType\n\tIO1VolumeType\n\tSC1VolumeType\n\tST1VolumeType\n)\n\nvar _availableVolumeTypes = []string{\n\t\"unknown\",\n\t\"gp2\",\n\t\"gp3\",\n\t\"io1\",\n\t\"sc1\",\n\t\"st1\",\n}\n\n//VolumeTypeFromString turns string into StorageType\nfunc VolumeTypeFromString(s string) VolumeType {\n\tfor i := 0; i < len(_availableVolumeTypes); i++ {\n\t\tif s == _availableVolumeTypes[i] {\n\t\t\treturn VolumeType(i)\n\t\t}\n\t}\n\treturn UnknownVolumeType\n}\n\nfunc VolumeTypesStrings() []string {\n\treturn _availableVolumeTypes[1:]\n}\n\nfunc (t VolumeType) String() string {\n\treturn _availableVolumeTypes[t]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (t VolumeType) MarshalText() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (t *VolumeType) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_availableVolumeTypes); i++ {\n\t\tif enum == _availableVolumeTypes[i] {\n\t\t\t*t = VolumeType(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*t = UnknownVolumeType\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (t *VolumeType) UnmarshalBinary(data []byte) error {\n\treturn t.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (t VolumeType) MarshalBinary() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n"
  },
  {
    "path": "pkg/types/clusterstate/clusterstate.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterstate\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/service/cloudformation\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/table\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n)\n\nconst (\n\tcontrolPlaneTemplate = \"eksctl-%s-cluster\"\n\toperatorTemplate     = \"eksctl-%s-nodegroup-cx-operator\"\n\n\tspotTemplatePrefix     = \"eksctl-%s-nodegroup-cx-ws\"\n\tonDemandTemplatePrefix = \"eksctl-%s-nodegroup-cx-wd\"\n)\n\ntype ClusterStacks struct {\n\tclusterName       string\n\tregion            string\n\tControlPlaneStack *cloudformation.StackSummary\n\tOperatorStack     *cloudformation.StackSummary\n\tNodeGroupsStacks  []*cloudformation.StackSummary\n}\n\nfunc (cs ClusterStacks) TableString() string {\n\tnumStacks := len(cs.NodeGroupsStacks)\n\tif cs.ControlPlaneStack != nil {\n\t\tnumStacks++\n\t}\n\tif cs.OperatorStack != nil {\n\t\tnumStacks++\n\t}\n\n\tidx := 0\n\trows := make([][]interface{}, numStacks)\n\n\tif cs.ControlPlaneStack != nil {\n\t\trows[idx] = []interface{}{\n\t\t\t*cs.ControlPlaneStack.StackName, *cs.ControlPlaneStack.StackStatus,\n\t\t}\n\t\tidx++\n\t}\n\tif cs.OperatorStack != nil {\n\t\trows[idx] = []interface{}{\n\t\t\t*cs.OperatorStack.StackName, *cs.OperatorStack.StackStatus,\n\t\t}\n\t\tidx++\n\t}\n\n\tfor _, stack := range cs.NodeGroupsStacks {\n\t\trows[idx] = []interface{}{*stack.StackName, *stack.StackStatus}\n\t\tidx++\n\t}\n\n\tt := table.Table{\n\t\tHeaders: []table.Header{\n\t\t\t{\n\t\t\t\tTitle: \"cloudformation stack name\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tTitle: \"status\",\n\t\t\t},\n\t\t},\n\t\tRows: rows,\n\t}\n\treturn t.MustFormat()\n}\n\nfunc GetClusterStacks(awsClient *aws.Client, accessConfig *clusterconfig.AccessConfig) (ClusterStacks, error) {\n\tcontrolPlaneStackName := fmt.Sprintf(controlPlaneTemplate, accessConfig.ClusterName)\n\toperatorStackName := fmt.Sprintf(operatorTemplate, accessConfig.ClusterName)\n\tspotStackNamePrefix := fmt.Sprintf(spotTemplatePrefix, accessConfig.ClusterName)\n\tonDemandStackNamePrefix := fmt.Sprintf(onDemandTemplatePrefix, accessConfig.ClusterName)\n\tnodeGroupStackPrefixesSet := strset.New(operatorStackName, spotStackNamePrefix, onDemandStackNamePrefix)\n\n\tstackSummaries, err := awsClient.ListEKSStacks(controlPlaneStackName, nodeGroupStackPrefixesSet)\n\tif err != nil {\n\t\treturn ClusterStacks{}, errors.Wrap(err, \"unable to get cluster state from cloudformation\")\n\t}\n\n\tvar controlPlaneStack, operatorStack *cloudformation.StackSummary\n\tvar ngStacks []*cloudformation.StackSummary\n\tfor _, stack := range stackSummaries {\n\t\tif stack == nil || stack.StackName == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif strings.HasPrefix(*stack.StackName, spotStackNamePrefix) || strings.HasPrefix(*stack.StackName, onDemandStackNamePrefix) {\n\t\t\tngStacks = append(ngStacks, stack)\n\t\t} else if *stack.StackName == controlPlaneStackName {\n\t\t\tcontrolPlaneStack = stack\n\t\t} else if *stack.StackName == operatorStackName {\n\t\t\toperatorStack = stack\n\t\t}\n\t}\n\n\treturn ClusterStacks{\n\t\tclusterName:       accessConfig.ClusterName,\n\t\tregion:            accessConfig.Region,\n\t\tControlPlaneStack: controlPlaneStack,\n\t\tOperatorStack:     operatorStack,\n\t\tNodeGroupsStacks:  ngStacks,\n\t}, nil\n}\n\nfunc GetClusterState(stacks ClusterStacks) State {\n\tcontrolPlaneStackName := fmt.Sprintf(controlPlaneTemplate, stacks.clusterName)\n\n\tif stacks.ControlPlaneStack == nil || stacks.ControlPlaneStack.StackName == nil {\n\t\treturn StateClusterDoesntExist\n\t}\n\tif *stacks.ControlPlaneStack.StackName == controlPlaneStackName {\n\t\tcontrolPlaneStatus := *stacks.ControlPlaneStack.StackStatus\n\n\t\tif slices.HasString([]string{\n\t\t\tcloudformation.StackStatusDeleteComplete,\n\t\t\tcloudformation.StackStatusDeleteInProgress,\n\t\t}, controlPlaneStatus) {\n\t\t\treturn StateClusterDoesntExist\n\t\t}\n\n\t\tif slices.HasString([]string{\n\t\t\tcloudformation.StackStatusCreateInProgress,\n\t\t\tcloudformation.StackStatusCreateComplete,\n\t\t\tcloudformation.StackStatusUpdateInProgress,\n\t\t\tcloudformation.StackStatusUpdateComplete,\n\t\t\tcloudformation.StackStatusRollbackInProgress,\n\t\t\tcloudformation.StackStatusRollbackComplete,\n\t\t\tcloudformation.StackStatusUpdateRollbackInProgress,\n\t\t\tcloudformation.StackStatusUpdateRollbackComplete,\n\t\t}, controlPlaneStatus) {\n\t\t\treturn StateClusterExists\n\t\t}\n\n\t\treturn StateClusterInUnexpectedState\n\t}\n\treturn StateClusterDoesntExist\n}\n\nfunc CloudFormationURL(clusterName string, region string) string {\n\treturn fmt.Sprintf(\"https://console.aws.amazon.com/cloudformation/home?region=%s#/stacks?filteringText=eksctl-%s-\", region, clusterName)\n}\n"
  },
  {
    "path": "pkg/types/clusterstate/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterstate\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n)\n\nconst (\n\tErrClusterDoesNotExist    = \"clusterstate.cluster_does_not_exist\"\n\tErrClusterAlreadyExists   = \"clusterstate.cluster_already_exists\"\n\tErrUnexpectedClusterState = \"clusterstate.unexpected_cluster_state\"\n)\n\nfunc ErrorClusterDoesNotExist(stacks ClusterStacks) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterDoesNotExist,\n\t\tMessage: fmt.Sprintf(\"there is no cluster named \\\"%s\\\" in %s\", stacks.clusterName, stacks.region),\n\t})\n}\n\nfunc ErrorClusterAlreadyExists(stacks ClusterStacks) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrClusterAlreadyExists,\n\t\tMessage: fmt.Sprintf(\"a cluster named \\\"%s\\\" already exists in %s\", stacks.clusterName, stacks.region),\n\t})\n}\n\nfunc ErrorUnexpectedClusterState(stacks ClusterStacks) error {\n\tmsg := fmt.Sprintf(\"cluster named \\\"%s\\\" in %s is in an unexpected state; if your CloudFormation stacks are updating, please wait for them to complete. Otherwise, run `cortex cluster down` to delete the cluster\\n\\n\", stacks.clusterName, stacks.region)\n\tmsg += fmt.Sprintf(stacks.TableString())\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:     ErrUnexpectedClusterState,\n\t\tMessage:  msg,\n\t\tMetadata: stacks,\n\t})\n}\n\nfunc AssertClusterState(stacks ClusterStacks, currentState, allowedState State) error {\n\tif currentState == allowedState {\n\t\treturn nil\n\t}\n\tswitch currentState {\n\tcase StateClusterDoesntExist:\n\t\treturn ErrorClusterDoesNotExist(stacks)\n\tcase StateClusterExists:\n\t\treturn ErrorClusterAlreadyExists(stacks)\n\tcase StateClusterInUnexpectedState:\n\t\treturn ErrorUnexpectedClusterState(stacks)\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/types/clusterstate/state.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage clusterstate\n\ntype State string\n\nconst (\n\tStateClusterExists            State = \"cluster_exists\"\n\tStateClusterDoesntExist       State = \"cluster_doesnt_exist\"\n\tStateClusterInUnexpectedState State = \"cluster_in_unexpected_state\"\n)\n"
  },
  {
    "path": "pkg/types/metrics/batch_metrics.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage metrics\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n)\n\ntype BatchMetrics struct {\n\tSucceeded           int      `json:\"succeeded\"`\n\tFailed              int      `json:\"failed\"`\n\tAverageTimePerBatch *float64 `json:\"average_time_per_batch\"`\n}\n\nfunc (batchMetrics BatchMetrics) Merge(right BatchMetrics) BatchMetrics {\n\tnewBatchMetrics := BatchMetrics{}\n\tnewBatchMetrics.MergeInPlace(batchMetrics)\n\tnewBatchMetrics.MergeInPlace(right)\n\treturn newBatchMetrics\n}\n\nfunc (batchMetrics *BatchMetrics) MergeInPlace(right BatchMetrics) {\n\tbatchMetrics.AverageTimePerBatch = mergeAvg(batchMetrics.AverageTimePerBatch, batchMetrics.Succeeded, right.AverageTimePerBatch, right.Succeeded)\n\tbatchMetrics.Succeeded = batchMetrics.Succeeded + right.Succeeded\n\tbatchMetrics.Failed = batchMetrics.Failed + right.Failed\n}\n\nfunc mergeAvg(left *float64, leftCount int, right *float64, rightCount int) *float64 {\n\tleftCountFloat64Ptr := pointer.Float64(float64(leftCount))\n\trightCountFloat64Ptr := pointer.Float64(float64(rightCount))\n\n\tavg, _ := slices.Float64PtrAvg([]*float64{left, right}, []*float64{leftCountFloat64Ptr, rightCountFloat64Ptr})\n\treturn avg\n}\n"
  },
  {
    "path": "pkg/types/metrics/metrics_test.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage metrics\n\nimport (\n\t\"testing\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestMergeAvg(t *testing.T) {\n\tfloatNilPtr := (*float64)(nil)\n\n\trequire.Equal(t, floatNilPtr, mergeAvg(nil, 0, nil, 0))\n\trequire.Equal(t, floatNilPtr, mergeAvg(nil, 1, nil, 0))\n\n\trequire.Equal(t, float64(1), *mergeAvg(pointer.Float64(1), 1, nil, 1))\n\trequire.Equal(t, pointer.Float64(1), mergeAvg(nil, 1, pointer.Float64(1), 1))\n\n\trequire.Equal(t, floatNilPtr, mergeAvg(pointer.Float64(1), 0, nil, 1))\n\trequire.Equal(t, floatNilPtr, mergeAvg(nil, 1, pointer.Float64(1), 0))\n\n\trequire.Equal(t, float64(1.25), *mergeAvg(pointer.Float64(1.25), 5, nil, 0))\n\trequire.Equal(t, float64(1.25), *mergeAvg(nil, 0, pointer.Float64(1.25), 5))\n\n\trequire.Equal(t, float64(1.25), *mergeAvg(pointer.Float64(1), 3, pointer.Float64(2), 1))\n\trequire.Equal(t, float64(1.25), *mergeAvg(pointer.Float64(2), 1, pointer.Float64(1), 3))\n}\n"
  },
  {
    "path": "pkg/types/metrics/queue_metrics.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage metrics\n\n// There could be Cortex specific messages in queue\ntype QueueMetrics struct {\n\tVisible    int `json:\"visible\"`\n\tNotVisible int `json:\"not_visible\"`\n}\n\nfunc (q QueueMetrics) IsEmpty() bool {\n\treturn q.TotalInQueue() == 0\n}\n\nfunc (q QueueMetrics) TotalInQueue() int {\n\treturn q.Visible + q.NotVisible\n}\n\nfunc (q QueueMetrics) TotalUserMessages() int {\n\ttotal := q.TotalInQueue()\n\tif total == 0 {\n\t\treturn 0\n\t}\n\n\treturn total - 1 // An extra message is added\n}\n"
  },
  {
    "path": "pkg/types/spec/api.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage spec\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"math\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/hash\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\tistioclientnetworking \"istio.io/client-go/pkg/apis/networking/v1beta1\"\n\tkapps \"k8s.io/api/apps/v1\"\n)\n\ntype API struct {\n\t*userconfig.API\n\tID           string `json:\"id\" yaml:\"id\"`\n\tSpecID       string `json:\"spec_id\" yaml:\"spec_id\"`\n\tPodID        string `json:\"pod_id\" yaml:\"pod_id\"`\n\tDeploymentID string `json:\"deployment_id\" yaml:\"deployment_id\"`\n\n\tKey string `json:\"key\" yaml:\"key\"`\n\n\tInitialDeploymentTime int64  `json:\"initial_deployment_time\" yaml:\"initial_deployment_time\"`\n\tLastUpdated           int64  `json:\"last_updated\" yaml:\"last_updated\"`\n\tMetadataRoot          string `json:\"metadata_root\" yaml:\"metadata_root\"`\n}\n\ntype Metadata struct {\n\t*userconfig.Resource\n\tAPIID        string `json:\"id\" yaml:\"id\"`\n\tDeploymentID string `json:\"deployment_id,omitempty\" yaml:\"deployment_id,omitempty\"`\n\tLastUpdated  int64  `json:\"last_updated\" yaml:\"last_updated\"`\n}\n\nfunc MetadataFromDeployment(deployment *kapps.Deployment) (*Metadata, error) {\n\tlastUpdated, err := TimeFromAPIID(deployment.Labels[\"apiID\"])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Metadata{\n\t\tResource: &userconfig.Resource{\n\t\t\tName: deployment.Labels[\"apiName\"],\n\t\t\tKind: userconfig.KindFromString(deployment.Labels[\"apiKind\"]),\n\t\t},\n\t\tAPIID:        deployment.Labels[\"apiID\"],\n\t\tDeploymentID: deployment.Labels[\"deploymentID\"],\n\t\tLastUpdated:  lastUpdated.Unix(),\n\t}, nil\n}\n\nfunc MetadataFromVirtualService(vs *istioclientnetworking.VirtualService) (*Metadata, error) {\n\tlastUpdated, err := TimeFromAPIID(vs.Labels[\"apiID\"])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Metadata{\n\t\tResource: &userconfig.Resource{\n\t\t\tName: vs.Labels[\"apiName\"],\n\t\t\tKind: userconfig.KindFromString(vs.Labels[\"apiKind\"]),\n\t\t},\n\t\tAPIID:        vs.Labels[\"apiID\"],\n\t\tDeploymentID: vs.Labels[\"deploymentID\"],\n\t\tLastUpdated:  lastUpdated.Unix(),\n\t}, nil\n}\n\n/*\n* ID (uniquely identifies an api configuration for a given deployment)\n\t* DeploymentID (used for refreshing a deployment)\n\t* SpecID (uniquely identifies api configuration specified by user)\n\t\t* PodID (an ID representing the pod spec)\n\t\t\t* Resource\n\t\t\t\t* Containers\n\t\t\t\t* Compute\n\t\t\t* Pod\n\t\t* Deployment Strategy\n\t\t* Autoscaling\n\t\t* Networking\n\t\t* APIs\n\ninitialDeploymentTime is Time.UnixNano()\n*/\nfunc GetAPISpec(apiConfig *userconfig.API, initialDeploymentTime int64, deploymentID string, clusterUID string) *API {\n\tvar buf bytes.Buffer\n\n\tbuf.WriteString(s.Obj(apiConfig.Resource))\n\tbuf.WriteString(s.Obj(apiConfig.Pod))\n\tpodID := hash.Bytes(buf.Bytes())\n\n\tbuf.Reset()\n\tbuf.WriteString(podID)\n\tbuf.WriteString(s.Obj(apiConfig.APIs))\n\tbuf.WriteString(s.Obj(apiConfig.Networking))\n\tbuf.WriteString(s.Obj(apiConfig.Autoscaling))\n\tbuf.WriteString(s.Obj(apiConfig.UpdateStrategy))\n\tbuf.WriteString(s.Obj(apiConfig.NodeGroups))\n\tspecID := hash.Bytes(buf.Bytes())[:32]\n\n\tapiID := fmt.Sprintf(\"%s-%s-%s\", MonotonicallyDecreasingID(), deploymentID, specID) // should be up to 60 characters long\n\n\treturn &API{\n\t\tAPI:                   apiConfig,\n\t\tID:                    apiID,\n\t\tSpecID:                specID,\n\t\tPodID:                 podID,\n\t\tKey:                   Key(apiConfig.Name, apiID, clusterUID),\n\t\tInitialDeploymentTime: initialDeploymentTime,\n\t\tDeploymentID:          deploymentID,\n\t\tLastUpdated:           time.Now().Unix(),\n\t\tMetadataRoot:          MetadataRoot(apiConfig.Name, clusterUID),\n\t}\n}\n\nfunc Key(apiName string, apiID string, clusterUID string) string {\n\treturn filepath.Join(\n\t\tclusterUID,\n\t\t\"apis\",\n\t\tapiName,\n\t\t\"api\",\n\t\tapiID,\n\t\tconsts.CortexVersion+\"-spec.json\",\n\t)\n}\n\n// The path to the directory which contains one subdirectory for each API ID (for its API spec)\nfunc KeysPrefix(apiName string, clusterUID string) string {\n\treturn filepath.Join(\n\t\tclusterUID,\n\t\t\"apis\",\n\t\tapiName,\n\t\t\"api\",\n\t) + \"/\"\n}\n\nfunc MetadataRoot(apiName string, clusterUID string) string {\n\treturn filepath.Join(\n\t\tclusterUID,\n\t\t\"apis\",\n\t\tapiName,\n\t\t\"metadata\",\n\t)\n}\n\n// Extract the timestamp from an API ID\nfunc TimeFromAPIID(apiID string) (time.Time, error) {\n\ttimeIDStr := strings.Split(apiID, \"-\")[0]\n\ttimeID, err := strconv.ParseInt(timeIDStr, 16, 64)\n\tif err != nil {\n\t\treturn time.Time{}, errors.Wrap(err, fmt.Sprintf(\"unable to parse API timestamp (%s)\", timeIDStr))\n\t}\n\ttimeNanos := math.MaxInt64 - timeID\n\treturn time.Unix(0, timeNanos), nil\n}\n"
  },
  {
    "path": "pkg/types/spec/errors.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage spec\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\nconst (\n\tErrMalformedConfig              = \"spec.malformed_config\"\n\tErrNoAPIs                       = \"spec.no_apis\"\n\tErrDuplicateName                = \"spec.duplicate_name\"\n\tErrDuplicateEndpointInOneDeploy = \"spec.duplicate_endpoint_in_one_deploy\"\n\tErrDuplicateEndpoint            = \"spec.duplicate_endpoint\"\n\tErrDuplicateContainerName       = \"spec.duplicate_container_name\"\n\tErrSpecifyExactlyOneField       = \"spec.specify_exactly_one_field\"\n\tErrSpecifyAllOrNone             = \"spec.specify_all_or_none\"\n\tErrOneOfPrerequisitesNotDefined = \"spec.one_of_prerequisites_not_defined\"\n\tErrConfigGreaterThanOtherConfig = \"spec.config_greater_than_other_config\"\n\n\tErrMinReplicasGreaterThanMax  = \"spec.min_replicas_greater_than_max\"\n\tErrInitReplicasGreaterThanMax = \"spec.init_replicas_greater_than_max\"\n\tErrInitReplicasLessThanMin    = \"spec.init_replicas_less_than_min\"\n\tErrTargetInFlightLimitReached = \"spec.target_in_flight_limit_reached\"\n\n\tErrInvalidSurgeOrUnavailable   = \"spec.invalid_surge_or_unavailable\"\n\tErrSurgeAndUnavailableBothZero = \"spec.surge_and_unavailable_both_zero\"\n\n\tErrShmCannotExceedMem = \"spec.shm_cannot_exceed_mem\"\n\n\tErrFieldMustBeSpecifiedForKind    = \"spec.field_must_be_specified_for_kind\"\n\tErrFieldIsNotSupportedForKind     = \"spec.field_is_not_supported_for_kind\"\n\tErrCortexPrefixedEnvVarNotAllowed = \"spec.cortex_prefixed_env_var_not_allowed\"\n\tErrDisallowedEnvVars              = \"spec.disallowed_env_vars\"\n\tErrComputeResourceConflict        = \"spec.compute_resource_conflict\"\n\tErrIncorrectTrafficSplitterWeight = \"spec.incorrect_traffic_splitter_weight\"\n\tErrTrafficSplitterAPIsNotUnique   = \"spec.traffic_splitter_apis_not_unique\"\n\tErrOneShadowPerTrafficSplitter    = \"spec.one_shadow_per_traffic_splitter\"\n\tErrUnexpectedDockerSecretData     = \"spec.unexpected_docker_secret_data\"\n)\n\nfunc ErrorMalformedConfig() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMalformedConfig,\n\t\tMessage: fmt.Sprintf(\"cortex YAML configuration files must contain a list of maps (see https://docs.cortexlabs.com/v/%s/ for api configuration schema)\", consts.CortexVersionMinor),\n\t})\n}\n\nfunc ErrorNoAPIs() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrNoAPIs,\n\t\tMessage: fmt.Sprintf(\"at least one API must be configured (see https://docs.cortexlabs.com/v/%s/ for api configuration schema)\", consts.CortexVersionMinor),\n\t})\n}\n\nfunc ErrorDuplicateName(apis []userconfig.API) error {\n\tfilePaths := strset.New()\n\tfor _, api := range apis {\n\t\tfilePaths.Add(api.FileName)\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDuplicateName,\n\t\tMessage: fmt.Sprintf(\"name %s must be unique across apis (defined in %s)\", s.UserStr(apis[0].Name), s.StrsAnd(filePaths.Slice())),\n\t})\n}\n\nfunc ErrorDuplicateEndpointInOneDeploy(apis []userconfig.API) error {\n\tnames := make([]string, len(apis))\n\tfor i, api := range apis {\n\t\tnames[i] = api.Name\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDuplicateEndpointInOneDeploy,\n\t\tMessage: fmt.Sprintf(\"endpoint %s must be unique across apis (defined in %s)\", s.UserStr(*apis[0].Networking.Endpoint), s.StrsAnd(names)),\n\t})\n}\n\nfunc ErrorDuplicateEndpoint(apiName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDuplicateEndpoint,\n\t\tMessage: fmt.Sprintf(\"endpoint is already being used by %s\", apiName),\n\t})\n}\n\nfunc ErrorDuplicateContainerName(containerName string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDuplicateContainerName,\n\t\tMessage: fmt.Sprintf(\"container name %s must be unique\", containerName),\n\t})\n}\n\nfunc ErrorSpecifyExactlyOneField(numSpecified int, fields ...string) error {\n\tvar msg string\n\n\tif len(fields) == 2 {\n\t\tif numSpecified == 0 {\n\t\t\tmsg = fmt.Sprintf(\"please specify either %s\", s.UserStrsOr(fields))\n\t\t} else {\n\t\t\tmsg = fmt.Sprintf(\"please specify either %s (both cannot be specified at the same time)\", s.UserStrsOr(fields))\n\t\t}\n\t} else {\n\t\tif numSpecified == 0 {\n\t\t\tmsg = fmt.Sprintf(\"please specify one of the following fields: %s\", s.UserStrsOr(fields))\n\t\t} else {\n\t\t\tmsg = fmt.Sprintf(\"please specify only one of the following fields: %s\", s.UserStrsOr(fields))\n\t\t}\n\t}\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSpecifyExactlyOneField,\n\t\tMessage: msg,\n\t})\n}\n\nfunc ErrorSpecifyAllOrNone(val string, vals ...string) error {\n\tallVals := append([]string{val}, vals...)\n\tmessage := fmt.Sprintf(\"please specify all or none of %s\", s.UserStrsAnd(allVals))\n\tif len(allVals) == 2 {\n\t\tmessage = fmt.Sprintf(\"please specify both %s and %s or neither of them\", s.UserStr(allVals[0]), s.UserStr(allVals[1]))\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSpecifyAllOrNone,\n\t\tMessage: message,\n\t})\n}\n\nfunc ErrorOneOfPrerequisitesNotDefined(argName string, prerequisite string, prerequisites ...string) error {\n\tallPrerequisites := append([]string{prerequisite}, prerequisites...)\n\tmessage := fmt.Sprintf(\"%s specified without specifying %s\", s.UserStr(argName), s.UserStrsOr(allPrerequisites))\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrOneOfPrerequisitesNotDefined,\n\t\tMessage: message,\n\t})\n}\n\nfunc ErrorConfigGreaterThanOtherConfig(tooBigKey string, tooBigVal interface{}, tooSmallKey string, tooSmallVal interface{}) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrConfigGreaterThanOtherConfig,\n\t\tMessage: fmt.Sprintf(\"%s (%s) cannot be greater than %s (%s)\", tooBigKey, s.UserStr(tooBigVal), tooSmallKey, s.UserStr(tooSmallVal)),\n\t})\n}\n\nfunc ErrorMinReplicasGreaterThanMax(min int32, max int32) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrMinReplicasGreaterThanMax,\n\t\tMessage: fmt.Sprintf(\"%s cannot be greater than %s (%d > %d)\", userconfig.MinReplicasKey, userconfig.MaxReplicasKey, min, max),\n\t})\n}\n\nfunc ErrorInitReplicasGreaterThanMax(init int32, max int32) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInitReplicasGreaterThanMax,\n\t\tMessage: fmt.Sprintf(\"%s cannot be greater than %s (%d > %d)\", userconfig.InitReplicasKey, userconfig.MaxReplicasKey, init, max),\n\t})\n}\n\nfunc ErrorInitReplicasLessThanMin(init int32, min int32) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInitReplicasLessThanMin,\n\t\tMessage: fmt.Sprintf(\"%s cannot be less than %s (%d < %d)\", userconfig.InitReplicasKey, userconfig.MinReplicasKey, init, min),\n\t})\n}\n\nfunc ErrorTargetInFlightLimitReached(targetInFlight float64, maxConcurrency, maxQueueLength int64) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrTargetInFlightLimitReached,\n\t\tMessage: fmt.Sprintf(\"%s cannot be greater than %s + %s (%f > %d + %d)\", userconfig.TargetInFlightKey, userconfig.MaxConcurrencyKey, userconfig.MaxQueueLengthKey, targetInFlight, maxConcurrency, maxQueueLength),\n\t})\n}\n\nfunc ErrorInvalidSurgeOrUnavailable(val string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrInvalidSurgeOrUnavailable,\n\t\tMessage: fmt.Sprintf(\"%s is not a valid value - must be an integer percentage (e.g. 25%%, to denote a percentage of desired replicas) or a positive integer (e.g. 5, to denote a number of replicas)\", s.UserStr(val)),\n\t})\n}\n\nfunc ErrorSurgeAndUnavailableBothZero() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrSurgeAndUnavailableBothZero,\n\t\tMessage: fmt.Sprintf(\"%s and %s cannot both be zero\", userconfig.MaxSurgeKey, userconfig.MaxUnavailableKey),\n\t})\n}\n\nfunc ErrorShmCannotExceedMem(shm k8s.Quantity, mem k8s.Quantity) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrShmCannotExceedMem,\n\t\tMessage: fmt.Sprintf(\"%s (%s) cannot exceed total compute %s (%s)\", userconfig.ShmKey, shm.UserString, userconfig.MemKey, mem.UserString),\n\t})\n}\n\nfunc ErrorFieldMustBeSpecifiedForKind(field string, kind userconfig.Kind) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFieldMustBeSpecifiedForKind,\n\t\tMessage: fmt.Sprintf(\"%s must be specified for %s kind\", field, kind.String()),\n\t})\n}\n\nfunc ErrorFieldIsNotSupportedForKind(field string, kind userconfig.Kind) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrFieldIsNotSupportedForKind,\n\t\tMessage: fmt.Sprintf(\"%s is not supported for %s kind\", field, kind.String()),\n\t})\n}\n\nfunc ErrorCortexPrefixedEnvVarNotAllowed(prefixes ...string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrCortexPrefixedEnvVarNotAllowed,\n\t\tMessage: fmt.Sprintf(\"environment variables starting with %s are reserved\", s.StrsOr(prefixes)),\n\t})\n}\n\nfunc ErrorDisallowedEnvVars(disallowedValues ...string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrDisallowedEnvVars,\n\t\tMessage: fmt.Sprintf(\"environment %s %s %s disallowed\", s.PluralS(\"variables\", len(disallowedValues)), s.StrsAnd(disallowedValues), s.PluralIs(len(disallowedValues))),\n\t})\n}\n\nfunc ErrorComputeResourceConflict(resourceA, resourceB string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrComputeResourceConflict,\n\t\tMessage: fmt.Sprintf(\"%s and %s resources cannot be used together\", resourceA, resourceB),\n\t})\n}\n\nfunc ErrorIncorrectTrafficSplitterWeightTotal(totalWeight int32) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrIncorrectTrafficSplitterWeight,\n\t\tMessage: fmt.Sprintf(\"expected weights of all non-shadow apis to sum to 100 but found %d\", totalWeight),\n\t})\n}\n\nfunc ErrorTrafficSplitterAPIsNotUnique(names []string) error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrTrafficSplitterAPIsNotUnique,\n\t\tMessage: fmt.Sprintf(\"%s not unique: %s\", s.PluralS(\"api\", len(names)), s.StrsSentence(names, \"\")),\n\t})\n}\n\nfunc ErrorOneShadowPerTrafficSplitter() error {\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrOneShadowPerTrafficSplitter,\n\t\tMessage: \"multiple shadow apis detected; only one api is allowed to be marked as a shadow\",\n\t})\n}\n\nvar _pwRegex = regexp.MustCompile(`\"password\":\"[^\"]+\"`)\nvar _authRegex = regexp.MustCompile(`\"auth\":\"[^\"]+\"`)\n\nfunc ErrorUnexpectedDockerSecretData(reason string, secretData map[string][]byte) error {\n\tsecretDataStrMap := map[string]string{}\n\n\tfor key, value := range secretData {\n\t\tvalueStr := string(value)\n\t\tvalueStr = _pwRegex.ReplaceAllString(valueStr, `\"password\":\"<omitted>\"`)\n\t\tvalueStr = _authRegex.ReplaceAllString(valueStr, `\"auth\":\"<omitted>\"`)\n\t\tsecretDataStrMap[key] = valueStr\n\t}\n\n\treturn errors.WithStack(&errors.Error{\n\t\tKind:    ErrUnexpectedDockerSecretData,\n\t\tMessage: fmt.Sprintf(\"docker registry secret named \\\"%s\\\" was found, but contains unexpected data (%s); got: %s\", _dockerPullSecretName, reason, s.UserStr(secretDataStrMap)),\n\t})\n}\n"
  },
  {
    "path": "pkg/types/spec/id_gen.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage spec\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n)\n\nvar idGenMutex = sync.Mutex{}\n\n// ID creation optimized for listing the most recently created jobs in S3. S3 objects are listed in ascending UTF-8 binary order. This should work until the year 2262.\nfunc MonotonicallyDecreasingID() string {\n\tidGenMutex.Lock()\n\tdefer idGenMutex.Unlock()\n\n\ti := math.MaxInt64 - time.Now().UnixNano()\n\treturn fmt.Sprintf(\"%x\", i)\n}\n"
  },
  {
    "path": "pkg/types/spec/job.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage spec\n\nimport (\n\t\"fmt\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\nconst (\n\tMetricsFileKey = \"metrics.json\"\n)\n\ntype JobKey struct {\n\tID      string          `json:\"job_id\" yaml:\"job_id\"`\n\tAPIName string          `json:\"api_name\" yaml:\"api_name\"`\n\tKind    userconfig.Kind `json:\"kind\" yaml:\"kind\"`\n}\n\nfunc (j JobKey) UserString() string {\n\treturn fmt.Sprintf(\"%s (%s api)\", j.ID, j.APIName)\n}\n\n// e.g. /<cluster UID>/jobs/<job_api_kind>/<cortex version>/<api_name>/<job_id>/spec.json\nfunc (j JobKey) SpecFilePath(clusterUID string) string {\n\treturn path.Join(j.Prefix(clusterUID), \"spec.json\")\n}\n\n// e.g. /<cluster UID>/jobs/<job_api_kind>/<cortex version>/<api_name>/<job_id>\nfunc (j JobKey) Prefix(clusterUID string) string {\n\treturn s.EnsureSuffix(path.Join(JobAPIPrefix(clusterUID, j.Kind, j.APIName), j.ID), \"/\")\n}\n\nfunc (j JobKey) K8sName() string {\n\treturn fmt.Sprintf(\"%s-%s\", j.APIName, j.ID)\n}\n\ntype SQSDeadLetterQueue struct {\n\tARN             string `json:\"arn\" yaml:\"arn\"`\n\tMaxReceiveCount int    `json:\"max_receive_count\" yaml:\"max_receive_count\"`\n}\n\ntype RuntimeBatchJobConfig struct {\n\tWorkers            int                    `json:\"workers\" yaml:\"workers\"`\n\tSQSDeadLetterQueue *SQSDeadLetterQueue    `json:\"sqs_dead_letter_queue\" yaml:\"sqs_dead_letter_queue\"`\n\tConfig             map[string]interface{} `json:\"config\" yaml:\"config\"`\n\tTimeout            *int                   `json:\"timeout\" yaml:\"timeout\"`\n}\n\ntype RuntimeTaskJobConfig struct {\n\tWorkers int                    `json:\"workers\" yaml:\"workers\"`\n\tConfig  map[string]interface{} `json:\"config\" yaml:\"config\"`\n\tTimeout *int                   `json:\"timeout\" yaml:\"timeout\"`\n}\n\ntype BatchJob struct {\n\tJobKey\n\tRuntimeBatchJobConfig\n\tAPIID           string    `json:\"api_id\" yaml:\"api_id\"`\n\tSQSUrl          string    `json:\"sqs_url\" yaml:\"sqs_url\"`\n\tTotalBatchCount int       `json:\"total_batch_count,omitempty\" yaml:\"total_batch_count,omitempty\"`\n\tStartTime       time.Time `json:\"start_time,omitempty\" yaml:\"start_time,omitempty\"`\n}\n\ntype TaskJob struct {\n\tJobKey\n\tRuntimeTaskJobConfig\n\tAPIID     string    `json:\"api_id\" yaml:\"api_id\"`\n\tSpecID    string    `json:\"spec_id\" yaml:\"spec_id\"`\n\tPodID     string    `json:\"pod_id\" yaml:\"pod_id\"`\n\tStartTime time.Time `json:\"start_time\" yaml:\"start_time\"`\n}\n\n// e.g. /<cluster UID>/jobs/<job_api_kind>/<cortex version>/<api_name>\nfunc JobAPIPrefix(clusterUID string, kind userconfig.Kind, apiName string) string {\n\treturn filepath.Join(clusterUID, \"jobs\", kind.String(), consts.CortexVersion, apiName)\n}\n\nfunc JobPayloadKey(clusterUID string, kind userconfig.Kind, apiName string, jobID string) string {\n\treturn filepath.Join(JobAPIPrefix(clusterUID, kind, apiName), jobID, \"payload.json\")\n}\n\nfunc JobBatchCountKey(clusterUID string, kind userconfig.Kind, apiName string, jobID string) string {\n\treturn filepath.Join(JobAPIPrefix(clusterUID, kind, apiName), jobID, \"max_batch_count\")\n}\n\nfunc JobMetricsKey(clusterUID string, kind userconfig.Kind, apiName string, jobID string) string {\n\treturn filepath.Join(JobAPIPrefix(clusterUID, kind, apiName), jobID, MetricsFileKey)\n}\n"
  },
  {
    "path": "pkg/types/spec/utils.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage spec\n\nimport (\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n)\n\nfunc FindDuplicateNames(apis []userconfig.API) []userconfig.API {\n\tnames := make(map[string][]userconfig.API)\n\n\tfor _, api := range apis {\n\t\tnames[api.Name] = append(names[api.Name], api)\n\t}\n\n\tfor name := range names {\n\t\tif len(names[name]) > 1 {\n\t\t\treturn names[name]\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc surgeOrUnavailableValidator(str string) (string, error) {\n\tif strings.HasSuffix(str, \"%\") {\n\t\tparsed, ok := s.ParseInt32(strings.TrimSuffix(str, \"%\"))\n\t\tif !ok {\n\t\t\treturn \"\", ErrorInvalidSurgeOrUnavailable(str)\n\t\t}\n\t\tif parsed < 0 || parsed > 100 {\n\t\t\treturn \"\", ErrorInvalidSurgeOrUnavailable(str)\n\t\t}\n\t} else {\n\t\tparsed, ok := s.ParseInt32(str)\n\t\tif !ok {\n\t\t\treturn \"\", ErrorInvalidSurgeOrUnavailable(str)\n\t\t}\n\t\tif parsed < 0 {\n\t\t\treturn \"\", ErrorInvalidSurgeOrUnavailable(str)\n\t\t}\n\t}\n\n\treturn str, nil\n}\n\nfunc verifyTotalWeight(apis []*userconfig.TrafficSplit) error {\n\ttotalWeight := int32(0)\n\tfor _, api := range apis {\n\t\tif !api.Shadow {\n\t\t\ttotalWeight += api.Weight\n\t\t}\n\t}\n\tif totalWeight == 100 {\n\t\treturn nil\n\t}\n\treturn errors.Wrap(ErrorIncorrectTrafficSplitterWeightTotal(totalWeight), userconfig.APIsKey)\n}\n\n// areTrafficSplitterAPIsUnique gives error if the same API is used multiple times in TrafficSplitter\nfunc areTrafficSplitterAPIsUnique(apis []*userconfig.TrafficSplit) error {\n\tnames := make(map[string][]userconfig.TrafficSplit)\n\tfor _, api := range apis {\n\t\tnames[api.Name] = append(names[api.Name], *api)\n\t}\n\tvar notUniqueAPIs []string\n\tfor name := range names {\n\t\tif len(names[name]) > 1 {\n\t\t\tnotUniqueAPIs = append(notUniqueAPIs, names[name][0].Name)\n\t\t}\n\t}\n\tif len(notUniqueAPIs) > 0 {\n\t\treturn errors.Wrap(ErrorTrafficSplitterAPIsNotUnique(notUniqueAPIs), userconfig.APIsKey)\n\t}\n\treturn nil\n}\n"
  },
  {
    "path": "pkg/types/spec/validations.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage spec\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/aws\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/cast\"\n\tcr \"github.com/cortexlabs/cortex/pkg/lib/configreader\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/docker\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/errors\"\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/regex\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/slices\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\tlibtime \"github.com/cortexlabs/cortex/pkg/lib/time\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\tdockertypes \"github.com/docker/docker/api/types\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n)\n\nvar AutoscalingTickInterval = 10 * time.Second\n\nconst _dockerPullSecretName = \"registry-credentials\"\n\nfunc apiValidation(resource userconfig.Resource) *cr.StructValidation {\n\tvar structFieldValidations []*cr.StructFieldValidation\n\n\tswitch resource.Kind {\n\tcase userconfig.RealtimeAPIKind:\n\t\tstructFieldValidations = append(resourceStructValidations,\n\t\t\tpodValidation(userconfig.RealtimeAPIKind),\n\t\t\tnodegroupsValidation(),\n\t\t\tnetworkingValidation(),\n\t\t\tautoscalingValidation(),\n\t\t\tupdateStrategyValidation(),\n\t\t)\n\tcase userconfig.AsyncAPIKind:\n\t\tstructFieldValidations = append(resourceStructValidations,\n\t\t\tpodValidation(userconfig.AsyncAPIKind),\n\t\t\tnodegroupsValidation(),\n\t\t\tnetworkingValidation(),\n\t\t\tautoscalingValidation(),\n\t\t\tupdateStrategyValidation(),\n\t\t)\n\tcase userconfig.BatchAPIKind:\n\t\tstructFieldValidations = append(resourceStructValidations,\n\t\t\tpodValidation(userconfig.BatchAPIKind),\n\t\t\tnodegroupsValidation(),\n\t\t\tnetworkingValidation(),\n\t\t)\n\tcase userconfig.TaskAPIKind:\n\t\tstructFieldValidations = append(resourceStructValidations,\n\t\t\tpodValidation(userconfig.TaskAPIKind),\n\t\t\tnodegroupsValidation(),\n\t\t\tnetworkingValidation(),\n\t\t)\n\tcase userconfig.TrafficSplitterKind:\n\t\tstructFieldValidations = append(resourceStructValidations,\n\t\t\tmultiAPIsValidation(),\n\t\t\tnetworkingValidation(),\n\t\t)\n\t}\n\treturn &cr.StructValidation{\n\t\tStructFieldValidations: structFieldValidations,\n\t}\n}\n\nvar resourceStructValidations = []*cr.StructFieldValidation{\n\t{\n\t\tStructField: \"Name\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tRequired:        true,\n\t\t\tDNS1035:         true,\n\t\t\tInvalidPrefixes: []string{\"b-\"}, // collides with our sqs names\n\t\t\tMaxLength:       42,             // k8s adds 21 characters to the pod name, and 63 is the max before it starts to truncate\n\t\t},\n\t},\n\t{\n\t\tStructField: \"Kind\",\n\t\tStringValidation: &cr.StringValidation{\n\t\t\tRequired:      true,\n\t\t\tAllowedValues: userconfig.KindStrings(),\n\t\t},\n\t\tParser: func(str string) (interface{}, error) {\n\t\t\treturn userconfig.KindFromString(str), nil\n\t\t},\n\t},\n}\n\nfunc multiAPIsValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"APIs\",\n\t\tStructListValidation: &cr.StructListValidation{\n\t\t\tRequired:         true,\n\t\t\tTreatNullAsEmpty: true,\n\t\t\tStructValidation: &cr.StructValidation{\n\t\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField: \"Name\",\n\t\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\t\tRequired:   true,\n\t\t\t\t\t\t\tAllowEmpty: false,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField: \"Weight\",\n\t\t\t\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\t\t\t\tRequired:             true,\n\t\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Int32(0),\n\t\t\t\t\t\t\tLessThanOrEqualTo:    pointer.Int32(100),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tStructField:    \"Shadow\",\n\t\t\t\t\t\tBoolValidation: &cr.BoolValidation{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc podValidation(kind userconfig.Kind) *cr.StructFieldValidation {\n\tvalidation := &cr.StructFieldValidation{\n\t\tStructField: \"Pod\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Port\",\n\t\t\t\t\tInt32PtrValidation: &cr.Int32PtrValidation{\n\t\t\t\t\t\tRequired:          false,\n\t\t\t\t\t\tDefault:           nil, // it's a pointer because it's not required for the task API\n\t\t\t\t\t\tAllowExplicitNull: true,\n\t\t\t\t\t\tGreaterThan:       pointer.Int32(0),\n\t\t\t\t\t\tLessThanOrEqualTo: pointer.Int32(65535),\n\t\t\t\t\t\tDisallowedValues:  consts.ReservedContainerPorts,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tcontainersValidation(kind),\n\t\t\t},\n\t\t},\n\t}\n\n\tif kind == userconfig.RealtimeAPIKind {\n\t\tvalidation.StructValidation.StructFieldValidations = append(validation.StructValidation.StructFieldValidations,\n\t\t\t&cr.StructFieldValidation{\n\t\t\t\tStructField: \"MaxQueueLength\",\n\t\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\t\tDefault:     consts.DefaultMaxQueueLength,\n\t\t\t\t\tGreaterThan: pointer.Int64(0),\n\t\t\t\t\t// the proxy can theoretically accept up to 32768 connections, but during testing,\n\t\t\t\t\t// it has been observed that the number is just slightly lower, so it has been offset by 2678\n\t\t\t\t\tLessThanOrEqualTo: pointer.Int64(30000),\n\t\t\t\t},\n\t\t\t},\n\t\t\t&cr.StructFieldValidation{\n\t\t\t\tStructField: \"MaxConcurrency\",\n\t\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\t\tDefault:     consts.DefaultMaxConcurrency,\n\t\t\t\t\tGreaterThan: pointer.Int64(0),\n\t\t\t\t\t// the proxy can theoretically accept up to 32768 connections, but during testing,\n\t\t\t\t\t// it has been observed that the number is just slightly lower, so it has been offset by 2678\n\t\t\t\t\tLessThanOrEqualTo: pointer.Int64(30000),\n\t\t\t\t},\n\t\t\t},\n\t\t)\n\t}\n\n\tif kind == userconfig.AsyncAPIKind {\n\t\tvalidation.StructValidation.StructFieldValidations = append(validation.StructValidation.StructFieldValidations,\n\t\t\t&cr.StructFieldValidation{\n\t\t\t\tStructField: \"MaxConcurrency\",\n\t\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\t\tDefault:           consts.DefaultMaxConcurrency,\n\t\t\t\t\tGreaterThan:       pointer.Int64(0),\n\t\t\t\t\tLessThanOrEqualTo: pointer.Int64(100),\n\t\t\t\t},\n\t\t\t},\n\t\t)\n\t}\n\n\treturn validation\n}\n\nfunc containersValidation(kind userconfig.Kind) *cr.StructFieldValidation {\n\tvalidations := []*cr.StructFieldValidation{\n\t\t{\n\t\t\tStructField: \"Name\",\n\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\tRequired:         true,\n\t\t\t\tAllowEmpty:       false,\n\t\t\t\tDNS1035:          true,\n\t\t\t\tMaxLength:        63,\n\t\t\t\tDisallowedValues: consts.ReservedContainerNames,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Image\",\n\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\tRequired:    true,\n\t\t\t\tAllowEmpty:  false,\n\t\t\t\tDockerImage: true,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Env\",\n\t\t\tStringMapValidation: &cr.StringMapValidation{\n\t\t\t\tRequired:   false,\n\t\t\t\tDefault:    map[string]string{},\n\t\t\t\tAllowEmpty: true,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Command\",\n\t\t\tStringListValidation: &cr.StringListValidation{\n\t\t\t\tRequired:          false,\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t\tAllowEmpty:        true,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"Args\",\n\t\t\tStringListValidation: &cr.StringListValidation{\n\t\t\t\tRequired:          false,\n\t\t\t\tAllowExplicitNull: true,\n\t\t\t\tAllowEmpty:        true,\n\t\t\t},\n\t\t},\n\t\tcomputeValidation(),\n\t\tprobeValidation(\"LivenessProbe\", true),\n\t}\n\n\tif kind == userconfig.RealtimeAPIKind {\n\t\tvalidations = append(validations, probeValidation(\"ReadinessProbe\", true))\n\t} else if kind == userconfig.AsyncAPIKind || kind == userconfig.BatchAPIKind {\n\t\tvalidations = append(validations, probeValidation(\"ReadinessProbe\", false))\n\t}\n\n\tif kind == userconfig.RealtimeAPIKind || kind == userconfig.AsyncAPIKind {\n\t\tvalidations = append(validations, preStopValidation())\n\t}\n\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"Containers\",\n\t\tStructListValidation: &cr.StructListValidation{\n\t\t\tRequired:         true,\n\t\t\tTreatNullAsEmpty: true,\n\t\t\tMinLength:        1,\n\t\t\tStructValidation: &cr.StructValidation{\n\t\t\t\tStructFieldValidations: validations,\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc nodegroupsValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"NodeGroups\",\n\t\tStringListValidation: &cr.StringListValidation{\n\t\t\tRequired:          false,\n\t\t\tDefault:           nil,\n\t\t\tAllowExplicitNull: true,\n\t\t\tAllowEmpty:        false,\n\t\t\tElementStringValidation: &cr.StringValidation{\n\t\t\t\tAlphaNumericDashUnderscore: true,\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc networkingValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"Networking\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Endpoint\",\n\t\t\t\t\tStringPtrValidation: &cr.StringPtrValidation{\n\t\t\t\t\t\tValidator: urls.ValidateEndpoint,\n\t\t\t\t\t\tMaxLength: 1000, // no particular reason other than it works\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc probeValidation(structFieldName string, hasExecProbe bool) *cr.StructFieldValidation {\n\tvalidations := []*cr.StructFieldValidation{\n\t\thttpGetHandlerValidation(),\n\t\ttcpSocketHandlerValidation(),\n\t\t{\n\t\t\tStructField: \"InitialDelaySeconds\",\n\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\tDefault:              0,\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int32(0),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"TimeoutSeconds\",\n\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\tDefault:              1,\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int32(0),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"PeriodSeconds\",\n\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\tDefault:              10,\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int32(0),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"SuccessThreshold\",\n\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\tDefault:              1,\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int32(0),\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tStructField: \"FailureThreshold\",\n\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\tDefault:              3,\n\t\t\t\tGreaterThanOrEqualTo: pointer.Int32(0),\n\t\t\t},\n\t\t},\n\t}\n\n\tif hasExecProbe {\n\t\tvalidations = append(validations, execHandlerValidation())\n\t}\n\n\treturn &cr.StructFieldValidation{\n\t\tStructField: structFieldName,\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tRequired:               false,\n\t\t\tAllowExplicitNull:      true,\n\t\t\tDefaultNil:             true,\n\t\t\tStructFieldValidations: validations,\n\t\t},\n\t}\n}\n\nfunc preStopValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"PreStop\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tRequired:          false,\n\t\t\tAllowExplicitNull: true,\n\t\t\tDefaultNil:        true,\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\thttpGetHandlerValidation(),\n\t\t\t\texecHandlerValidation(),\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc httpGetHandlerValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"HTTPGet\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tRequired:          false,\n\t\t\tAllowExplicitNull: true,\n\t\t\tDefaultNil:        true,\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Path\",\n\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\tRequired:  false,\n\t\t\t\t\t\tDefault:   \"/\",\n\t\t\t\t\t\tValidator: urls.ValidateEndpointAllowEmptyPath,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Port\",\n\t\t\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\t\t\tRequired:          true,\n\t\t\t\t\t\tGreaterThan:       pointer.Int32(0),\n\t\t\t\t\t\tLessThanOrEqualTo: pointer.Int32(65535),\n\t\t\t\t\t\tDisallowedValues:  consts.ReservedContainerPorts,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc tcpSocketHandlerValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"TCPSocket\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tRequired:          false,\n\t\t\tAllowExplicitNull: true,\n\t\t\tDefaultNil:        true,\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Port\",\n\t\t\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\t\t\tRequired:          true,\n\t\t\t\t\t\tGreaterThan:       pointer.Int32(0),\n\t\t\t\t\t\tLessThanOrEqualTo: pointer.Int32(65535),\n\t\t\t\t\t\tDisallowedValues:  consts.ReservedContainerPorts,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc execHandlerValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"Exec\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tRequired:          false,\n\t\t\tAllowExplicitNull: true,\n\t\t\tDefaultNil:        true,\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Command\",\n\t\t\t\t\tStringListValidation: &cr.StringListValidation{\n\t\t\t\t\t\tRequired: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc computeValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"Compute\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"CPU\",\n\t\t\t\t\tStringPtrValidation: &cr.StringPtrValidation{\n\t\t\t\t\t\tDefault:           pointer.String(\"200m\"),\n\t\t\t\t\t\tAllowExplicitNull: true,\n\t\t\t\t\t\tCastNumeric:       true,\n\t\t\t\t\t},\n\t\t\t\t\tParser: k8s.QuantityParser(&k8s.QuantityValidation{\n\t\t\t\t\t\tGreaterThanOrEqualTo: k8s.QuantityPtr(kresource.MustParse(\"20m\")),\n\t\t\t\t\t}),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Mem\",\n\t\t\t\t\tStringPtrValidation: &cr.StringPtrValidation{\n\t\t\t\t\t\tDefault:           nil,\n\t\t\t\t\t\tAllowExplicitNull: true,\n\t\t\t\t\t},\n\t\t\t\t\tParser: k8s.QuantityParser(&k8s.QuantityValidation{\n\t\t\t\t\t\tGreaterThanOrEqualTo: k8s.QuantityPtr(kresource.MustParse(\"20Mi\")),\n\t\t\t\t\t}),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"GPU\",\n\t\t\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\t\t\tDefault:              0,\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Inf\",\n\t\t\t\t\tInt64Validation: &cr.Int64Validation{\n\t\t\t\t\t\tDefault:              0,\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Int64(0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Shm\",\n\t\t\t\t\tStringPtrValidation: &cr.StringPtrValidation{\n\t\t\t\t\t\tDefault:           nil,\n\t\t\t\t\t\tAllowExplicitNull: true,\n\t\t\t\t\t},\n\t\t\t\t\tParser: k8s.QuantityParser(&k8s.QuantityValidation{}),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc autoscalingValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"Autoscaling\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"MinReplicas\",\n\t\t\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\t\t\tDefault:              1,\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Int32(0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"MaxReplicas\",\n\t\t\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\t\t\tDefault:     100,\n\t\t\t\t\t\tGreaterThan: pointer.Int32(0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField:  \"InitReplicas\",\n\t\t\t\t\tDefaultField: \"MinReplicas\",\n\t\t\t\t\tInt32Validation: &cr.Int32Validation{\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Int32(0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"TargetInFlight\",\n\t\t\t\t\tFloat64PtrValidation: &cr.Float64PtrValidation{\n\t\t\t\t\t\tDefault:     nil,\n\t\t\t\t\t\tGreaterThan: pointer.Float64(0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"Window\",\n\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\tDefault: \"60s\",\n\t\t\t\t\t},\n\t\t\t\t\tParser: cr.DurationParser(&cr.DurationValidation{\n\t\t\t\t\t\tGreaterThanOrEqualTo: &AutoscalingTickInterval,\n\t\t\t\t\t\tMultipleOf:           &AutoscalingTickInterval,\n\t\t\t\t\t}),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"DownscaleStabilizationPeriod\",\n\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\tDefault: \"5m\",\n\t\t\t\t\t},\n\t\t\t\t\tParser: cr.DurationParser(&cr.DurationValidation{\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Duration(libtime.MustParseDuration(\"0s\")),\n\t\t\t\t\t}),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"UpscaleStabilizationPeriod\",\n\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\tDefault: \"1m\",\n\t\t\t\t\t},\n\t\t\t\t\tParser: cr.DurationParser(&cr.DurationValidation{\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Duration(libtime.MustParseDuration(\"0s\")),\n\t\t\t\t\t}),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"MaxDownscaleFactor\",\n\t\t\t\t\tFloat64Validation: &cr.Float64Validation{\n\t\t\t\t\t\tDefault:              0.75,\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Float64(0),\n\t\t\t\t\t\tLessThan:             pointer.Float64(1),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"MaxUpscaleFactor\",\n\t\t\t\t\tFloat64Validation: &cr.Float64Validation{\n\t\t\t\t\t\tDefault:     1.5,\n\t\t\t\t\t\tGreaterThan: pointer.Float64(1),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"DownscaleTolerance\",\n\t\t\t\t\tFloat64Validation: &cr.Float64Validation{\n\t\t\t\t\t\tDefault:              0.05,\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Float64(0),\n\t\t\t\t\t\tLessThan:             pointer.Float64(1),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"UpscaleTolerance\",\n\t\t\t\t\tFloat64Validation: &cr.Float64Validation{\n\t\t\t\t\t\tDefault:              0.05,\n\t\t\t\t\t\tGreaterThanOrEqualTo: pointer.Float64(0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc updateStrategyValidation() *cr.StructFieldValidation {\n\treturn &cr.StructFieldValidation{\n\t\tStructField: \"UpdateStrategy\",\n\t\tStructValidation: &cr.StructValidation{\n\t\t\tStructFieldValidations: []*cr.StructFieldValidation{\n\t\t\t\t{\n\t\t\t\t\tStructField: \"MaxSurge\",\n\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\tDefault:   \"25%\",\n\t\t\t\t\t\tCastInt:   true,\n\t\t\t\t\t\tValidator: surgeOrUnavailableValidator,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tStructField: \"MaxUnavailable\",\n\t\t\t\t\tStringValidation: &cr.StringValidation{\n\t\t\t\t\t\tDefault:   \"25%\",\n\t\t\t\t\t\tCastInt:   true,\n\t\t\t\t\t\tValidator: surgeOrUnavailableValidator,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nvar resourceStructValidation = cr.StructValidation{\n\tAllowExtraFields:       true,\n\tStructFieldValidations: resourceStructValidations,\n}\n\nfunc ExtractAPIConfigs(configBytes []byte, configFileName string) ([]userconfig.API, error) {\n\tvar err error\n\n\tconfigData, err := cr.ReadYAMLBytes(configBytes)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, configFileName)\n\t}\n\n\tconfigDataSlice, ok := cast.InterfaceToStrInterfaceMapSlice(configData)\n\tif !ok {\n\t\treturn nil, errors.Wrap(ErrorMalformedConfig(), configFileName)\n\t}\n\n\tapis := make([]userconfig.API, len(configDataSlice))\n\tfor i, data := range configDataSlice {\n\t\tapi := userconfig.API{}\n\t\tvar resourceStruct userconfig.Resource\n\t\terrs := cr.Struct(&resourceStruct, data, &resourceStructValidation)\n\t\tif errors.HasError(errs) {\n\t\t\tname, _ := data[userconfig.NameKey].(string)\n\t\t\tkindString, _ := data[userconfig.KindKey].(string)\n\t\t\tkind := userconfig.KindFromString(kindString)\n\t\t\terr = errors.Wrap(errors.FirstError(errs...), userconfig.IdentifyAPI(configFileName, name, kind, i))\n\t\t\treturn nil, errors.Append(err, fmt.Sprintf(\"\\n\\napi configuration schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t\t}\n\n\t\terrs = cr.Struct(&api, data, apiValidation(resourceStruct))\n\t\tif errors.HasError(errs) {\n\t\t\tname, _ := data[userconfig.NameKey].(string)\n\t\t\tkindString, _ := data[userconfig.KindKey].(string)\n\t\t\tkind := userconfig.KindFromString(kindString)\n\t\t\terr = errors.Wrap(errors.FirstError(errs...), userconfig.IdentifyAPI(configFileName, name, kind, i))\n\t\t\treturn nil, errors.Append(err, fmt.Sprintf(\"\\n\\napi configuration schema can be found at https://docs.cortexlabs.com/v/%s/\", consts.CortexVersionMinor))\n\t\t}\n\t\tapi.Index = i\n\t\tapi.FileName = configFileName\n\n\t\tinterfaceMap, ok := cast.JSONMarshallable(data)\n\t\tif !ok {\n\t\t\treturn nil, errors.ErrorUnexpected(\"unable to cast api spec to json\") // unexpected\n\t\t}\n\t\tapi.SubmittedAPISpec = interfaceMap\n\t\tapis[i] = api\n\t}\n\n\treturn apis, nil\n}\n\nfunc ValidateAPI(\n\tapi *userconfig.API,\n\tawsClient *aws.Client,\n\tk8sClient *k8s.Client,\n) error {\n\n\tif api.Networking.Endpoint == nil {\n\t\tapi.Networking.Endpoint = pointer.String(\"/\" + api.Name)\n\t}\n\n\tif api.Pod != nil {\n\t\tif err := validatePod(api, awsClient, k8sClient); err != nil {\n\t\t\treturn errors.Wrap(err, userconfig.PodKey)\n\t\t}\n\t}\n\n\tif api.Autoscaling != nil {\n\t\tif err := validateAutoscaling(api); err != nil {\n\t\t\treturn errors.Wrap(err, userconfig.AutoscalingKey)\n\t\t}\n\t}\n\n\tif api.UpdateStrategy != nil {\n\t\tif err := validateUpdateStrategy(api.UpdateStrategy); err != nil {\n\t\t\treturn errors.Wrap(err, userconfig.UpdateStrategyKey)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc ValidateTrafficSplitter(api *userconfig.API) error {\n\tif api.Networking.Endpoint == nil {\n\t\tapi.Networking.Endpoint = pointer.String(\"/\" + api.Name)\n\t}\n\tif err := verifyTotalWeight(api.APIs); err != nil {\n\t\treturn err\n\t}\n\tif err := areTrafficSplitterAPIsUnique(api.APIs); err != nil {\n\t\treturn err\n\t}\n\n\thasShadow := false\n\tfor _, api := range api.APIs {\n\t\tif api.Shadow {\n\t\t\tif hasShadow {\n\t\t\t\treturn ErrorOneShadowPerTrafficSplitter()\n\t\t\t}\n\t\t\thasShadow = true\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc validatePod(\n\tapi *userconfig.API,\n\tawsClient *aws.Client,\n\tk8sClient *k8s.Client,\n) error {\n\n\tif api.Pod.Port != nil && api.Kind == userconfig.TaskAPIKind {\n\t\treturn ErrorFieldIsNotSupportedForKind(userconfig.PortKey, api.Kind)\n\t}\n\tif api.Pod.Port == nil && api.Kind != userconfig.TaskAPIKind {\n\t\tapi.Pod.Port = pointer.Int32(consts.DefaultUserPodPortInt32)\n\t}\n\n\tif err := validateCompute(api); err != nil {\n\t\treturn errors.Wrap(err, userconfig.ComputeKey)\n\t}\n\n\tif err := validateContainers(api.Pod.Containers, api.Kind, awsClient, k8sClient); err != nil {\n\t\treturn errors.Wrap(err, userconfig.ContainersKey)\n\t}\n\n\treturn nil\n}\n\nfunc validateContainers(\n\tcontainers []*userconfig.Container,\n\tkind userconfig.Kind,\n\tawsClient *aws.Client,\n\tk8sClient *k8s.Client,\n) error {\n\tcontainerNames := []string{}\n\n\tfor i, container := range containers {\n\t\tif slices.HasString(containerNames, container.Name) {\n\t\t\treturn errors.Wrap(ErrorDuplicateContainerName(container.Name), s.Index(i), userconfig.ImageKey)\n\t\t}\n\t\tcontainerNames = append(containerNames, container.Name)\n\n\t\tif container.Command == nil && (kind == userconfig.BatchAPIKind || kind == userconfig.TaskAPIKind) {\n\t\t\treturn errors.Wrap(ErrorFieldMustBeSpecifiedForKind(userconfig.CommandKey, kind), s.Index(i), userconfig.CommandKey)\n\t\t}\n\n\t\tif err := validateDockerImagePath(container.Image, awsClient, k8sClient); err != nil {\n\t\t\treturn errors.Wrap(err, s.Index(i), userconfig.ImageKey)\n\t\t}\n\n\t\tfor key := range container.Env {\n\t\t\tif strings.HasPrefix(key, \"CORTEX_\") || strings.HasPrefix(key, \"KUBEXIT_\") {\n\t\t\t\treturn errors.Wrap(ErrorCortexPrefixedEnvVarNotAllowed(\"CORTEX_\", \"KUBEXIT_\"), s.Index(i), userconfig.EnvKey, key)\n\t\t\t}\n\t\t}\n\n\t\tif kind == userconfig.TaskAPIKind && container.ReadinessProbe != nil {\n\t\t\treturn errors.Wrap(ErrorFieldIsNotSupportedForKind(userconfig.ReadinessProbeKey, kind), s.Index(i), userconfig.ReadinessProbeKey)\n\t\t}\n\n\t\tif container.ReadinessProbe != nil {\n\t\t\tsupportsExecProbe := kind == userconfig.RealtimeAPIKind\n\t\t\tif err := validateProbe(*container.ReadinessProbe, supportsExecProbe); err != nil {\n\t\t\t\treturn errors.Wrap(err, s.Index(i), userconfig.ReadinessProbeKey)\n\t\t\t}\n\t\t}\n\n\t\tif container.LivenessProbe != nil {\n\t\t\tif err := validateProbe(*container.LivenessProbe, true); err != nil {\n\t\t\t\treturn errors.Wrap(err, s.Index(i), userconfig.LivenessProbeKey)\n\t\t\t}\n\t\t}\n\n\t\tif container.PreStop != nil {\n\t\t\tif err := validatePreStop(*container.PreStop); err != nil {\n\t\t\t\treturn errors.Wrap(err, s.Index(i), userconfig.PreStopKey)\n\t\t\t}\n\t\t}\n\n\t\tcompute := container.Compute\n\t\tif compute.Shm != nil && compute.Mem != nil && compute.Shm.Cmp(compute.Mem.Quantity) > 0 {\n\t\t\treturn errors.Wrap(ErrorShmCannotExceedMem(*compute.Shm, *compute.Mem), s.Index(i), userconfig.ComputeKey)\n\t\t}\n\n\t}\n\n\treturn nil\n}\n\nfunc validateProbe(probe userconfig.Probe, supportsExecProbe bool) error {\n\tnumSpecifiedHandlers := 0\n\tif probe.HTTPGet != nil {\n\t\tnumSpecifiedHandlers++\n\t}\n\tif probe.TCPSocket != nil {\n\t\tnumSpecifiedHandlers++\n\t}\n\tif probe.Exec != nil {\n\t\tnumSpecifiedHandlers++\n\t}\n\n\tif numSpecifiedHandlers != 1 {\n\t\tvalidHandlers := []string{userconfig.HTTPGetKey, userconfig.TCPSocketKey}\n\t\tif supportsExecProbe {\n\t\t\tvalidHandlers = append(validHandlers, userconfig.ExecKey)\n\t\t}\n\t\treturn ErrorSpecifyExactlyOneField(numSpecifiedHandlers, validHandlers...)\n\t}\n\n\treturn nil\n}\n\nfunc validatePreStop(preStop userconfig.PreStop) error {\n\tnumSpecifiedHandlers := 0\n\tif preStop.HTTPGet != nil {\n\t\tnumSpecifiedHandlers++\n\t}\n\tif preStop.Exec != nil {\n\t\tnumSpecifiedHandlers++\n\t}\n\n\tif numSpecifiedHandlers != 1 {\n\t\treturn ErrorSpecifyExactlyOneField(numSpecifiedHandlers, userconfig.HTTPGetKey, userconfig.ExecKey)\n\t}\n\n\treturn nil\n}\n\nfunc validateAutoscaling(api *userconfig.API) error {\n\tautoscaling := api.Autoscaling\n\tpod := api.Pod\n\n\tif api.Kind == userconfig.RealtimeAPIKind {\n\t\tif autoscaling.TargetInFlight == nil {\n\t\t\tautoscaling.TargetInFlight = pointer.Float64(float64(pod.MaxConcurrency))\n\t\t}\n\t\tif *autoscaling.TargetInFlight > float64(pod.MaxConcurrency)+float64(pod.MaxQueueLength) {\n\t\t\treturn ErrorTargetInFlightLimitReached(*autoscaling.TargetInFlight, pod.MaxConcurrency, pod.MaxQueueLength)\n\t\t}\n\t}\n\n\tif api.Kind == userconfig.AsyncAPIKind {\n\t\tif autoscaling.TargetInFlight == nil {\n\t\t\tautoscaling.TargetInFlight = pointer.Float64(float64(pod.MaxConcurrency))\n\t\t}\n\t}\n\n\tif autoscaling.MinReplicas > autoscaling.MaxReplicas {\n\t\treturn ErrorMinReplicasGreaterThanMax(autoscaling.MinReplicas, autoscaling.MaxReplicas)\n\t}\n\n\tif autoscaling.InitReplicas > autoscaling.MaxReplicas {\n\t\treturn ErrorInitReplicasGreaterThanMax(autoscaling.InitReplicas, autoscaling.MaxReplicas)\n\t}\n\n\tif autoscaling.InitReplicas < autoscaling.MinReplicas {\n\t\treturn ErrorInitReplicasLessThanMin(autoscaling.InitReplicas, autoscaling.MinReplicas)\n\t}\n\n\treturn nil\n}\n\nfunc validateCompute(api *userconfig.API) error {\n\tcompute := userconfig.GetPodComputeRequest(api)\n\n\tif compute.GPU > 0 && compute.Inf > 0 {\n\t\treturn ErrorComputeResourceConflict(userconfig.GPUKey, userconfig.InfKey)\n\t}\n\n\treturn nil\n}\n\nfunc validateUpdateStrategy(updateStrategy *userconfig.UpdateStrategy) error {\n\tif (updateStrategy.MaxSurge == \"0\" || updateStrategy.MaxSurge == \"0%\") && (updateStrategy.MaxUnavailable == \"0\" || updateStrategy.MaxUnavailable == \"0%\") {\n\t\treturn ErrorSurgeAndUnavailableBothZero()\n\t}\n\n\treturn nil\n}\n\nfunc validateDockerImagePath(\n\timage string,\n\tawsClient *aws.Client,\n\tk8sClient *k8s.Client,\n) error {\n\tdockerClient, err := docker.GetDockerClient()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdockerAuthStr := docker.NoAuth\n\n\tif regex.IsValidECRURL(image) {\n\t\tdockerAuthStr, err = docker.AWSAuthConfig(awsClient)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if k8sClient != nil {\n\t\tdockerAuthStr, err = getDockerAuthStrFromK8s(dockerClient, k8sClient)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif err := docker.CheckImageAccessible(dockerClient, image, dockerAuthStr); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc getDockerAuthStrFromK8s(dockerClient *docker.Client, k8sClient *k8s.Client) (string, error) {\n\tsecretData, err := k8sClient.GetSecretData(_dockerPullSecretName)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\t// check if the user provided the registry auth secret\n\tif secretData == nil {\n\t\treturn docker.NoAuth, nil\n\t}\n\n\tauthData, ok := secretData[\".dockerconfigjson\"]\n\tif !ok {\n\t\treturn \"\", ErrorUnexpectedDockerSecretData(\"should contain \\\".dockerconfigjson\\\" key\", secretData)\n\t}\n\n\tvar authSecret struct {\n\t\tAuths map[string]struct {\n\t\t\tUsername string `json:\"username\"`\n\t\t\tPassword string `json:\"password\"`\n\t\t} `json:\"auths\"`\n\t}\n\n\terr = libjson.Unmarshal(authData, &authSecret)\n\tif err != nil {\n\t\treturn \"\", ErrorUnexpectedDockerSecretData(errors.Message(err), secretData)\n\t}\n\tif len(authSecret.Auths) != 1 {\n\t\treturn \"\", ErrorUnexpectedDockerSecretData(\"should contain a single set of credentials\", secretData)\n\t}\n\n\tvar dockerAuth dockertypes.AuthConfig\n\tfor registryAddress, creds := range authSecret.Auths {\n\t\tdockerAuth = dockertypes.AuthConfig{\n\t\t\tUsername:      creds.Username,\n\t\t\tPassword:      creds.Password,\n\t\t\tServerAddress: registryAddress,\n\t\t}\n\t}\n\tif dockerAuth.Username == \"\" {\n\t\treturn \"\", ErrorUnexpectedDockerSecretData(\"missing username\", secretData)\n\t}\n\tif dockerAuth.Password == \"\" {\n\t\treturn \"\", ErrorUnexpectedDockerSecretData(\"missing password\", secretData)\n\t}\n\tif dockerAuth.ServerAddress == \"\" {\n\t\treturn \"\", ErrorUnexpectedDockerSecretData(\"missing registry address\", secretData)\n\t}\n\n\t_, err = dockerClient.RegistryLogin(context.Background(), dockerAuth)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tdockerAuthStr, err := docker.EncodeAuthConfig(dockerAuth)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn dockerAuthStr, nil\n}\n"
  },
  {
    "path": "pkg/types/status/job_code.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage status\n\n// JobCode is an enum to represent a job status\n// +kubebuilder:validation:Type=string\ntype JobCode int\n\n// Possible values for JobCode\nconst (\n\tJobPending JobCode = iota // pending should be the first status in this list\n\tJobEnqueuing\n\tJobRunning\n\tJobEnqueueFailed\n\tJobCompletedWithFailures\n\tJobSucceeded\n\tJobUnexpectedError\n\tJobWorkerError\n\tJobWorkerOOM\n\tJobTimedOut\n\tJobStopped\n\tJobUnknown\n)\n\nvar _jobCodes = []string{\n\t\"pending\",\n\t\"enqueuing\",\n\t\"running\",\n\t\"enqueue_failed\",\n\t\"completed_with_failures\",\n\t\"succeeded\",\n\t\"unexpected_error\",\n\t\"worker_error\",\n\t\"worker_oom\",\n\t\"timed_out\",\n\t\"stopped\",\n\t\"unknown\",\n}\n\nvar _ = [1]int{}[int(JobUnknown)-(len(_jobCodes)-1)] // Ensure list length matches\n\nvar _jobCodeMessages = []string{\n\t\"pending\",\n\t\"enqueuing\",\n\t\"running\",\n\t\"failed while enqueuing\",\n\t\"completed with failures\",\n\t\"succeeded\",\n\t\"unexpected error\",\n\t\"worker error\",\n\t\"out of memory\",\n\t\"timed out\",\n\t\"stopped\",\n\t\"unknown\",\n}\n\nvar _ = [1]int{}[int(JobUnknown)-(len(_jobCodeMessages)-1)] // Ensure list length matches\n\nfunc (code JobCode) IsNotStarted() bool {\n\treturn code == JobPending || code == JobEnqueuing\n}\n\nfunc (code JobCode) IsInProgress() bool {\n\treturn code == JobEnqueuing || code == JobRunning\n}\n\nfunc (code JobCode) IsCompleted() bool {\n\treturn code == JobEnqueueFailed || code == JobCompletedWithFailures ||\n\t\tcode == JobSucceeded || code == JobUnexpectedError ||\n\t\tcode == JobWorkerError || code == JobWorkerOOM ||\n\t\tcode == JobStopped || code == JobTimedOut\n}\n\nfunc (code JobCode) String() string {\n\tif int(code) < 0 || int(code) >= len(_jobCodes) {\n\t\treturn _jobCodes[JobUnknown]\n\t}\n\treturn _jobCodes[code]\n}\n\nfunc (code JobCode) Message() string {\n\tif int(code) < 0 || int(code) >= len(_jobCodeMessages) {\n\t\treturn _jobCodeMessages[JobUnknown]\n\t}\n\treturn _jobCodeMessages[code]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (code JobCode) MarshalText() ([]byte, error) {\n\treturn []byte(code.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (code *JobCode) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_jobCodes); i++ {\n\t\tif enum == _jobCodes[i] {\n\t\t\t*code = JobCode(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*code = JobUnknown\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (code *JobCode) UnmarshalBinary(data []byte) error {\n\treturn code.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (code JobCode) MarshalBinary() ([]byte, error) {\n\treturn []byte(code.String()), nil\n}\n"
  },
  {
    "path": "pkg/types/status/job_status.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage status\n\nimport (\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n)\n\ntype BatchJobStatus struct {\n\tspec.BatchJob\n\tStatus         JobCode       `json:\"status\" yaml:\"status\"`\n\tEndTime        *time.Time    `json:\"end_time,omitempty\" yaml:\"end_time,omitempty\"`\n\tBatchesInQueue int           `json:\"batches_in_queue\" yaml:\"batches_in_queue\"`\n\tWorkerCounts   *WorkerCounts `json:\"worker_counts,omitempty\" yaml:\"worker_counts,omitempty\"`\n}\n\ntype TaskJobStatus struct {\n\tspec.TaskJob\n\tEndTime      *time.Time    `json:\"end_time,omitempty\" yaml:\"end_time,omitempty\"`\n\tStatus       JobCode       `json:\"status\" yaml:\"status\"`\n\tWorkerCounts *WorkerCounts `json:\"worker_counts,omitempty\" yaml:\"worker_counts,omitempty\"`\n}\n"
  },
  {
    "path": "pkg/types/status/status.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage status\n\nimport (\n\tkapps \"k8s.io/api/apps/v1\"\n)\n\ntype Status struct {\n\tReady         int32          `json:\"ready\" yaml:\"ready\"`           // deployment-reported number of ready replicas (latest + out of date)\n\tRequested     int32          `json:\"requested\" yaml:\"requested\"`   // deployment-reported number of requested replicas\n\tUpToDate      int32          `json:\"up_to_date\" yaml:\"up_to_date\"` // deployment-reported number of up-to-date replicas (in whichever phase they are found in)\n\tReplicaCounts *ReplicaCounts `json:\"replica_counts,omitempty\" yaml:\"replica_counts,omitempty\"`\n}\n\ntype ReplicaCountType string\n\nconst (\n\tReplicaCountRequested      ReplicaCountType = \"Requested\"      // requested number of replicas (for up-to-date pods)\n\tReplicaCountPending        ReplicaCountType = \"Pending\"        // pods that are in the pending state (for up-to-date pods)\n\tReplicaCountCreating       ReplicaCountType = \"Creating\"       // pods that that have their init/non-init containers in the process of being created (for up-to-date pods)\n\tReplicaCountNotReady       ReplicaCountType = \"NotReady\"       // pods that are not passing the readiness checks (for up-to-date pods)\n\tReplicaCountReady          ReplicaCountType = \"Ready\"          // pods that are passing the readiness checks (for up-to-date pods)\n\tReplicaCountReadyOutOfDate ReplicaCountType = \"ReadyOutOfDate\" // pods that are passing the readiness checks (for out-of-date pods)\n\tReplicaCountErrImagePull   ReplicaCountType = \"ErrImagePull\"   // pods that couldn't pull the containers' images (for up-to-date pods)\n\tReplicaCountTerminating    ReplicaCountType = \"Terminating\"    // pods that are in a terminating state (for up-to-date pods)\n\tReplicaCountFailed         ReplicaCountType = \"Failed\"         // pods that have had their containers erroring (for up-to-date pods)\n\tReplicaCountKilled         ReplicaCountType = \"Killed\"         // pods that have had their container processes killed (for up-to-date pods)\n\tReplicaCountKilledOOM      ReplicaCountType = \"KilledOOM\"      // pods that have had their containers OOM (for up-to-date pods)\n\tReplicaCountStalled        ReplicaCountType = \"Stalled\"        // pods that have been in a pending state for more than 15 mins (for up-to-date pods)\n\tReplicaCountUnknown        ReplicaCountType = \"Unknown\"        // pods that are in an unknown state (for up-to-date pods)\n)\n\nvar ReplicaCountTypes []ReplicaCountType = []ReplicaCountType{\n\tReplicaCountRequested, ReplicaCountPending, ReplicaCountCreating,\n\tReplicaCountNotReady, ReplicaCountReady, ReplicaCountReadyOutOfDate,\n\tReplicaCountErrImagePull, ReplicaCountTerminating, ReplicaCountFailed,\n\tReplicaCountKilled, ReplicaCountKilledOOM, ReplicaCountStalled,\n\tReplicaCountUnknown,\n}\n\ntype ReplicaCounts struct {\n\tRequested      int32 `json:\"requested\" yaml:\"requested\"`\n\tPending        int32 `json:\"pending\" yaml:\"pending\"`\n\tCreating       int32 `json:\"creating\" yaml:\"creating\"`\n\tNotReady       int32 `json:\"not_ready\" yaml:\"not_ready\"`\n\tReady          int32 `json:\"ready\" yaml:\"ready\"`\n\tReadyOutOfDate int32 `json:\"ready_out_of_date\" yaml:\"ready_out_of_date\"`\n\tErrImagePull   int32 `json:\"err_image_pull\" yaml:\"err_image_pull\"`\n\tTerminating    int32 `json:\"terminating\" yaml:\"terminating\"` // includes up-to-date and out-of-date pods\n\tFailed         int32 `json:\"failed\" yaml:\"failed\"`\n\tKilled         int32 `json:\"killed\" yaml:\"killed\"`\n\tKilledOOM      int32 `json:\"killed_oom\" yaml:\"killed_oom\"`\n\tStalled        int32 `json:\"stalled\" yaml:\"stalled\"` // pending for a long time\n\tUnknown        int32 `json:\"unknown\" yaml:\"unknown\"`\n}\n\n// Worker counts don't have as many failure variations because Jobs clean up dead pods, so counting different failure scenarios isn't interesting\ntype WorkerCounts struct {\n\tPending      int32 `json:\"pending,omitempty\" yaml:\"pending,omitempty\"`\n\tCreating     int32 `json:\"creating,omitempty\" yaml:\"creating,omitempty\"`\n\tNotReady     int32 `json:\"not_ready,omitempty\" yaml:\"not_ready,omitempty\"`\n\tReady        int32 `json:\"ready,omitempty\" yaml:\"ready,omitempty\"`\n\tSucceeded    int32 `json:\"succeeded,omitempty\" yaml:\"succeeded,omitempty\"`\n\tErrImagePull int32 `json:\"err_image_pull,omitempty\" yaml:\"err_image_pull,omitempty\"`\n\tTerminating  int32 `json:\"terminating,omitempty\" yaml:\"terminating,omitempty\"`\n\tFailed       int32 `json:\"failed,omitempty\" yaml:\"failed,omitempty\"`\n\tKilled       int32 `json:\"killed,omitempty\" yaml:\"killed,omitempty\"`\n\tKilledOOM    int32 `json:\"killed_oom,omitempty\" yaml:\"killed_oom,omitempty\"`\n\tStalled      int32 `json:\"stalled,omitempty\" yaml:\"stalled,omitempty\"` // pending for a long time\n\tUnknown      int32 `json:\"unknown,omitempty\" yaml:\"unknown,omitempty\"`\n}\n\nfunc FromDeployment(deployment *kapps.Deployment) *Status {\n\treturn &Status{\n\t\tReady:     deployment.Status.ReadyReplicas,\n\t\tRequested: deployment.Status.Replicas,\n\t\tUpToDate:  deployment.Status.UpdatedReplicas,\n\t}\n}\n\nfunc (counts *ReplicaCounts) GetCountBy(replicaType ReplicaCountType) int32 {\n\tswitch replicaType {\n\tcase ReplicaCountRequested:\n\t\treturn counts.Requested\n\tcase ReplicaCountPending:\n\t\treturn counts.Pending\n\tcase ReplicaCountCreating:\n\t\treturn counts.Creating\n\tcase ReplicaCountNotReady:\n\t\treturn counts.NotReady\n\tcase ReplicaCountReady:\n\t\treturn counts.Ready\n\tcase ReplicaCountReadyOutOfDate:\n\t\treturn counts.ReadyOutOfDate\n\tcase ReplicaCountErrImagePull:\n\t\treturn counts.ErrImagePull\n\tcase ReplicaCountTerminating:\n\t\treturn counts.Terminating\n\tcase ReplicaCountFailed:\n\t\treturn counts.Failed\n\tcase ReplicaCountKilled:\n\t\treturn counts.Killed\n\tcase ReplicaCountKilledOOM:\n\t\treturn counts.KilledOOM\n\tcase ReplicaCountStalled:\n\t\treturn counts.Stalled\n\t}\n\treturn counts.Unknown\n}\n\nfunc (counts *ReplicaCounts) TotalFailed() int32 {\n\treturn counts.ErrImagePull + counts.Failed + counts.Killed + counts.KilledOOM + counts.Unknown\n}\n"
  },
  {
    "path": "pkg/types/userconfig/api.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage userconfig\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/sets/strset\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/urls\"\n\t\"github.com/cortexlabs/yaml\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n\tkmeta \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\ntype API struct {\n\tResource\n\n\tPod              *Pod            `json:\"pod\" yaml:\"pod\"`\n\tNodeGroups       []string        `json:\"node_groups\" yaml:\"node_groups\"`\n\tAPIs             []*TrafficSplit `json:\"apis\" yaml:\"apis\"`\n\tNetworking       *Networking     `json:\"networking\" yaml:\"networking\"`\n\tAutoscaling      *Autoscaling    `json:\"autoscaling\" yaml:\"autoscaling\"`\n\tUpdateStrategy   *UpdateStrategy `json:\"update_strategy\" yaml:\"update_strategy\"`\n\tIndex            int             `json:\"index\" yaml:\"-\"`\n\tFileName         string          `json:\"file_name\" yaml:\"-\"`\n\tSubmittedAPISpec interface{}     `json:\"submitted_api_spec\" yaml:\"submitted_api_spec\"`\n}\n\ntype Pod struct {\n\tPort           *int32       `json:\"port\" yaml:\"port\"`\n\tMaxQueueLength int64        `json:\"max_queue_length\" yaml:\"max_queue_length\"`\n\tMaxConcurrency int64        `json:\"max_concurrency\" yaml:\"max_concurrency\"`\n\tContainers     []*Container `json:\"containers\" yaml:\"containers\"`\n}\n\ntype Container struct {\n\tName  string            `json:\"name\" yaml:\"name\"`\n\tImage string            `json:\"image\" yaml:\"image\"`\n\tEnv   map[string]string `json:\"env\" yaml:\"env\"`\n\n\tCommand []string `json:\"command\" yaml:\"command\"`\n\tArgs    []string `json:\"args\" yaml:\"args\"`\n\n\tReadinessProbe *Probe   `json:\"readiness_probe\" yaml:\"readiness_probe\"`\n\tLivenessProbe  *Probe   `json:\"liveness_probe\" yaml:\"liveness_probe\"`\n\tPreStop        *PreStop `json:\"pre_stop\" yaml:\"pre_stop\"`\n\n\tCompute *Compute `json:\"compute\" yaml:\"compute\"`\n}\n\ntype TrafficSplit struct {\n\tName   string `json:\"name\" yaml:\"name\"`\n\tWeight int32  `json:\"weight\" yaml:\"weight\"`\n\tShadow bool   `json:\"shadow\" yaml:\"shadow\"`\n}\n\ntype Networking struct {\n\tEndpoint *string `json:\"endpoint\" yaml:\"endpoint\"`\n}\n\ntype Probe struct {\n\tHTTPGet             *HTTPGetHandler   `json:\"http_get\" yaml:\"http_get\"`\n\tTCPSocket           *TCPSocketHandler `json:\"tcp_socket\" yaml:\"tcp_socket\"`\n\tExec                *ExecHandler      `json:\"exec\" yaml:\"exec\"`\n\tInitialDelaySeconds int32             `json:\"initial_delay_seconds\" yaml:\"initial_delay_seconds\"`\n\tTimeoutSeconds      int32             `json:\"timeout_seconds\" yaml:\"timeout_seconds\"`\n\tPeriodSeconds       int32             `json:\"period_seconds\" yaml:\"period_seconds\"`\n\tSuccessThreshold    int32             `json:\"success_threshold\" yaml:\"success_threshold\"`\n\tFailureThreshold    int32             `json:\"failure_threshold\" yaml:\"failure_threshold\"`\n}\n\ntype PreStop struct {\n\tHTTPGet *HTTPGetHandler `json:\"http_get\" yaml:\"http_get\"`\n\tExec    *ExecHandler    `json:\"exec\" yaml:\"exec\"`\n}\n\ntype HTTPGetHandler struct {\n\tPath string `json:\"path\" yaml:\"path\"`\n\tPort int32  `json:\"port\" yaml:\"port\"`\n}\n\ntype TCPSocketHandler struct {\n\tPort int32 `json:\"port\" yaml:\"port\"`\n}\n\ntype ExecHandler struct {\n\tCommand []string `json:\"command\" yaml:\"command\"`\n}\n\ntype Compute struct {\n\tCPU *k8s.Quantity `json:\"cpu\" yaml:\"cpu\"`\n\tMem *k8s.Quantity `json:\"mem\" yaml:\"mem\"`\n\tGPU int64         `json:\"gpu\" yaml:\"gpu\"`\n\tInf int64         `json:\"inf\" yaml:\"inf\"`\n\tShm *k8s.Quantity `json:\"shm\" yaml:\"shm\"`\n}\n\ntype Autoscaling struct {\n\tMinReplicas                  int32         `json:\"min_replicas\" yaml:\"min_replicas\"`\n\tMaxReplicas                  int32         `json:\"max_replicas\" yaml:\"max_replicas\"`\n\tInitReplicas                 int32         `json:\"init_replicas\" yaml:\"init_replicas\"`\n\tTargetInFlight               *float64      `json:\"target_in_flight\" yaml:\"target_in_flight\"`\n\tWindow                       time.Duration `json:\"window\" yaml:\"window\"`\n\tDownscaleStabilizationPeriod time.Duration `json:\"downscale_stabilization_period\" yaml:\"downscale_stabilization_period\"`\n\tUpscaleStabilizationPeriod   time.Duration `json:\"upscale_stabilization_period\" yaml:\"upscale_stabilization_period\"`\n\tMaxDownscaleFactor           float64       `json:\"max_downscale_factor\" yaml:\"max_downscale_factor\"`\n\tMaxUpscaleFactor             float64       `json:\"max_upscale_factor\" yaml:\"max_upscale_factor\"`\n\tDownscaleTolerance           float64       `json:\"downscale_tolerance\" yaml:\"downscale_tolerance\"`\n\tUpscaleTolerance             float64       `json:\"upscale_tolerance\" yaml:\"upscale_tolerance\"`\n}\n\ntype UpdateStrategy struct {\n\tMaxSurge       string `json:\"max_surge\" yaml:\"max_surge\"`\n\tMaxUnavailable string `json:\"max_unavailable\" yaml:\"max_unavailable\"`\n}\n\nfunc (api *API) Identify() string {\n\treturn IdentifyAPI(api.FileName, api.Name, api.Kind, api.Index)\n}\n\nfunc IdentifyAPI(filePath string, name string, kind Kind, index int) string {\n\tstr := \"\"\n\n\tif filePath != \"\" {\n\t\tstr += filePath + \": \"\n\t}\n\n\tif name != \"\" {\n\t\tstr += name\n\t\tif kind != UnknownKind {\n\t\t\tstr += \" (\" + kind.String() + \")\"\n\t\t}\n\t\treturn str\n\t} else if index >= 0 {\n\t\treturn str + \"resource at \" + s.Index(index)\n\t}\n\treturn str + \"resource\"\n}\n\n// InitReplicas was left out deliberately\nfunc (api *API) ToK8sAnnotations() map[string]string {\n\tannotations := map[string]string{}\n\n\tif len(api.APIs) > 0 {\n\t\tannotations[NumTrafficSplitterTargetsAnnotationKey] = s.Int32(int32(len(api.APIs)))\n\t}\n\n\tif api.Pod != nil && api.Kind == RealtimeAPIKind {\n\t\tannotations[MaxConcurrencyAnnotationKey] = s.Int64(api.Pod.MaxConcurrency)\n\t\tannotations[MaxQueueLengthAnnotationKey] = s.Int64(api.Pod.MaxQueueLength)\n\t}\n\n\tif api.Pod != nil && api.Kind == AsyncAPIKind {\n\t\tannotations[MaxConcurrencyAnnotationKey] = s.Int64(api.Pod.MaxConcurrency)\n\t}\n\n\tif api.Networking != nil {\n\t\tannotations[EndpointAnnotationKey] = *api.Networking.Endpoint\n\t}\n\n\tif api.Autoscaling != nil {\n\t\tannotations[MinReplicasAnnotationKey] = s.Int32(api.Autoscaling.MinReplicas)\n\t\tannotations[MaxReplicasAnnotationKey] = s.Int32(api.Autoscaling.MaxReplicas)\n\t\tannotations[TargetInFlightAnnotationKey] = s.Float64(*api.Autoscaling.TargetInFlight)\n\t\tannotations[WindowAnnotationKey] = api.Autoscaling.Window.String()\n\t\tannotations[DownscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.DownscaleStabilizationPeriod.String()\n\t\tannotations[UpscaleStabilizationPeriodAnnotationKey] = api.Autoscaling.UpscaleStabilizationPeriod.String()\n\t\tannotations[MaxDownscaleFactorAnnotationKey] = s.Float64(api.Autoscaling.MaxDownscaleFactor)\n\t\tannotations[MaxUpscaleFactorAnnotationKey] = s.Float64(api.Autoscaling.MaxUpscaleFactor)\n\t\tannotations[DownscaleToleranceAnnotationKey] = s.Float64(api.Autoscaling.DownscaleTolerance)\n\t\tannotations[UpscaleToleranceAnnotationKey] = s.Float64(api.Autoscaling.UpscaleTolerance)\n\t}\n\treturn annotations\n}\n\nfunc AutoscalingFromAnnotations(k8sObj kmeta.Object) (*Autoscaling, error) {\n\ta := Autoscaling{}\n\n\tminReplicas, err := k8s.ParseInt32Annotation(k8sObj, MinReplicasAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.MinReplicas = minReplicas\n\n\tmaxReplicas, err := k8s.ParseInt32Annotation(k8sObj, MaxReplicasAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.MaxReplicas = maxReplicas\n\n\ttargetInFlight, err := k8s.ParseFloat64Annotation(k8sObj, TargetInFlightAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.TargetInFlight = pointer.Float64(targetInFlight)\n\n\twindow, err := k8s.ParseDurationAnnotation(k8sObj, WindowAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.Window = window\n\n\tdownscaleStabilizationPeriod, err := k8s.ParseDurationAnnotation(k8sObj, DownscaleStabilizationPeriodAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.DownscaleStabilizationPeriod = downscaleStabilizationPeriod\n\n\tupscaleStabilizationPeriod, err := k8s.ParseDurationAnnotation(k8sObj, UpscaleStabilizationPeriodAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.UpscaleStabilizationPeriod = upscaleStabilizationPeriod\n\n\tmaxDownscaleFactor, err := k8s.ParseFloat64Annotation(k8sObj, MaxDownscaleFactorAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.MaxDownscaleFactor = maxDownscaleFactor\n\n\tmaxUpscaleFactor, err := k8s.ParseFloat64Annotation(k8sObj, MaxUpscaleFactorAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.MaxUpscaleFactor = maxUpscaleFactor\n\n\tdownscaleTolerance, err := k8s.ParseFloat64Annotation(k8sObj, DownscaleToleranceAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.DownscaleTolerance = downscaleTolerance\n\n\tupscaleTolerance, err := k8s.ParseFloat64Annotation(k8sObj, UpscaleToleranceAnnotationKey)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ta.UpscaleTolerance = upscaleTolerance\n\n\treturn &a, nil\n}\n\nfunc TrafficSplitterTargetsFromAnnotations(k8sObj kmeta.Object) (int32, error) {\n\ttargets, err := k8s.ParseInt32Annotation(k8sObj, NumTrafficSplitterTargetsAnnotationKey)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\treturn targets, nil\n}\n\nfunc EndpointFromAnnotation(k8sObj kmeta.Object) (string, error) {\n\tendpoint, err := k8s.GetAnnotation(k8sObj, EndpointAnnotationKey)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn endpoint, nil\n}\n\nfunc ConcurrencyFromAnnotations(k8sObj kmeta.Object) (int, int, error) {\n\tmaxQueueLength, err := k8s.ParseIntAnnotation(k8sObj, MaxQueueLengthAnnotationKey)\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\n\tmaxConcurrency, err := k8s.ParseIntAnnotation(k8sObj, MaxConcurrencyAnnotationKey)\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\n\treturn maxQueueLength, maxConcurrency, nil\n}\n\nfunc (api *API) UserStr() string {\n\tvar sb strings.Builder\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", NameKey, api.Name))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", KindKey, api.Kind.String()))\n\n\tif api.Kind == TrafficSplitterKind {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", APIsKey))\n\t\tfor _, api := range api.APIs {\n\t\t\tsb.WriteString(s.Indent(api.UserStr(), \"  \"))\n\t\t}\n\t}\n\n\tif api.Pod != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", PodKey))\n\t\tsb.WriteString(s.Indent(api.Pod.UserStr(api.Kind), \"  \"))\n\t}\n\n\tif api.Networking != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", NetworkingKey))\n\t\tsb.WriteString(s.Indent(api.Networking.UserStr(), \"  \"))\n\t}\n\n\tif api.Autoscaling != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", AutoscalingKey))\n\t\tsb.WriteString(s.Indent(api.Autoscaling.UserStr(), \"  \"))\n\t}\n\n\tif api.NodeGroups == nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: null\\n\", NodeGroupsKey))\n\t} else {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", NodeGroupsKey, s.ObjFlatNoQuotes(api.NodeGroups)))\n\t}\n\n\tif api.UpdateStrategy != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", UpdateStrategyKey))\n\t\tsb.WriteString(s.Indent(api.UpdateStrategy.UserStr(), \"  \"))\n\t}\n\n\treturn sb.String()\n}\n\nfunc (trafficSplit *TrafficSplit) UserStr() string {\n\tvar sb strings.Builder\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", NameKey, trafficSplit.Name))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", WeightKey, s.Int32(trafficSplit.Weight)))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", ShadowKey, s.Bool(trafficSplit.Shadow)))\n\treturn sb.String()\n}\n\nfunc (pod *Pod) UserStr(kind Kind) string {\n\tvar sb strings.Builder\n\tif pod.Port != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %d\\n\", PortKey, *pod.Port))\n\t}\n\n\tif kind == RealtimeAPIKind {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MaxConcurrencyKey, s.Int64(pod.MaxConcurrency)))\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MaxQueueLengthKey, s.Int64(pod.MaxQueueLength)))\n\t}\n\n\tif kind == AsyncAPIKind {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MaxConcurrencyKey, s.Int64(pod.MaxConcurrency)))\n\t}\n\n\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", ContainersKey))\n\tfor _, container := range pod.Containers {\n\t\tcontainerUserStr := s.Indent(container.UserStr(), \"    \")\n\t\tcontainerUserStr = containerUserStr[:2] + \"-\" + containerUserStr[3:]\n\t\tsb.WriteString(containerUserStr)\n\t}\n\n\treturn sb.String()\n}\n\nfunc (container *Container) UserStr() string {\n\tvar sb strings.Builder\n\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", ContainerNameKey, container.Name))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", ImageKey, container.Image))\n\n\tif len(container.Env) > 0 {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", EnvKey))\n\t\td, _ := yaml.Marshal(&container.Env)\n\t\tsb.WriteString(s.Indent(string(d), \"  \"))\n\t}\n\n\tif container.Command != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", CommandKey, s.ObjFlatNoQuotes(container.Command)))\n\t}\n\n\tif container.Args != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", ArgsKey, s.ObjFlatNoQuotes(container.Args)))\n\t}\n\n\tif container.ReadinessProbe != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", ReadinessProbeKey))\n\t\tsb.WriteString(s.Indent(container.ReadinessProbe.UserStr(), \"  \"))\n\t}\n\n\tif container.LivenessProbe != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", LivenessProbeKey))\n\t\tsb.WriteString(s.Indent(container.LivenessProbe.UserStr(), \"  \"))\n\t}\n\n\tif container.PreStop != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", PreStopKey))\n\t\tsb.WriteString(s.Indent(container.PreStop.UserStr(), \"  \"))\n\t}\n\n\tif container.Compute != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", ComputeKey))\n\t\tsb.WriteString(s.Indent(container.Compute.UserStr(), \"  \"))\n\t}\n\n\treturn sb.String()\n}\n\nfunc (networking *Networking) UserStr() string {\n\tvar sb strings.Builder\n\tif networking.Endpoint != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", EndpointKey, *networking.Endpoint))\n\t}\n\treturn sb.String()\n}\n\nfunc (probe *Probe) UserStr() string {\n\tvar sb strings.Builder\n\n\tif probe.HTTPGet != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", HTTPGetKey))\n\t\tsb.WriteString(s.Indent(probe.HTTPGet.UserStr(), \"  \"))\n\t}\n\tif probe.TCPSocket != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", TCPSocketKey))\n\t\tsb.WriteString(s.Indent(probe.TCPSocket.UserStr(), \"  \"))\n\t}\n\tif probe.Exec != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", ExecKey))\n\t\tsb.WriteString(s.Indent(probe.Exec.UserStr(), \"  \"))\n\t}\n\n\tsb.WriteString(fmt.Sprintf(\"%s: %d\\n\", InitialDelaySecondsKey, probe.InitialDelaySeconds))\n\tsb.WriteString(fmt.Sprintf(\"%s: %d\\n\", TimeoutSecondsKey, probe.TimeoutSeconds))\n\tsb.WriteString(fmt.Sprintf(\"%s: %d\\n\", PeriodSecondsKey, probe.PeriodSeconds))\n\tsb.WriteString(fmt.Sprintf(\"%s: %d\\n\", SuccessThresholdKey, probe.SuccessThreshold))\n\tsb.WriteString(fmt.Sprintf(\"%s: %d\\n\", FailureThresholdKey, probe.FailureThreshold))\n\n\treturn sb.String()\n}\n\nfunc (preStop *PreStop) UserStr() string {\n\tvar sb strings.Builder\n\n\tif preStop.HTTPGet != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", HTTPGetKey))\n\t\tsb.WriteString(s.Indent(preStop.HTTPGet.UserStr(), \"  \"))\n\t}\n\tif preStop.Exec != nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s:\\n\", ExecKey))\n\t\tsb.WriteString(s.Indent(preStop.Exec.UserStr(), \"  \"))\n\t}\n\n\treturn sb.String()\n}\n\nfunc (httpHandler *HTTPGetHandler) UserStr() string {\n\tvar sb strings.Builder\n\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", PathKey, httpHandler.Path))\n\tsb.WriteString(fmt.Sprintf(\"%s: %d\\n\", PortKey, httpHandler.Port))\n\n\treturn sb.String()\n}\n\nfunc (tcpSocketHandler *TCPSocketHandler) UserStr() string {\n\tvar sb strings.Builder\n\tsb.WriteString(fmt.Sprintf(\"%s: %d\\n\", PortKey, tcpSocketHandler.Port))\n\treturn sb.String()\n}\n\nfunc (execHandler *ExecHandler) UserStr() string {\n\tvar sb strings.Builder\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", CommandKey, s.ObjFlatNoQuotes(execHandler.Command)))\n\treturn sb.String()\n}\n\nfunc (compute *Compute) UserStr() string {\n\tvar sb strings.Builder\n\tif compute.CPU == nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: null  # no limit\\n\", CPUKey))\n\t} else {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", CPUKey, compute.CPU.UserString))\n\t}\n\tif compute.GPU > 0 {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", GPUKey, s.Int64(compute.GPU)))\n\t}\n\tif compute.Inf > 0 {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", InfKey, s.Int64(compute.Inf)))\n\t}\n\tif compute.Mem == nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: null  # no limit\\n\", MemKey))\n\t} else {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MemKey, compute.Mem.UserString))\n\t}\n\tif compute.Shm == nil {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: null  # not configured\\n\", ShmKey))\n\t} else {\n\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", ShmKey, compute.Shm.UserString))\n\t}\n\treturn sb.String()\n}\n\nfunc (autoscaling *Autoscaling) UserStr() string {\n\tvar sb strings.Builder\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MinReplicasKey, s.Int32(autoscaling.MinReplicas)))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MaxReplicasKey, s.Int32(autoscaling.MaxReplicas)))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", InitReplicasKey, s.Int32(autoscaling.InitReplicas)))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", TargetInFlightKey, s.Float64(*autoscaling.TargetInFlight)))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", WindowKey, autoscaling.Window.String()))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", DownscaleStabilizationPeriodKey, autoscaling.DownscaleStabilizationPeriod.String()))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", UpscaleStabilizationPeriodKey, autoscaling.UpscaleStabilizationPeriod.String()))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MaxDownscaleFactorKey, s.Float64(autoscaling.MaxDownscaleFactor)))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MaxUpscaleFactorKey, s.Float64(autoscaling.MaxUpscaleFactor)))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", DownscaleToleranceKey, s.Float64(autoscaling.DownscaleTolerance)))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", UpscaleToleranceKey, s.Float64(autoscaling.UpscaleTolerance)))\n\n\treturn sb.String()\n}\n\nfunc (updateStrategy *UpdateStrategy) UserStr() string {\n\tvar sb strings.Builder\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MaxSurgeKey, updateStrategy.MaxSurge))\n\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", MaxUnavailableKey, updateStrategy.MaxUnavailable))\n\treturn sb.String()\n}\n\nfunc ZeroCompute() Compute {\n\treturn Compute{\n\t\tCPU: &k8s.Quantity{},\n\t\tMem: &k8s.Quantity{},\n\t\tGPU: 0,\n\t}\n}\n\nfunc GetPodComputeRequest(api *API) Compute {\n\tvar cpuQtys []kresource.Quantity\n\tvar memQtys []kresource.Quantity\n\tvar shmQtys []kresource.Quantity\n\tvar totalGPU int64\n\tvar totalInf int64\n\n\tfor _, container := range api.Pod.Containers {\n\t\tif container == nil || container.Compute == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif container.Compute.CPU != nil {\n\t\t\tcpuQtys = append(cpuQtys, container.Compute.CPU.Quantity)\n\t\t}\n\t\tif container.Compute.Mem != nil {\n\t\t\tmemQtys = append(memQtys, container.Compute.Mem.Quantity)\n\t\t}\n\t\tif container.Compute.Shm != nil {\n\t\t\tshmQtys = append(shmQtys, container.Compute.Shm.Quantity)\n\t\t}\n\t\ttotalGPU += container.Compute.GPU\n\t\ttotalInf += container.Compute.Inf\n\t}\n\n\tif api.Kind == RealtimeAPIKind {\n\t\tcpuQtys = append(cpuQtys, consts.CortexProxyCPU)\n\t\tmemQtys = append(memQtys, consts.CortexProxyMem)\n\t} else if api.Kind == AsyncAPIKind || api.Kind == BatchAPIKind {\n\t\tcpuQtys = append(cpuQtys, consts.CortexDequeuerCPU)\n\t\tmemQtys = append(memQtys, consts.CortexDequeuerMem)\n\t}\n\n\treturn Compute{\n\t\tCPU: k8s.NewSummed(cpuQtys...),\n\t\tMem: k8s.NewSummed(memQtys...),\n\t\tShm: k8s.NewSummed(shmQtys...),\n\t\tGPU: totalGPU,\n\t\tInf: totalInf,\n\t}\n}\n\nfunc GetContainerNames(containers []*Container) strset.Set {\n\tcontainerNames := strset.New()\n\tfor _, container := range containers {\n\t\tif container != nil {\n\t\t\tcontainerNames.Add(container.Name)\n\t\t}\n\t}\n\treturn containerNames\n}\n\nfunc (api *API) TelemetryEvent() map[string]interface{} {\n\tevent := map[string]interface{}{\"kind\": api.Kind}\n\n\tif len(api.APIs) > 0 {\n\t\tevent[\"apis._is_defined\"] = true\n\t\tevent[\"apis._len\"] = len(api.APIs)\n\t}\n\n\tif api.Networking != nil {\n\t\tevent[\"networking._is_defined\"] = true\n\t\tif api.Networking.Endpoint != nil {\n\t\t\tevent[\"networking.endpoint._is_defined\"] = true\n\t\t\tif urls.CanonicalizeEndpoint(api.Name) != *api.Networking.Endpoint {\n\t\t\t\tevent[\"networking.endpoint._is_custom\"] = true\n\t\t\t}\n\t\t}\n\t}\n\n\tif api.Pod != nil {\n\t\tevent[\"pod._is_defined\"] = true\n\t\tif api.Pod.Port != nil {\n\t\t\tevent[\"pod.port\"] = *api.Pod.Port\n\t\t}\n\n\t\tevent[\"pod.max_concurrency\"] = api.Pod.MaxConcurrency\n\t\tevent[\"pod.max_queue_length\"] = api.Pod.MaxQueueLength\n\n\t\tevent[\"pod.containers._len\"] = len(api.Pod.Containers)\n\n\t\tvar numReadinessProbes int\n\t\tvar numLivenessProbes int\n\t\tvar numPreStops int\n\t\tfor _, container := range api.Pod.Containers {\n\t\t\tif container.ReadinessProbe != nil {\n\t\t\t\tnumReadinessProbes++\n\t\t\t}\n\t\t\tif container.LivenessProbe != nil {\n\t\t\t\tnumLivenessProbes++\n\t\t\t}\n\t\t\tif container.PreStop != nil {\n\t\t\t\tnumPreStops++\n\t\t\t}\n\t\t}\n\n\t\tevent[\"pod.containers._num_readiness_probes\"] = numReadinessProbes\n\t\tevent[\"pod.containers._num_liveness_probes\"] = numLivenessProbes\n\t\tevent[\"pod.containers._num_pre_stops\"] = numPreStops\n\n\t\ttotalCompute := GetPodComputeRequest(api)\n\t\tif totalCompute.CPU != nil {\n\t\t\tevent[\"pod.containers.compute.cpu._is_defined\"] = true\n\t\t\tevent[\"pod.containers.compute.cpu\"] = float64(totalCompute.CPU.MilliValue()) / 1000\n\t\t}\n\t\tif totalCompute.Mem != nil {\n\t\t\tevent[\"pod.containers.compute.mem._is_defined\"] = true\n\t\t\tevent[\"pod.containers.compute.mem\"] = totalCompute.Mem.Value()\n\t\t}\n\t\tif totalCompute.Shm != nil {\n\t\t\tevent[\"pod.containers.compute.shm._is_defined\"] = true\n\t\t\tevent[\"pod.containers.compute.shm\"] = totalCompute.Shm.Value()\n\t\t}\n\t\tevent[\"pod.containers.compute.gpu\"] = totalCompute.GPU\n\t\tevent[\"pod.containers.compute.inf\"] = totalCompute.Inf\n\t}\n\n\tevent[\"node_groups._len\"] = len(api.NodeGroups)\n\n\tif api.UpdateStrategy != nil {\n\t\tevent[\"update_strategy._is_defined\"] = true\n\t\tevent[\"update_strategy.max_surge\"] = api.UpdateStrategy.MaxSurge\n\t\tevent[\"update_strategy.max_unavailable\"] = api.UpdateStrategy.MaxUnavailable\n\t}\n\n\tif api.Autoscaling != nil {\n\t\tevent[\"autoscaling._is_defined\"] = true\n\t\tevent[\"autoscaling.min_replicas\"] = api.Autoscaling.MinReplicas\n\t\tevent[\"autoscaling.max_replicas\"] = api.Autoscaling.MaxReplicas\n\t\tevent[\"autoscaling.init_replicas\"] = api.Autoscaling.InitReplicas\n\t\tif api.Autoscaling.TargetInFlight != nil {\n\t\t\tevent[\"autoscaling.target_in_flight._is_defined\"] = true\n\t\t\tevent[\"autoscaling.target_in_flight\"] = *api.Autoscaling.TargetInFlight\n\t\t}\n\t\tevent[\"autoscaling.window\"] = api.Autoscaling.Window.Seconds()\n\t\tevent[\"autoscaling.downscale_stabilization_period\"] = api.Autoscaling.DownscaleStabilizationPeriod.Seconds()\n\t\tevent[\"autoscaling.upscale_stabilization_period\"] = api.Autoscaling.UpscaleStabilizationPeriod.Seconds()\n\t\tevent[\"autoscaling.max_downscale_factor\"] = api.Autoscaling.MaxDownscaleFactor\n\t\tevent[\"autoscaling.max_upscale_factor\"] = api.Autoscaling.MaxUpscaleFactor\n\t\tevent[\"autoscaling.downscale_tolerance\"] = api.Autoscaling.DownscaleTolerance\n\t\tevent[\"autoscaling.upscale_tolerance\"] = api.Autoscaling.UpscaleTolerance\n\t}\n\n\treturn event\n}\n"
  },
  {
    "path": "pkg/types/userconfig/config_key.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage userconfig\n\nconst (\n\t// API\n\tNameKey           = \"name\"\n\tKindKey           = \"kind\"\n\tNetworkingKey     = \"networking\"\n\tComputeKey        = \"compute\"\n\tAutoscalingKey    = \"autoscaling\"\n\tUpdateStrategyKey = \"update_strategy\"\n\n\t// TrafficSplitter\n\tAPIsKey   = \"apis\"\n\tWeightKey = \"weight\"\n\tShadowKey = \"shadow\"\n\n\t// Pod\n\tPodKey            = \"pod\"\n\tNodeGroupsKey     = \"node_groups\"\n\tPortKey           = \"port\"\n\tMaxConcurrencyKey = \"max_concurrency\"\n\tMaxQueueLengthKey = \"max_queue_length\"\n\tContainersKey     = \"containers\"\n\n\t// Containers\n\tContainerNameKey  = \"name\"\n\tImageKey          = \"image\"\n\tEnvKey            = \"env\"\n\tCommandKey        = \"command\"\n\tArgsKey           = \"args\"\n\tReadinessProbeKey = \"readiness_probe\"\n\tLivenessProbeKey  = \"liveness_probe\"\n\tPreStopKey        = \"pre_stop\"\n\n\t// Probe\n\tHTTPGetKey             = \"http_get\"\n\tTCPSocketKey           = \"tcp_socket\"\n\tExecKey                = \"exec\"\n\tInitialDelaySecondsKey = \"initial_delay_seconds\"\n\tTimeoutSecondsKey      = \"timeout_seconds\"\n\tPeriodSecondsKey       = \"period_seconds\"\n\tSuccessThresholdKey    = \"success_threshold\"\n\tFailureThresholdKey    = \"failure_threshold\"\n\n\t// Probe types\n\tPathKey = \"path\"\n\n\t// Compute\n\tCPUKey = \"cpu\"\n\tMemKey = \"mem\"\n\tGPUKey = \"gpu\"\n\tInfKey = \"inf\"\n\tShmKey = \"shm\"\n\n\t// Networking\n\tEndpointKey = \"endpoint\"\n\n\t// Autoscaling\n\tMinReplicasKey                  = \"min_replicas\"\n\tMaxReplicasKey                  = \"max_replicas\"\n\tInitReplicasKey                 = \"init_replicas\"\n\tTargetInFlightKey               = \"target_in_flight\"\n\tWindowKey                       = \"window\"\n\tDownscaleStabilizationPeriodKey = \"downscale_stabilization_period\"\n\tUpscaleStabilizationPeriodKey   = \"upscale_stabilization_period\"\n\tMaxDownscaleFactorKey           = \"max_downscale_factor\"\n\tMaxUpscaleFactorKey             = \"max_upscale_factor\"\n\tDownscaleToleranceKey           = \"downscale_tolerance\"\n\tUpscaleToleranceKey             = \"upscale_tolerance\"\n\n\t// UpdateStrategy\n\tMaxSurgeKey       = \"max_surge\"\n\tMaxUnavailableKey = \"max_unavailable\"\n\n\t// K8s annotation\n\tEndpointAnnotationKey                     = \"networking.cortex.dev/endpoint\"\n\tMaxConcurrencyAnnotationKey               = \"pod.cortex.dev/max-concurrency\"\n\tMaxQueueLengthAnnotationKey               = \"pod.cortex.dev/max-queue-length\"\n\tNumTrafficSplitterTargetsAnnotationKey    = \"apis.cortex.dev/traffic-splitter-targets\"\n\tMinReplicasAnnotationKey                  = \"autoscaling.cortex.dev/min-replicas\"\n\tMaxReplicasAnnotationKey                  = \"autoscaling.cortex.dev/max-replicas\"\n\tTargetInFlightAnnotationKey               = \"autoscaling.cortex.dev/target-in-flight\"\n\tWindowAnnotationKey                       = \"autoscaling.cortex.dev/window\"\n\tDownscaleStabilizationPeriodAnnotationKey = \"autoscaling.cortex.dev/downscale-stabilization-period\"\n\tUpscaleStabilizationPeriodAnnotationKey   = \"autoscaling.cortex.dev/upscale-stabilization-period\"\n\tMaxDownscaleFactorAnnotationKey           = \"autoscaling.cortex.dev/max-downscale-factor\"\n\tMaxUpscaleFactorAnnotationKey             = \"autoscaling.cortex.dev/max-upscale-factor\"\n\tDownscaleToleranceAnnotationKey           = \"autoscaling.cortex.dev/downscale-tolerance\"\n\tUpscaleToleranceAnnotationKey             = \"autoscaling.cortex.dev/upscale-tolerance\"\n)\n"
  },
  {
    "path": "pkg/types/userconfig/kind.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage userconfig\n\ntype Kind int\n\nconst (\n\tUnknownKind Kind = iota\n\tRealtimeAPIKind\n\tBatchAPIKind\n\tTrafficSplitterKind\n\tTaskAPIKind\n\tAsyncAPIKind\n)\n\nvar _kinds = []string{\n\t\"unknown\",\n\t\"RealtimeAPI\",\n\t\"BatchAPI\",\n\t\"TrafficSplitter\",\n\t\"TaskAPI\",\n\t\"AsyncAPI\",\n}\n\nfunc KindFromString(s string) Kind {\n\tfor i := 0; i < len(_kinds); i++ {\n\t\tif s == _kinds[i] {\n\t\t\treturn Kind(i)\n\t\t}\n\t}\n\treturn UnknownKind\n}\n\nfunc KindStrings() []string {\n\treturn _kinds[1:]\n}\n\nfunc (t Kind) String() string {\n\treturn _kinds[t]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (t Kind) MarshalText() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (t *Kind) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_kinds); i++ {\n\t\tif enum == _kinds[i] {\n\t\t\t*t = Kind(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*t = UnknownKind\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (t *Kind) UnmarshalBinary(data []byte) error {\n\treturn t.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (t Kind) MarshalBinary() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n"
  },
  {
    "path": "pkg/types/userconfig/log_level.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage userconfig\n\nimport \"go.uber.org/zap/zapcore\"\n\ntype LogLevel int\n\nconst (\n\tUnknownLogLevel LogLevel = iota\n\tDebugLogLevel\n\tInfoLogLevel\n\tWarningLogLevel\n\tErrorLogLevel\n)\n\nvar _logLevels = []string{\n\t\"unknown\",\n\t\"debug\",\n\t\"info\",\n\t\"warning\",\n\t\"error\",\n}\n\nfunc LogLevelFromString(s string) LogLevel {\n\tfor i := 0; i < len(_logLevels); i++ {\n\t\tif s == _logLevels[i] {\n\t\t\treturn LogLevel(i)\n\t\t}\n\t}\n\treturn UnknownLogLevel\n}\n\nfunc LogLevelTypes() []string {\n\treturn _logLevels[1:]\n}\n\nfunc (t LogLevel) String() string {\n\treturn _logLevels[t]\n}\n\n// MarshalText satisfies TextMarshaler\nfunc (t LogLevel) MarshalText() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\n// UnmarshalText satisfies TextUnmarshaler\nfunc (t *LogLevel) UnmarshalText(text []byte) error {\n\tenum := string(text)\n\tfor i := 0; i < len(_logLevels); i++ {\n\t\tif enum == _logLevels[i] {\n\t\t\t*t = LogLevel(i)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\t*t = UnknownLogLevel\n\treturn nil\n}\n\n// UnmarshalBinary satisfies BinaryUnmarshaler\n// Needed for msgpack\nfunc (t *LogLevel) UnmarshalBinary(data []byte) error {\n\treturn t.UnmarshalText(data)\n}\n\n// MarshalBinary satisfies BinaryMarshaler\nfunc (t LogLevel) MarshalBinary() ([]byte, error) {\n\treturn []byte(t.String()), nil\n}\n\nfunc ToZapLogLevel(logLevel LogLevel) zapcore.Level {\n\tswitch logLevel {\n\tcase InfoLogLevel:\n\t\treturn zapcore.InfoLevel\n\tcase WarningLogLevel:\n\t\treturn zapcore.WarnLevel\n\tcase ErrorLogLevel:\n\t\treturn zapcore.ErrorLevel\n\tdefault:\n\t\treturn zapcore.DebugLevel\n\t}\n}\n"
  },
  {
    "path": "pkg/types/userconfig/resource.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage userconfig\n\nimport \"fmt\"\n\ntype Resource struct {\n\tName string `json:\"name\" yaml:\"name\"`\n\tKind Kind   `json:\"kind\" yaml:\"kind\"`\n}\n\nfunc (r Resource) UserString() string {\n\treturn fmt.Sprintf(\"%s (%s)\", r.Name, r.Kind.String())\n}\n"
  },
  {
    "path": "pkg/workloads/configmap.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage workloads\n\nimport (\n\tlibjson \"github.com/cortexlabs/cortex/pkg/lib/json\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\ntype ConfigMapConfig struct {\n\tBatchJob *spec.BatchJob\n\tTaskJob  *spec.TaskJob\n\tProbes   map[string]kcore.Probe\n}\n\nfunc (c *ConfigMapConfig) GenerateConfigMapData() (map[string]string, error) {\n\tif c == nil {\n\t\treturn nil, nil\n\t}\n\n\tdata := map[string]string{}\n\tif len(c.Probes) > 0 {\n\t\tprobesEncoded, err := libjson.MarshalIndent(c.Probes)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdata[\"probes.json\"] = string(probesEncoded)\n\t}\n\n\tif c.TaskJob != nil {\n\t\tjobSpecEncoded, err := libjson.MarshalIndent(*c.TaskJob)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdata[\"job.json\"] = string(jobSpecEncoded)\n\t}\n\n\tif c.BatchJob != nil {\n\t\tjobSpecEncoded, err := libjson.MarshalIndent(*c.BatchJob)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdata[\"job.json\"] = string(jobSpecEncoded)\n\t}\n\n\treturn data, nil\n}\n"
  },
  {
    "path": "pkg/workloads/helpers.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage workloads\n\nimport (\n\t\"path\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\tkcore \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nfunc K8sName(apiName string) string {\n\treturn \"api-\" + apiName\n}\n\nfunc GetProbeSpec(probe *userconfig.Probe) *kcore.Probe {\n\tif probe == nil {\n\t\treturn nil\n\t}\n\n\tvar httpGetAction *kcore.HTTPGetAction\n\tvar tcpSocketAction *kcore.TCPSocketAction\n\tvar execAction *kcore.ExecAction\n\n\tif probe.HTTPGet != nil {\n\t\thttpGetAction = &kcore.HTTPGetAction{\n\t\t\tPath: probe.HTTPGet.Path,\n\t\t\tPort: intstr.IntOrString{\n\t\t\t\tIntVal: probe.HTTPGet.Port,\n\t\t\t},\n\t\t}\n\t}\n\tif probe.TCPSocket != nil {\n\t\ttcpSocketAction = &kcore.TCPSocketAction{\n\t\t\tPort: intstr.IntOrString{\n\t\t\t\tIntVal: probe.TCPSocket.Port,\n\t\t\t},\n\t\t}\n\t}\n\tif probe.Exec != nil {\n\t\texecAction = &kcore.ExecAction{\n\t\t\tCommand: probe.Exec.Command,\n\t\t}\n\t}\n\n\treturn &kcore.Probe{\n\t\tHandler: kcore.Handler{\n\t\t\tHTTPGet:   httpGetAction,\n\t\t\tTCPSocket: tcpSocketAction,\n\t\t\tExec:      execAction,\n\t\t},\n\t\tInitialDelaySeconds: probe.InitialDelaySeconds,\n\t\tTimeoutSeconds:      probe.TimeoutSeconds,\n\t\tPeriodSeconds:       probe.PeriodSeconds,\n\t\tSuccessThreshold:    probe.SuccessThreshold,\n\t\tFailureThreshold:    probe.FailureThreshold,\n\t}\n}\n\nfunc GetLifecycleSpec(preStop *userconfig.PreStop) *kcore.Lifecycle {\n\tif preStop == nil {\n\t\treturn nil\n\t}\n\n\tvar httpGetAction *kcore.HTTPGetAction\n\tvar execAction *kcore.ExecAction\n\n\tif preStop.HTTPGet != nil {\n\t\thttpGetAction = &kcore.HTTPGetAction{\n\t\t\tPath: strings.TrimPrefix(preStop.HTTPGet.Path, \"/\"), // the leading / is automatically added by k8s\n\t\t\tPort: intstr.IntOrString{\n\t\t\t\tIntVal: preStop.HTTPGet.Port,\n\t\t\t},\n\t\t}\n\t}\n\tif preStop.Exec != nil {\n\t\texecAction = &kcore.ExecAction{\n\t\t\tCommand: preStop.Exec.Command,\n\t\t}\n\t}\n\n\treturn &kcore.Lifecycle{\n\t\tPreStop: &kcore.Handler{\n\t\t\tHTTPGet: httpGetAction,\n\t\t\tExec:    execAction,\n\t\t},\n\t}\n}\n\nfunc GetReadinessProbesFromContainers(containers []*userconfig.Container) map[string]kcore.Probe {\n\tprobes := map[string]kcore.Probe{}\n\n\tfor _, container := range containers {\n\t\t// this should never happen, it's just a precaution\n\t\tif container == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif container.ReadinessProbe != nil {\n\t\t\tprobes[container.Name] = *GetProbeSpec(container.ReadinessProbe)\n\t\t}\n\t}\n\n\treturn probes\n}\n\nfunc HasReadinessProbesTargetingPort(containers []*userconfig.Container, targetPort int32) bool {\n\tfor _, container := range containers {\n\t\tif container == nil || container.ReadinessProbe == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tprobe := container.ReadinessProbe\n\t\tif (probe.TCPSocket != nil && probe.TCPSocket.Port == targetPort) ||\n\t\t\tprobe.HTTPGet != nil && probe.HTTPGet.Port == targetPort {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc BaseClusterEnvVars() []kcore.EnvFromSource {\n\tenvVars := []kcore.EnvFromSource{\n\t\t{\n\t\t\tConfigMapRef: &kcore.ConfigMapEnvSource{\n\t\t\t\tLocalObjectReference: kcore.LocalObjectReference{\n\t\t\t\t\tName: \"env-vars\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\treturn envVars\n}\n\nfunc getKubexitEnvVars(containerName string, deathDeps []string, birthDeps []string) []kcore.EnvVar {\n\tenvVars := []kcore.EnvVar{\n\t\t{\n\t\t\tName:  \"KUBEXIT_NAME\",\n\t\t\tValue: containerName,\n\t\t},\n\t\t{\n\t\t\tName:  \"KUBEXIT_GRAVEYARD\",\n\t\t\tValue: _kubexitGraveyardMountPath,\n\t\t},\n\t}\n\n\tif deathDeps != nil {\n\t\tenvVars = append(envVars,\n\t\t\tkcore.EnvVar{\n\t\t\t\tName:  \"KUBEXIT_DEATH_DEPS\",\n\t\t\t\tValue: strings.Join(deathDeps, \",\"),\n\t\t\t},\n\t\t\tkcore.EnvVar{\n\t\t\t\tName:  \"KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS\",\n\t\t\t\tValue: \"true\",\n\t\t\t},\n\t\t)\n\t}\n\n\tif birthDeps != nil {\n\t\tenvVars = append(envVars,\n\t\t\tkcore.EnvVar{\n\t\t\t\tName:  \"KUBEXIT_BIRTH_DEPS\",\n\t\t\t\tValue: strings.Join(birthDeps, \",\"),\n\t\t\t},\n\t\t\tkcore.EnvVar{\n\t\t\t\tName:  \"KUBEXIT_IGNORE_CODE_ON_DEATH_DEPS\",\n\t\t\t\tValue: \"true\",\n\t\t\t},\n\t\t)\n\t}\n\n\treturn envVars\n}\n\nfunc MntVolume() kcore.Volume {\n\treturn k8s.EmptyDirVolume(_emptyDirVolumeName)\n}\n\nfunc CortexVolume() kcore.Volume {\n\treturn k8s.EmptyDirVolume(_cortexDirVolumeName)\n}\n\nfunc APIConfigVolume(name string) kcore.Volume {\n\treturn kcore.Volume{\n\t\tName: name,\n\t\tVolumeSource: kcore.VolumeSource{\n\t\t\tConfigMap: &kcore.ConfigMapVolumeSource{\n\t\t\t\tLocalObjectReference: kcore.LocalObjectReference{\n\t\t\t\t\tName: name,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc ClientConfigVolume() kcore.Volume {\n\treturn kcore.Volume{\n\t\tName: _clientConfigDirVolume,\n\t\tVolumeSource: kcore.VolumeSource{\n\t\t\tConfigMap: &kcore.ConfigMapVolumeSource{\n\t\t\t\tLocalObjectReference: kcore.LocalObjectReference{\n\t\t\t\t\tName: _clientConfigConfigMap,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc ClusterConfigVolume() kcore.Volume {\n\treturn kcore.Volume{\n\t\tName: _clusterConfigDirVolume,\n\t\tVolumeSource: kcore.VolumeSource{\n\t\t\tConfigMap: &kcore.ConfigMapVolumeSource{\n\t\t\t\tLocalObjectReference: kcore.LocalObjectReference{\n\t\t\t\t\tName: _clusterConfigConfigMap,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc ShmVolume(q resource.Quantity, volumeName string) kcore.Volume {\n\treturn kcore.Volume{\n\t\tName: volumeName,\n\t\tVolumeSource: kcore.VolumeSource{\n\t\t\tEmptyDir: &kcore.EmptyDirVolumeSource{\n\t\t\t\tMedium:    kcore.StorageMediumMemory,\n\t\t\t\tSizeLimit: k8s.QuantityPtr(q),\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc KubexitVolume() kcore.Volume {\n\treturn k8s.EmptyDirVolume(_kubexitGraveyardName)\n}\n\nfunc MntMount() kcore.VolumeMount {\n\treturn k8s.EmptyDirVolumeMount(_emptyDirVolumeName, _emptyDirMountPath)\n}\n\nfunc CortexMount() kcore.VolumeMount {\n\treturn k8s.EmptyDirVolumeMount(_cortexDirVolumeName, _cortexDirMountPath)\n}\n\nfunc APIConfigMount(name string) kcore.VolumeMount {\n\treturn kcore.VolumeMount{\n\t\tName:      name,\n\t\tMountPath: path.Join(_cortexDirMountPath, \"spec\"),\n\t}\n}\n\nfunc ClientConfigMount() kcore.VolumeMount {\n\treturn kcore.VolumeMount{\n\t\tName:      _clientConfigDirVolume,\n\t\tMountPath: path.Join(_clientConfigDir, \"cli.yaml\"),\n\t\tSubPath:   \"cli.yaml\",\n\t}\n}\n\nfunc ClusterConfigMount() kcore.VolumeMount {\n\treturn kcore.VolumeMount{\n\t\tName:      _clusterConfigDirVolume,\n\t\tMountPath: path.Join(_clusterConfigDir, \"cluster.yaml\"),\n\t\tSubPath:   \"cluster.yaml\",\n\t}\n}\n\nfunc ShmMount(volumeName string) kcore.VolumeMount {\n\treturn k8s.EmptyDirVolumeMount(volumeName, _shmDirMountPath)\n}\n\nfunc KubexitMount() kcore.VolumeMount {\n\treturn k8s.EmptyDirVolumeMount(_kubexitGraveyardName, _kubexitGraveyardMountPath)\n}\n"
  },
  {
    "path": "pkg/workloads/init.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage workloads\n\nimport (\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\tkcore \"k8s.io/api/core/v1\"\n)\n\nconst (\n\t_kubexitInitContainerName = \"kubexit\"\n)\n\nfunc KubexitInitContainer() kcore.Container {\n\treturn kcore.Container{\n\t\tName:            _kubexitInitContainerName,\n\t\tImage:           config.ClusterConfig.ImageKubexit,\n\t\tImagePullPolicy: kcore.PullAlways,\n\t\tCommand:         []string{\"cp\", \"/bin/kubexit\", \"/cortex/kubexit\"},\n\t\tVolumeMounts: []kcore.VolumeMount{\n\t\t\tCortexMount(),\n\t\t},\n\t}\n}\n"
  },
  {
    "path": "pkg/workloads/k8s.go",
    "content": "/*\nCopyright 2022 Cortex Labs, Inc.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage workloads\n\nimport (\n\t\"fmt\"\n\t\"path\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/cortexlabs/cortex/pkg/config\"\n\t\"github.com/cortexlabs/cortex/pkg/consts\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/k8s\"\n\t\"github.com/cortexlabs/cortex/pkg/lib/pointer\"\n\ts \"github.com/cortexlabs/cortex/pkg/lib/strings\"\n\t\"github.com/cortexlabs/cortex/pkg/types/clusterconfig\"\n\t\"github.com/cortexlabs/cortex/pkg/types/spec\"\n\t\"github.com/cortexlabs/cortex/pkg/types/userconfig\"\n\tkcore \"k8s.io/api/core/v1\"\n\tkresource \"k8s.io/apimachinery/pkg/api/resource\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n)\n\nconst (\n\tServiceAccountName = \"default\"\n)\n\nconst (\n\t_cortexDirVolumeName = \"cortex\"\n\t_cortexDirMountPath  = \"/cortex\"\n\t_clientConfigDir     = \"/cortex/client\"\n\n\t_emptyDirVolumeName = \"mnt\"\n\t_emptyDirMountPath  = \"/mnt\"\n\n\tProxyContainerName    = \"proxy\"\n\tDequeuerContainerName = \"dequeuer\"\n\tGatewayContainerName  = \"gateway\"\n\n\t_kubexitGraveyardName      = \"graveyard\"\n\t_kubexitGraveyardMountPath = \"/graveyard\"\n\n\t_shmDirMountPath = \"/dev/shm\"\n\n\t_clientConfigDirVolume = \"client-config\"\n\t_clientConfigConfigMap = \"client-config\"\n\n\t_clusterConfigDirVolume = \"cluster-config\"\n\t_clusterConfigConfigMap = \"cluster-config\"\n\t_clusterConfigDir       = \"/configs/cluster\"\n)\n\nvar (\n\t_statsdAddress = fmt.Sprintf(\"prometheus-statsd-exporter.%s:9125\", consts.PrometheusNamespace)\n\n\t// each Inferentia chip requires 128 HugePages with each HugePage having a size of 2Mi\n\t_hugePagesMemPerInf = int64(128 * 2 * 1024 * 1024) // bytes\n)\n\nfunc asyncDequeuerProxyContainer(api spec.API, queueURL string) (kcore.Container, kcore.Volume) {\n\treturn kcore.Container{\n\t\tName:            DequeuerContainerName,\n\t\tImage:           config.ClusterConfig.ImageDequeuer,\n\t\tImagePullPolicy: kcore.PullAlways,\n\t\tCommand: []string{\n\t\t\t\"/dequeuer\",\n\t\t},\n\t\tArgs: []string{\n\t\t\t\"--cluster-config\", consts.DefaultInClusterConfigPath,\n\t\t\t\"--cluster-uid\", config.ClusterConfig.ClusterUID,\n\t\t\t\"--probes-path\", path.Join(_cortexDirMountPath, \"spec\", \"probes.json\"),\n\t\t\t\"--queue\", queueURL,\n\t\t\t\"--api-kind\", api.Kind.String(),\n\t\t\t\"--api-name\", api.Name,\n\t\t\t\"--statsd-address\", _statsdAddress,\n\t\t\t\"--user-port\", s.Int32(*api.Pod.Port),\n\t\t\t\"--admin-port\", consts.AdminPortStr,\n\t\t\t\"--workers\", s.Int64(api.Pod.MaxConcurrency),\n\t\t},\n\t\tEnv:     BaseEnvVars,\n\t\tEnvFrom: BaseClusterEnvVars(),\n\t\tPorts: []kcore.ContainerPort{\n\t\t\t{\n\t\t\t\tName:          consts.AdminPortName,\n\t\t\t\tContainerPort: consts.AdminPortInt32,\n\t\t\t},\n\t\t},\n\t\tResources: kcore.ResourceRequirements{\n\t\t\tRequests: kcore.ResourceList{\n\t\t\t\tkcore.ResourceCPU:    consts.CortexDequeuerCPU,\n\t\t\t\tkcore.ResourceMemory: consts.CortexDequeuerMem,\n\t\t\t},\n\t\t},\n\t\tReadinessProbe: &kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tHTTPGet: &kcore.HTTPGetAction{\n\t\t\t\t\tPath: \"/healthz\",\n\t\t\t\t\tPort: intstr.FromInt(int(consts.AdminPortInt32)),\n\t\t\t\t},\n\t\t\t},\n\t\t\tInitialDelaySeconds: 1,\n\t\t\tTimeoutSeconds:      1,\n\t\t\tPeriodSeconds:       10,\n\t\t\tSuccessThreshold:    1,\n\t\t\tFailureThreshold:    1,\n\t\t},\n\t\tVolumeMounts: []kcore.VolumeMount{\n\t\t\tClusterConfigMount(),\n\t\t},\n\t}, ClusterConfigVolume()\n}\n\nfunc batchDequeuerProxyContainer(api spec.API, jobID, queueURL string) (kcore.Container, kcore.Volume) {\n\treturn kcore.Container{\n\t\tName:            DequeuerContainerName,\n\t\tImage:           config.ClusterConfig.ImageDequeuer,\n\t\tImagePullPolicy: kcore.PullAlways,\n\t\tCommand: []string{\n\t\t\t\"/dequeuer\",\n\t\t},\n\t\tArgs: []string{\n\t\t\t\"--cluster-config\", consts.DefaultInClusterConfigPath,\n\t\t\t\"--cluster-uid\", config.ClusterConfig.ClusterUID,\n\t\t\t\"--probes-path\", path.Join(_cortexDirMountPath, \"spec\", \"probes.json\"),\n\t\t\t\"--queue\", queueURL,\n\t\t\t\"--api-kind\", api.Kind.String(),\n\t\t\t\"--api-name\", api.Name,\n\t\t\t\"--job-id\", jobID,\n\t\t\t\"--statsd-address\", _statsdAddress,\n\t\t\t\"--user-port\", s.Int32(*api.Pod.Port),\n\t\t\t\"--admin-port\", consts.AdminPortStr,\n\t\t},\n\t\tEnv:     BaseEnvVars,\n\t\tEnvFrom: BaseClusterEnvVars(),\n\t\tResources: kcore.ResourceRequirements{\n\t\t\tRequests: kcore.ResourceList{\n\t\t\t\tkcore.ResourceCPU:    consts.CortexDequeuerCPU,\n\t\t\t\tkcore.ResourceMemory: consts.CortexDequeuerMem,\n\t\t\t},\n\t\t},\n\t\tReadinessProbe: &kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tHTTPGet: &kcore.HTTPGetAction{\n\t\t\t\t\tPath: \"/healthz\",\n\t\t\t\t\tPort: intstr.FromInt(int(consts.AdminPortInt32)),\n\t\t\t\t},\n\t\t\t},\n\t\t\tInitialDelaySeconds: 1,\n\t\t\tTimeoutSeconds:      1,\n\t\t\tPeriodSeconds:       10,\n\t\t\tSuccessThreshold:    1,\n\t\t\tFailureThreshold:    1,\n\t\t},\n\t\tVolumeMounts: []kcore.VolumeMount{\n\t\t\tClusterConfigMount(),\n\t\t\tCortexMount(),\n\t\t},\n\t}, ClusterConfigVolume()\n}\n\nfunc realtimeProxyContainer(api spec.API) (kcore.Container, kcore.Volume) {\n\tproxyHasTCPProbe := !HasReadinessProbesTargetingPort(api.Pod.Containers, *api.Pod.Port)\n\n\treturn kcore.Container{\n\t\tName:            ProxyContainerName,\n\t\tImage:           config.ClusterConfig.ImageProxy,\n\t\tImagePullPolicy: kcore.PullAlways,\n\t\tArgs: []string{\n\t\t\t\"--cluster-config\",\n\t\t\tconsts.DefaultInClusterConfigPath,\n\t\t\t\"--port\",\n\t\t\tconsts.ProxyPortStr,\n\t\t\t\"--admin-port\",\n\t\t\tconsts.AdminPortStr,\n\t\t\t\"--user-port\",\n\t\t\ts.Int32(*api.Pod.Port),\n\t\t\t\"--max-concurrency\",\n\t\t\ts.Int32(int32(api.Pod.MaxConcurrency)),\n\t\t\t\"--max-queue-length\",\n\t\t\ts.Int32(int32(api.Pod.MaxQueueLength)),\n\t\t\t\"--has-tcp-probe\",\n\t\t\ts.Bool(proxyHasTCPProbe),\n\t\t},\n\t\tPorts: []kcore.ContainerPort{\n\t\t\t{Name: consts.AdminPortName, ContainerPort: consts.AdminPortInt32},\n\t\t\t{ContainerPort: consts.ProxyPortInt32},\n\t\t},\n\t\tEnv:     BaseEnvVars,\n\t\tEnvFrom: BaseClusterEnvVars(),\n\t\tVolumeMounts: []kcore.VolumeMount{\n\t\t\tClusterConfigMount(),\n\t\t},\n\t\tResources: kcore.ResourceRequirements{\n\t\t\tRequests: kcore.ResourceList{\n\t\t\t\tkcore.ResourceCPU:    consts.CortexProxyCPU,\n\t\t\t\tkcore.ResourceMemory: consts.CortexProxyMem,\n\t\t\t},\n\t\t},\n\t\tReadinessProbe: &kcore.Probe{\n\t\t\tHandler: kcore.Handler{\n\t\t\t\tHTTPGet: &kcore.HTTPGetAction{\n\t\t\t\t\tPath: \"/healthz\",\n\t\t\t\t\tPort: intstr.FromInt(int(consts.AdminPortInt32)),\n\t\t\t\t},\n\t\t\t},\n\t\t\tInitialDelaySeconds: 1,\n\t\t\tTimeoutSeconds:      3,\n\t\t\tPeriodSeconds:       10,\n\t\t\tSuccessThreshold:    1,\n\t\t\tFailureThreshold:    3,\n\t\t},\n\t}, ClusterConfigVolume()\n}\n\nfunc RealtimeContainers(api spec.API) ([]kcore.Container, []kcore.Volume) {\n\tcontainers, volumes := userPodContainers(api)\n\tproxyContainer, proxyVolume := realtimeProxyContainer(api)\n\n\tcontainers = append(containers, proxyContainer)\n\tvolumes = append(volumes, proxyVolume)\n\n\treturn containers, volumes\n}\n\nfunc AsyncContainers(api spec.API, queueURL string) ([]kcore.Container, []kcore.Volume) {\n\tk8sName := K8sName(api.Name)\n\n\tcontainers, volumes := userPodContainers(api)\n\tdequeuerContainer, dequeuerVolume := asyncDequeuerProxyContainer(api, queueURL)\n\tdequeuerContainer.VolumeMounts = append(dequeuerContainer.VolumeMounts, APIConfigMount(k8sName))\n\n\tcontainers = append(containers, dequeuerContainer)\n\tvolumes = append(volumes, dequeuerVolume, APIConfigVolume(k8sName))\n\n\treturn containers, volumes\n}\n\nfunc TaskContainers(api spec.API, job *spec.JobKey) ([]kcore.Container, []kcore.Volume) {\n\tcontainers, volumes := userPodContainers(api)\n\tk8sName := job.K8sName()\n\n\tvolumes = append(volumes,\n\t\tKubexitVolume(),\n\t\tAPIConfigVolume(k8sName),\n\t)\n\n\tcontainerNames := userconfig.GetContainerNames(api.Pod.Containers)\n\tfor i, c := range containers {\n\t\tcontainers[i].VolumeMounts = append(containers[i].VolumeMounts,\n\t\t\tKubexitMount(),\n\t\t\tAPIConfigMount(k8sName),\n\t\t)\n\n\t\tcontainerDeathDependencies := containerNames.Copy()\n\t\tcontainerDeathDependencies.Remove(c.Name)\n\t\tcontainerDeathEnvVars := getKubexitEnvVars(c.Name, containerDeathDependencies.SliceSorted(), nil)\n\t\tcontainers[i].Env = append(containers[i].Env, containerDeathEnvVars...)\n\n\t\tif c.Command[0] != \"/cortex/kubexit\" {\n\t\t\tcontainers[i].Command = append([]string{\"/cortex/kubexit\"}, c.Command...)\n\t\t}\n\t}\n\n\treturn containers, volumes\n}\n\nfunc BatchContainers(api spec.API, job *spec.BatchJob) ([]kcore.Container, []kcore.Volume) {\n\tuserContainers, userVolumes := userPodContainers(api)\n\tdequeuerContainer, dequeuerVolume := batchDequeuerProxyContainer(api, job.ID, job.SQSUrl)\n\n\t// make sure the dequeuer starts first to allow it to start watching the graveyard before user containers begin\n\tcontainers := append([]kcore.Container{dequeuerContainer}, userContainers...)\n\tvolumes := append([]kcore.Volume{dequeuerVolume}, userVolumes...)\n\n\tk8sName := job.K8sName()\n\n\tvolumes = append(volumes,\n\t\tKubexitVolume(),\n\t\tAPIConfigVolume(k8sName),\n\t)\n\n\tcontainerNames := userconfig.GetContainerNames(api.Pod.Containers)\n\tcontainerNames.Add(dequeuerContainer.Name)\n\n\tfor i, c := range containers {\n\t\tcontainers[i].VolumeMounts = append(containers[i].VolumeMounts,\n\t\t\tKubexitMount(),\n\t\t\tAPIConfigMount(k8sName),\n\t\t)\n\n\t\tcontainerDeathDependencies := containerNames.Copy()\n\t\tcontainerDeathDependencies.Remove(c.Name)\n\t\tcontainerDeathEnvVars := getKubexitEnvVars(c.Name, containerDeathDependencies.SliceSorted(), nil)\n\t\tcontainers[i].Env = append(containers[i].Env, containerDeathEnvVars...)\n\n\t\tif c.Command[0] != \"/cortex/kubexit\" {\n\t\t\tcontainers[i].Command = append([]string{\"/cortex/kubexit\"}, c.Command...)\n\t\t}\n\t}\n\n\treturn containers, volumes\n}\n\nfunc userPodContainers(api spec.API) ([]kcore.Container, []kcore.Volume) {\n\tvolumes := []kcore.Volume{\n\t\tMntVolume(),\n\t\tCortexVolume(),\n\t\tClientConfigVolume(),\n\t}\n\tcontainerMounts := []kcore.VolumeMount{\n\t\tMntMount(),\n\t\tCortexMount(),\n\t\tClientConfigMount(),\n\t}\n\n\tcontainers := make([]kcore.Container, len(api.Pod.Containers))\n\tfor i, container := range api.Pod.Containers {\n\t\tcontainerResourceList := kcore.ResourceList{}\n\t\tcontainerResourceLimitsList := kcore.ResourceList{}\n\t\tsecurityContext := kcore.SecurityContext{\n\t\t\tPrivileged: pointer.Bool(true),\n\t\t}\n\n\t\tvar readinessProbe *kcore.Probe\n\t\tif api.Kind == userconfig.RealtimeAPIKind {\n\t\t\treadinessProbe = GetProbeSpec(container.ReadinessProbe)\n\t\t}\n\n\t\tif container.Compute.CPU != nil {\n\t\t\tcontainerResourceList[kcore.ResourceCPU] = *k8s.QuantityPtr(container.Compute.CPU.Quantity.DeepCopy())\n\t\t}\n\n\t\tif container.Compute.Mem != nil {\n\t\t\tcontainerResourceList[kcore.ResourceMemory] = *k8s.QuantityPtr(container.Compute.Mem.Quantity.DeepCopy())\n\t\t}\n\n\t\tif container.Compute.GPU > 0 {\n\t\t\tcontainerResourceList[\"nvidia.com/gpu\"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI)\n\t\t\tcontainerResourceLimitsList[\"nvidia.com/gpu\"] = *kresource.NewQuantity(container.Compute.GPU, kresource.DecimalSI)\n\t\t}\n\n\t\tif container.Compute.Inf > 0 {\n\t\t\ttotalHugePages := container.Compute.Inf * _hugePagesMemPerInf\n\t\t\tcontainerResourceList[\"aws.amazon.com/neuron\"] = *kresource.NewQuantity(container.Compute.Inf, kresource.DecimalSI)\n\t\t\tcontainerResourceList[\"hugepages-2Mi\"] = *kresource.NewQuantity(totalHugePages, kresource.BinarySI)\n\t\t\tcontainerResourceLimitsList[\"aws.amazon.com/neuron\"] = *kresource.NewQuantity(container.Compute.Inf, kresource.DecimalSI)\n\t\t\tcontainerResourceLimitsList[\"hugepages-2Mi\"] = *kresource.NewQuantity(totalHugePages, kresource.BinarySI)\n\n\t\t\tsecurityContext.Capabilities = &kcore.Capabilities{\n\t\t\t\tAdd: []kcore.Capability{\n\t\t\t\t\t\"SYS_ADMIN\",\n\t\t\t\t\t\"IPC_LOCK\",\n\t\t\t\t},\n\t\t\t}\n\t\t}\n\n\t\tif container.Compute.Shm != nil {\n\t\t\tvolumes = append(volumes, ShmVolume(container.Compute.Shm.Quantity, \"dshm-\"+container.Name))\n\t\t\tcontainerMounts = append(containerMounts, ShmMount(\"dshm-\"+container.Name))\n\t\t}\n\n\t\tcontainerEnvVars := BaseEnvVars\n\n\t\tcontainerEnvVars = append(containerEnvVars, kcore.EnvVar{\n\t\t\tName:  \"CORTEX_CLI_CONFIG_DIR\",\n\t\t\tValue: _clientConfigDir,\n\t\t})\n\n\t\tif api.Kind != userconfig.TaskAPIKind {\n\t\t\tcontainerEnvVars = append(containerEnvVars, kcore.EnvVar{\n\t\t\t\tName:  \"CORTEX_PORT\",\n\t\t\t\tValue: s.Int32(*api.Pod.Port),\n\t\t\t})\n\t\t}\n\n\t\tenvVarNames := make([]string, 0, len(container.Env))\n\t\tfor envVarName := range container.Env {\n\t\t\tenvVarNames = append(envVarNames, envVarName)\n\t\t}\n\n\t\t// k8s deployments will replace pods if env vars are re-ordered\n\t\tsort.Strings(envVarNames)\n\n\t\tfor _, envVarName := range envVarNames {\n\t\t\tcontainerEnvVars = append(containerEnvVars, kcore.EnvVar{\n\t\t\t\tName:  envVarName,\n\t\t\t\tValue: container.Env[envVarName],\n\t\t\t})\n\t\t}\n\n\t\tcontainers[i] = kcore.Container{\n\t\t\tName:           container.Name,\n\t\t\tImage:          container.Image,\n\t\t\tCommand:        container.Command,\n\t\t\tArgs:           container.Args,\n\t\t\tEnv:            containerEnvVars,\n\t\t\tVolumeMounts:   containerMounts,\n\t\t\tLivenessProbe:  GetProbeSpec(container.LivenessProbe),\n\t\t\tReadinessProbe: readinessProbe,\n\t\t\tLifecycle:      GetLifecycleSpec(container.PreStop),\n\t\t\tResources: kcore.ResourceRequirements{\n\t\t\t\tRequests: containerResourceList,\n\t\t\t\tLimits:   containerResourceLimitsList,\n\t\t\t},\n\t\t\tImagePullPolicy: kcore.PullAlways,\n\t\t\tSecurityContext: &securityContext,\n\t\t}\n\t}\n\n\treturn containers, volumes\n}\n\nfunc NodeSelectors() map[string]string {\n\treturn map[string]string{\n\t\t\"workload\": \"true\",\n\t}\n}\n\nfunc GenerateResourceTolerations() []kcore.Toleration {\n\ttolerations := []kcore.Toleration{\n\t\t{\n\t\t\tKey:      \"workload\",\n\t\t\tOperator: kcore.TolerationOpEqual,\n\t\t\tValue:    \"true\",\n\t\t\tEffect:   kcore.TaintEffectNoSchedule,\n\t\t},\n\t\t{\n\t\t\tKey:      \"nvidia.com/gpu\",\n\t\t\tOperator: kcore.TolerationOpExists,\n\t\t\tEffect:   kcore.TaintEffectNoSchedule,\n\t\t},\n\t\t{\n\t\t\tKey:      \"aws.amazon.com/neuron\",\n\t\t\tOperator: kcore.TolerationOpEqual,\n\t\t\tValue:    \"true\",\n\t\t\tEffect:   kcore.TaintEffectNoSchedule,\n\t\t},\n\t}\n\n\treturn tolerations\n}\n\nfunc GenerateNodeAffinities(apiNodeGroups []string) *kcore.Affinity {\n\tvar nodeGroups []*clusterconfig.NodeGroup\n\tfor _, clusterNodeGroup := range config.ClusterConfig.NodeGroups {\n\t\tfor _, apiNodeGroupName := range apiNodeGroups {\n\t\t\tif clusterNodeGroup.Name == apiNodeGroupName {\n\t\t\t\tnodeGroups = append(nodeGroups, clusterNodeGroup)\n\t\t\t}\n\t\t}\n\t}\n\n\tif apiNodeGroups == nil {\n\t\tnodeGroups = config.ClusterConfig.NodeGroups\n\t}\n\n\trequiredNodeGroups := make([]string, len(nodeGroups))\n\tpreferredAffinities := make([]kcore.PreferredSchedulingTerm, len(nodeGroups))\n\tfor i, nodeGroup := range nodeGroups {\n\t\tvar nodeGroupPrefix string\n\t\tif nodeGroup.Spot {\n\t\t\tnodeGroupPrefix = \"cx-ws-\"\n\t\t} else {\n\t\t\tnodeGroupPrefix = \"cx-wd-\"\n\t\t}\n\n\t\tpreferredAffinities[i] = kcore.PreferredSchedulingTerm{\n\t\t\tWeight: int32(nodeGroup.Priority),\n\t\t\tPreference: kcore.NodeSelectorTerm{\n\t\t\t\tMatchExpressions: []kcore.NodeSelectorRequirement{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:      \"alpha.eksctl.io/nodegroup-name\",\n\t\t\t\t\t\tOperator: kcore.NodeSelectorOpIn,\n\t\t\t\t\t\tValues:   []string{nodeGroupPrefix + nodeGroup.Name},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\n\t\trequiredNodeGroups[i] = nodeGroupPrefix + nodeGroup.Name\n\t}\n\n\tvar requiredNodeSelector *kcore.NodeSelector\n\tif apiNodeGroups != nil {\n\t\trequiredNodeSelector = &kcore.NodeSelector{\n\t\t\tNodeSelectorTerms: []kcore.NodeSelectorTerm{\n\t\t\t\t{\n\t\t\t\t\tMatchExpressions: []kcore.NodeSelectorRequirement{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tKey:      \"alpha.eksctl.io/nodegroup-name\",\n\t\t\t\t\t\t\tOperator: kcore.NodeSelectorOpIn,\n\t\t\t\t\t\t\tValues:   requiredNodeGroups,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t}\n\n\treturn &kcore.Affinity{\n\t\tNodeAffinity: &kcore.NodeAffinity{\n\t\t\tPreferredDuringSchedulingIgnoredDuringExecution: preferredAffinities,\n\t\t\tRequiredDuringSchedulingIgnoredDuringExecution:  requiredNodeSelector,\n\t\t},\n\t}\n}\n\nvar BaseEnvVars = []kcore.EnvVar{\n\t{\n\t\tName:  \"CORTEX_VERSION\",\n\t\tValue: consts.CortexVersion,\n\t},\n\t{\n\t\tName:  \"CORTEX_LOG_LEVEL\",\n\t\tValue: strings.ToUpper(userconfig.InfoLogLevel.String()),\n\t},\n}\n"
  },
  {
    "path": "python/client/README.md",
    "content": "Cost-effective serverless computing - [docs.cortexlabs.com](https://www.docs.cortexlabs.com)\n"
  },
  {
    "path": "python/client/cortex/__init__.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport json\nfrom typing import Optional, List\n\nfrom cortex.binary import run_cli\nfrom cortex.client import Client\nfrom cortex.exceptions import NotFound\nfrom cortex.telemetry import sentry_wrapper\n\n\n__version__ = \"master\"  # CORTEX_VERSION\n\n\n@sentry_wrapper\ndef client(env_name: Optional[str] = None) -> Client:\n    \"\"\"\n    Initialize a client based on the specified environment. If no environment is specified, it will attempt to use the default environment.\n\n    Args:\n        env_name: Name of the environment to use.\n\n    Returns:\n        Cortex client that can be used to deploy and manage APIs in the specified environment.\n    \"\"\"\n    environments = env_list()\n\n    if env_name is None:\n        if not environments.get(\"default_environment\"):\n            raise NotFound(\"no default environment configured\")\n        env_name = environments[\"default_environment\"]\n\n    found = False\n    for environment in environments[\"environments\"]:\n        if environment[\"name\"] == env_name:\n            found = True\n            break\n    if not found:\n        raise NotFound(\n            f\"can't find environment {env_name}, create one by calling `cortex.new_client()`\"\n        )\n\n    return Client(environment)\n\n\n@sentry_wrapper\ndef new_client(\n    env_name: str,\n    operator_endpoint: str,\n) -> Client:\n    \"\"\"\n    Create a new environment to connect to an existing cluster, and initialize a client to deploy and manage APIs on that cluster.\n\n    Args:\n        env_name: Name of the environment to create.\n        operator_endpoint: The endpoint for the operator of your Cortex cluster. You can get this endpoint by running the CLI command `cortex cluster info`.\n\n    Returns:\n        Cortex client that can be used to deploy and manage APIs on a cluster.\n    \"\"\"\n    cli_args = [\n        \"env\",\n        \"configure\",\n        env_name,\n        \"--operator-endpoint\",\n        operator_endpoint,\n    ]\n\n    run_cli(cli_args, hide_output=True)\n\n    return client(env_name)\n\n\n@sentry_wrapper\ndef env_list() -> List:\n    \"\"\"\n    List all environments configured on this machine.\n    \"\"\"\n    output = run_cli([\"env\", \"list\", \"--output\", \"json\"], hide_output=True)\n    return json.loads(output.strip())\n\n\n@sentry_wrapper\ndef env_delete(name: str):\n    \"\"\"\n    Delete an environment configured on this machine.\n\n    Args:\n        name: Name of the environment to delete.\n    \"\"\"\n    run_cli([\"env\", \"delete\", name], hide_output=True)\n"
  },
  {
    "path": "python/client/cortex/binary/__init__.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport subprocess\nimport sys\nfrom typing import List\n\nfrom cortex.exceptions import CortexBinaryException\n\n\ndef run():\n    \"\"\"\n    Runs the CLI from terminal.\n    \"\"\"\n    try:\n        env = os.environ.copy()\n        if \"AWS_VPC_K8S_CNI_LOG_FILE\" not in env:\n            env[\"AWS_VPC_K8S_CNI_LOG_FILE\"] = \"/dev/null\"\n        process = subprocess.run([get_cli_path()] + sys.argv[1:], cwd=os.getcwd(), env=env)\n    except KeyboardInterrupt:\n        sys.exit(130)  # Ctrl + C\n    sys.exit(process.returncode)\n\n\ndef run_cli(\n    args: List[str],\n    hide_output: bool = False,\n) -> str:\n    \"\"\"\n    Runs the Cortex binary with the specified arguments.\n\n    Args:\n        args: Arguments to use when invoking the Cortex CLI.\n        hide_output: Flag to prevent streaming CLI output to stdout.\n\n    Raises:\n        CortexBinaryException: Cortex CLI command returned an error.\n\n    Returns:\n        The stdout from the Cortex CLI command.\n    \"\"\"\n\n    env = os.environ.copy()\n    env[\"CORTEX_CLI_INVOKER\"] = \"python\"\n    if \"AWS_VPC_K8S_CNI_LOG_FILE\" not in env:\n        env[\"AWS_VPC_K8S_CNI_LOG_FILE\"] = \"/dev/null\"\n    process = subprocess.Popen(\n        [get_cli_path()] + args,\n        stderr=subprocess.PIPE,\n        stdout=subprocess.PIPE,\n        encoding=\"utf8\",\n        env=env,\n    )\n\n    output = \"\"\n    result = \"\"\n\n    for c in iter(lambda: process.stdout.read(1), \"\"):\n        output += c\n\n        if not hide_output:\n            sys.stdout.write(c)\n            sys.stdout.flush()\n\n    process.wait()\n\n    if process.returncode == 0:\n        return output\n\n    if result != \"\":\n        raise CortexBinaryException(result + \"\\n\" + process.stderr.read())\n\n    raise CortexBinaryException(process.stderr.read())\n\n\ndef get_cli_path() -> str:\n    \"\"\"\n    Get the location of the CLI.\n\n    Default location is the directory containing the `cortex.binary` package.\n    The location can be overridden by setting the `CORTEX_CLI_PATH` environment variable.\n\n    Raises:\n        Exception: Unable to find the CLI.\n\n    Returns:\n        str: The location of the CLI in the local filesystem.\n    \"\"\"\n    if os.environ.get(\"CORTEX_CLI_PATH\") is not None:\n        cli_path = os.environ[\"CORTEX_CLI_PATH\"]\n        if not os.path.exists(cli_path):\n            raise Exception(\n                f\"unable to find cortex binary at {cli_path} as specified in `CORTEX_CLI_PATH` environment variable\"\n            )\n        return cli_path\n\n    try:\n        import importlib.resources as pkg_resources\n    except ImportError:\n        # Try backported to PY<37 `importlib_resources`.\n        import importlib_resources as pkg_resources\n\n    from cortex import binary\n\n    try:\n        with pkg_resources.path(binary, \"cli\") as p:\n            cli_path = p\n    except FileNotFoundError as e:\n        raise Exception(\n            \"unable to find cortex binary, please reinstall the cortex client by running `pip uninstall cortex` and then `pip install cortex`\"\n        ) from e\n\n    return str(cli_path)\n"
  },
  {
    "path": "python/client/cortex/client.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport json\nimport os\nimport shutil\nimport time\nimport yaml\nfrom typing import List, Dict, Any\n\nfrom cortex import util\nfrom cortex.binary import run_cli\nfrom cortex.telemetry import sentry_wrapper\n\n\nclass Client:\n    @sentry_wrapper\n    def __init__(self, env_config: Dict):\n        \"\"\"\n        A client to deploy and manage APIs in the specified environment.\n        This constructor is not meant to be invoked directly.\n        Use `cortex.client()` and `cortex.new_client()` to initialize a new cortex client.\n\n        Args:\n            env_config: Environment config\n        \"\"\"\n\n        self.env = env_config\n        self.env_name = env_config[\"name\"]\n\n    # CORTEX_VERSION_MINOR\n    @sentry_wrapper\n    def deploy(\n        self,\n        api_spec: Dict[str, Any],\n        force: bool = True,\n        wait: bool = False,\n    ):\n        \"\"\"\n        Deploy or update an API.\n\n        Args:\n            api_spec: A dictionary defining a single Cortex API. See https://docs.cortexlabs.com/v/master/ for schema.\n            force: Override any in-progress api updates.\n            wait: Block until the API is ready.\n\n        Returns:\n            Deployment status, API specification, and endpoint for each API.\n        \"\"\"\n\n        temp_deploy_dir = util.cli_config_dir() / \"deployments\" / api_spec[\"name\"]\n        if temp_deploy_dir.exists():\n            shutil.rmtree(str(temp_deploy_dir))\n        temp_deploy_dir.mkdir(parents=True)\n\n        cortex_yaml_path = os.path.join(temp_deploy_dir, \"cortex.yaml\")\n\n        with util.open_temporarily(cortex_yaml_path, \"w\", delete_parent_if_empty=True) as f:\n            yaml.dump([api_spec], f)  # write a list\n            return self.deploy_from_file(cortex_yaml_path, force=force, wait=wait)\n\n    # CORTEX_VERSION_MINOR\n    @sentry_wrapper\n    def deploy_from_file(\n        self,\n        config_file: str,\n        force: bool = False,\n        wait: bool = False,\n    ) -> Dict:\n        \"\"\"\n        Deploy or update APIs specified in a configuration file.\n\n        Args:\n            config_file: Local path to a yaml file defining Cortex API(s). See https://docs.cortexlabs.com/v/master/ for schema.\n            force: Override any in-progress api updates.\n            wait: Block until the API is ready.\n\n        Returns:\n            Deployment status, API specification, and endpoint for each API.\n        \"\"\"\n\n        args = [\n            \"deploy\",\n            config_file,\n            \"--env\",\n            self.env_name,\n            \"-o\",\n            \"json\",\n            \"-y\",\n        ]\n\n        if force:\n            args.append(\"--force\")\n\n        output = run_cli(args, hide_output=True)\n\n        deploy_results = json.loads(output.strip())\n\n        deploy_result = deploy_results[0]\n\n        if not wait:\n            return deploy_result\n\n        api_name = deploy_result[\"api\"][\"spec\"][\"name\"]\n        if (\n            deploy_result[\"api\"][\"spec\"][\"kind\"] != \"RealtimeAPI\"\n            and deploy_result[\"api\"][\"spec\"][\"kind\"] != \"AsyncAPI\"\n        ):\n            return deploy_result\n\n        while True:\n            time.sleep(5)\n            api = self.get_api(api_name)\n            if api[\"status\"][\"status_code\"] != \"status_updating\":\n                break\n\n        return api\n\n    @sentry_wrapper\n    def get_api(self, api_name: str) -> Dict:\n        \"\"\"\n        Get information about an API.\n\n        Args:\n            api_name: Name of the API.\n\n        Returns:\n            Information about the API, including the API specification, endpoint, status, and metrics (if applicable).\n        \"\"\"\n        output = run_cli([\"get\", api_name, \"--env\", self.env_name, \"-o\", \"json\"], hide_output=True)\n\n        apis = json.loads(output.strip())\n        return apis[0]\n\n    @sentry_wrapper\n    def list_apis(self) -> List:\n        \"\"\"\n        List all APIs in the environment.\n\n        Returns:\n            List of APIs, including information such as the API specification, endpoint, status, and metrics (if applicable).\n        \"\"\"\n        args = [\"get\", \"-o\", \"json\", \"--env\", self.env_name]\n\n        output = run_cli(args, hide_output=True)\n\n        return json.loads(output.strip())\n\n    @sentry_wrapper\n    def get_job(self, api_name: str, job_id: str) -> Dict:\n        \"\"\"\n        Get information about a submitted job.\n\n        Args:\n            api_name: Name of the Batch/Task API.\n            job_id: Job ID.\n\n        Returns:\n            Information about the job, including the job status, worker status, and job progress.\n        \"\"\"\n        args = [\"get\", api_name, job_id, \"--env\", self.env_name, \"-o\", \"json\"]\n\n        output = run_cli(args, hide_output=True)\n\n        return json.loads(output.strip())\n\n    @sentry_wrapper\n    def refresh(self, api_name: str, force: bool = False):\n        \"\"\"\n        Restart all of the replicas for a Realtime API without downtime.\n\n        Args:\n            api_name: Name of the API to refresh.\n            force: Override an already in-progress API update.\n        \"\"\"\n        args = [\"refresh\", api_name, \"--env\", self.env_name, \"-o\", \"json\"]\n\n        if force:\n            args.append(\"--force\")\n\n        run_cli(args, hide_output=True)\n\n    @sentry_wrapper\n    def delete(self, api_name: str, keep_cache: bool = False):\n        \"\"\"\n        Delete an API.\n\n        Args:\n            api_name: Name of the API to delete.\n            keep_cache: Whether to retain the cached data for this API.\n        \"\"\"\n        args = [\n            \"delete\",\n            api_name,\n            \"--env\",\n            self.env_name,\n            \"--force\",\n            \"-o\",\n            \"json\",\n        ]\n\n        if keep_cache:\n            args.append(\"--keep-cache\")\n\n        run_cli(args, hide_output=True)\n\n    @sentry_wrapper\n    def stop_job(self, api_name: str, job_id: str, keep_cache: bool = False):\n        \"\"\"\n        Stop a running job.\n\n        Args:\n            api_name: Name of the Batch/Task API.\n            job_id: ID of the Job to stop.\n        \"\"\"\n        args = [\n            \"delete\",\n            api_name,\n            job_id,\n            \"--env\",\n            self.env_name,\n            \"-o\",\n            \"json\",\n        ]\n\n        run_cli(args)\n"
  },
  {
    "path": "python/client/cortex/consts.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nCORTEX_VERSION = \"master\"  # CORTEX_VERSION\nCORTEX_TELEMETRY_SENTRY_DSN = \"https://5cea3d2d67194d028f7191fcc6ebca14@sentry.io/1825326\"\nCORTEX_TELEMETRY_SENTRY_ENVIRONMENT = \"client\"\n"
  },
  {
    "path": "python/client/cortex/exceptions.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nclass CortexException(Exception):\n    \"\"\"\n    Base class for all Cortex's errors. Each custom exception should be derived from this class.\n    \"\"\"\n\n    pass\n\n\nclass CortexBinaryException(CortexException):\n    \"\"\"\n    Raise when binary execution returns an unexpected non-zero return code.\n    \"\"\"\n\n    pass\n\n\nclass NotFound(CortexException):\n    \"\"\"\n    Raise when the specified resource or name is not found.\n    \"\"\"\n\n    pass\n"
  },
  {
    "path": "python/client/cortex/telemetry.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nfrom typing import Dict\nfrom uuid import uuid4\n\nimport sentry_sdk\nfrom cortex import util\nfrom cortex.consts import (\n    CORTEX_VERSION,\n    CORTEX_TELEMETRY_SENTRY_DSN,\n    CORTEX_TELEMETRY_SENTRY_ENVIRONMENT,\n)\nfrom cortex.exceptions import CortexBinaryException\nfrom sentry_sdk.integrations.dedupe import DedupeIntegration\nfrom sentry_sdk.integrations.modules import ModulesIntegration\nfrom sentry_sdk.integrations.stdlib import StdlibIntegration\n\n\ndef _sentry_client(\n    disabled: bool = False,\n) -> sentry_sdk.Client:\n    \"\"\"\n    Initialize sentry. You can override the default values with the following env vars:\n    1. CORTEX_TELEMETRY_SENTRY_DSN\n    2. CORTEX_TELEMETRY_SENTRY_ENVIRONMENT\n    3. CORTEX_TELEMETRY_DISABLE\n    \"\"\"\n\n    dsn = CORTEX_TELEMETRY_SENTRY_DSN\n    environment = CORTEX_TELEMETRY_SENTRY_ENVIRONMENT\n\n    if disabled or os.getenv(\"CORTEX_TELEMETRY_DISABLE\", \"\").lower() == \"true\":\n        return\n\n    if os.getenv(\"CORTEX_TELEMETRY_SENTRY_DSN\", \"\") != \"\":\n        dsn = os.environ[\"CORTEX_TELEMETRY_SENTRY_DSN\"]\n\n    if os.getenv(\"CORTEX_TELEMETRY_SENTRY_ENVIRONMENT\", \"\") != \"\":\n        environment = os.environ[\"CORTEX_TELEMETRY_SENTRY_ENVIRONMENT\"]\n\n    client = sentry_sdk.Client(\n        dsn=dsn,\n        environment=environment,\n        release=CORTEX_VERSION,\n        ignore_errors=[CortexBinaryException],  # exclude CortexBinaryException exceptions\n        in_app_include=[\"cortex\"],  # for better grouping of events in sentry\n        attach_stacktrace=True,\n        default_integrations=False,  # disable all default integrations\n        auto_enabling_integrations=False,\n        integrations=[\n            DedupeIntegration(),  # prevent duplication of events\n            StdlibIntegration(),  # adds breadcrumbs (aka more info)\n            ModulesIntegration(),  # adds info about installed modules\n        ],\n        # debug=True,\n    )\n\n    return client\n\n\ndef _create_default_scope(optional_tags: Dict = {}) -> sentry_sdk.Scope:\n    \"\"\"\n    Creates default scope. Adds user ID as tag to the reported event.\n    Can add optional tags.\n    \"\"\"\n\n    scope = sentry_sdk.Scope()\n\n    user_id = None\n    client_id_file_path = util.cli_config_dir() / \"client-id.txt\"\n    if not client_id_file_path.is_file():\n        client_id_file_path.parent.mkdir(parents=True, exist_ok=True)\n        client_id_file_path.write_text(str(uuid4()))\n    user_id = client_id_file_path.read_text()\n\n    if user_id:\n        scope.set_user({\"id\": user_id})\n\n    for k, v in optional_tags.items():\n        scope.set_tag(k, v)\n\n    return scope\n\n\n# only one instance of this is required\nhub = sentry_sdk.Hub(_sentry_client(), _create_default_scope())\n\n\ndef sentry_wrapper(func):\n    def wrapper(*args, **kwargs):\n        with hub:\n            try:\n                return func(*args, **kwargs)\n            except:\n                sentry_sdk.capture_exception()\n                sentry_sdk.flush()\n                raise\n\n    return wrapper\n"
  },
  {
    "path": "python/client/cortex/util.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport shutil\nfrom contextlib import contextmanager\nfrom pathlib import Path\n\n\ndef cli_config_dir() -> Path:\n    cli_config_dir = os.environ.get(\"CORTEX_CLI_CONFIG_DIR\", \"\")\n    if cli_config_dir == \"\":\n        return Path.home() / \".cortex\"\n    return Path(cli_config_dir).expanduser().resolve()\n\n\n@contextmanager\ndef open_temporarily(path, mode, delete_parent_if_empty: bool = False):\n    parentDir = Path(path).parent\n    parentDir.mkdir(parents=True, exist_ok=True)\n\n    file = open(path, mode)\n\n    try:\n        yield file\n    finally:\n        file.close()\n        os.remove(path)\n        if delete_parent_if_empty and len(os.listdir(str(parentDir))) == 0:\n            shutil.rmtree(str(parentDir))\n\n\n@contextmanager\ndef open_tempdir(dir_path):\n    Path(dir_path).mkdir(parents=True)\n\n    try:\n        yield dir_path\n    finally:\n        shutil.rmtree(dir_path)\n"
  },
  {
    "path": "python/client/setup.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport pathlib\n\nfrom setuptools import setup, find_packages\nfrom setuptools.command.install import install\n\n\nclass InstallBinary(install):\n    def run(self):\n        install.run(self)\n\n        import requests\n        from pathlib import Path\n        import sys\n        import os\n        import stat\n        import shutil\n\n        dest_dir = os.path.join(self.install_lib, \"cortex\", \"binary\")\n\n        zip_file_path = os.path.join(dest_dir, \"cli.zip\")\n        cli_file_path = os.path.join(dest_dir, \"cli\")\n\n        if not os.path.exists(cli_file_path):\n            platform = sys.platform\n\n            # recommended way to check platform: https://docs.python.org/3/library/sys.html#sys.platform\n            if sys.platform.startswith(\"darwin\"):\n                platform = \"darwin\"\n            if sys.platform.startswith(\"linux\"):\n                platform = \"linux\"\n\n            if platform != \"darwin\" and platform != \"linux\":\n                raise Exception(\n                    f\"platform {platform} is not supported; cortex is only supported on mac and linux\"\n                )\n\n            cortex_version = self.config_vars[\"dist_version\"]\n\n            if \"dev\" in cortex_version:\n                cortex_version = \"master\"\n\n            download_url = f\"https://s3-us-west-2.amazonaws.com/get-cortex/{cortex_version}/cli/{platform}/cortex.zip\"\n\n            print(\"downloading cortex cli...\")\n            with requests.get(download_url, stream=True) as r:\n                with open(zip_file_path, \"wb\") as f:\n                    shutil.copyfileobj(r.raw, f)\n\n            zip_dir = os.path.join(dest_dir, \"cli_dir\")\n\n            print(\"unzipping cortex cli...\")\n            shutil.unpack_archive(zip_file_path, zip_dir)\n            shutil.move(os.path.join(zip_dir, \"cortex\"), cli_file_path)\n            shutil.rmtree(zip_dir)\n            os.remove(zip_file_path)\n\n            f = Path(cli_file_path)\n            f.chmod(f.stat().st_mode | stat.S_IEXEC)\n\n\nlong_description = \"\"\nif pathlib.Path(\"README.md\").is_file():\n    with open(\"README.md\") as f:\n        long_description = f.read()\n\nsetup(\n    name=\"cortex\",\n    version=\"master\",  # CORTEX_VERSION\n    description=\"Cost-effective serverless computing\",\n    author=\"cortexlabs.com\",\n    author_email=\"dev@cortexlabs.com\",\n    license=\"Apache License 2.0\",\n    long_description_content_type=\"text/markdown\",\n    long_description=long_description,\n    url=\"https://www.cortexlabs.com\",\n    setup_requires=([\"setuptools\", \"requests\", \"wheel\"]),\n    packages=find_packages(),\n    package_data={\"cortex.binary\": [\"cli\"]},\n    entry_points={\n        \"console_scripts\": [\n            \"cortex = cortex.binary:run\",\n        ],\n    },\n    install_requires=(\n        [\n            \"importlib-resources; python_version < '3.7'\",\n            \"pyyaml>=5.4.1\",\n            \"sentry-sdk>=1.4.3\",\n        ]\n    ),\n    python_requires=\">=3.6\",\n    cmdclass={\n        \"install\": InstallBinary,\n    },\n    classifiers=[\n        \"Operating System :: MacOS\",\n        \"Operating System :: POSIX :: Linux\",\n        \"Programming Language :: Python :: 3.6\",\n        \"Programming Language :: Python :: 3.7\",\n        \"Programming Language :: Python :: 3.8\",\n        \"Intended Audience :: Developers\",\n    ],\n    project_urls={\n        \"Bug Reports\": \"https://github.com/cortexlabs/cortex/issues\",\n        \"Community\": \"https://gitter.im/cortexlabs/cortex\",\n        \"Docs\": \"https://docs.cortexlabs.com\",\n        \"Source Code\": \"https://github.com/cortexlabs/cortex\",\n    },\n)\n"
  },
  {
    "path": "test/README.md",
    "content": "# Cortex Tests\n\n- [Example APIs](apis)\n- [End-to-end Tests](e2e)\n- [Testing Utilities](utils)\n"
  },
  {
    "path": "test/apis/async/hello-world/app/main.py",
    "content": "import os\n\nfrom fastapi import FastAPI\nfrom fastapi.responses import PlainTextResponse\n\napp = FastAPI()\napp.response_str = os.getenv(\"RESPONSE\", \"hello world\")\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    return PlainTextResponse(\"ok\")\n\n\n@app.post(\"/\")\ndef handler():\n    return {\"message\": app.response_str}\n"
  },
  {
    "path": "test/apis/async/hello-world/app/requirements.txt",
    "content": "uvicorn[standard]\nfastapi\n"
  },
  {
    "path": "test/apis/async/hello-world/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"async-hello-world-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/async/hello-world/cortex_cpu.yaml",
    "content": "- name: hello-world\n  kind: AsyncAPI\n  pod:\n    port: 8080\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/async-hello-world-cpu:latest\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 128Mi\n"
  },
  {
    "path": "test/apis/async/hello-world/cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/async/text-generator/app/main.py",
    "content": "import os\n\nfrom fastapi import FastAPI, status\nfrom fastapi.responses import PlainTextResponse\nfrom pydantic import BaseModel\nfrom transformers import GPT2Tokenizer, GPT2LMHeadModel\n\napp = FastAPI()\napp.device = os.getenv(\"TARGET_DEVICE\", \"cpu\")\napp.ready = False\n\n\n@app.on_event(\"startup\")\ndef startup():\n    app.tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n    app.model = GPT2LMHeadModel.from_pretrained(\"gpt2\").to(app.device)\n    app.ready = True\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    if app.ready:\n        return PlainTextResponse(\"ok\")\n    return PlainTextResponse(\"service unavailable\", status_code=status.HTTP_503_SERVICE_UNAVAILABLE)\n\n\nclass Body(BaseModel):\n    text: str\n\n\n@app.post(\"/\")\ndef text_generator(body: Body):\n    input_length = len(body.text.split())\n    tokens = app.tokenizer.encode(body.text, return_tensors=\"pt\").to(app.device)\n    prediction = app.model.generate(tokens, max_length=input_length + 20, do_sample=True)\n    return {\"text\": app.tokenizer.decode(prediction[0])}\n"
  },
  {
    "path": "test/apis/async/text-generator/app/requirements-cpu.txt",
    "content": "uvicorn[standard]\nfastapi\ntransformers==3.0.*\n-f https://download.pytorch.org/whl/torch_stable.html\ntorch==1.7.1+cpu\n"
  },
  {
    "path": "test/apis/async/text-generator/app/requirements-gpu.txt",
    "content": "uvicorn[standard]==0.16.0\nsentencepiece==0.1.94\nfastapi\ntransformers==3.0.*\ntorch==1.10.2\n"
  },
  {
    "path": "test/apis/async/text-generator/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"async-text-generator-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/async/text-generator/build-gpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-gpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"async-text-generator-gpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/async/text-generator/cortex_cpu.yaml",
    "content": "- name: text-generator\n  kind: AsyncAPI\n  pod:\n    port: 8080\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/async-text-generator-cpu:latest\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 1\n        mem: 2.5Gi\n"
  },
  {
    "path": "test/apis/async/text-generator/cortex_gpu.yaml",
    "content": "- name: text-generator\n  kind: AsyncAPI\n  pod:\n    port: 8080\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/async-text-generator-gpu:latest\n      env:\n        TARGET_DEVICE: \"cuda\"\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 1\n        gpu: 1\n        mem: 512Mi\n"
  },
  {
    "path": "test/apis/async/text-generator/cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements-cpu.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/async/text-generator/expectations.yaml",
    "content": "response:\n  content_type: \"json\"\n  json_schema:\n    type: \"object\"\n    properties:\n      text:\n        type: \"string\"\n    required:\n      - \"text\"\n"
  },
  {
    "path": "test/apis/async/text-generator/gpu.Dockerfile",
    "content": "FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu18.04\n\nRUN apt-get update \\\n    && apt-get install -y \\\n        python3 \\\n        python3-pip \\\n        pkg-config \\\n        build-essential \\\n        git \\\n        cmake \\\n    && apt-get clean -qq && rm -rf /var/lib/apt/lists/*\n\nENV LC_ALL=C.UTF-8 LANG=C.UTF-8\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements-gpu.txt /app/requirements.txt\nRUN pip3 install \\\n    --no-cache-dir \\\n    --extra-index-url https://download.pytorch.org/whl/cu113 \\\n    -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/async/text-generator/sample.json",
    "content": "{\n    \"text\": \"machine learning is\"\n}\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/app/main.py",
    "content": "import os, json, re\nimport requests\nimport torch\nimport torchvision\nimport boto3\nimport uuid\n\nfrom typing import List\nfrom PIL import Image\nfrom io import BytesIO\nfrom torchvision import transforms\nfrom fastapi import FastAPI, Request, status\nfrom fastapi.responses import PlainTextResponse\n\napp = FastAPI()\napp.device = os.getenv(\"TARGET_DEVICE\", \"cpu\")\napp.ready = False\ns3 = boto3.client(\"s3\")\n\n\n@app.on_event(\"startup\")\ndef startup():\n    # read job spec\n    with open(\"/cortex/spec/job.json\", \"r\") as f:\n        job_spec = json.load(f)\n    print(json.dumps(job_spec, indent=2))\n\n    # get metadata\n    config = job_spec[\"config\"]\n    app.job_id = job_spec[\"job_id\"]\n    if len(config.get(\"dest_s3_dir\", \"\")) == 0:\n        raise Exception(\"'dest_s3_dir' field was not provided in job submission\")\n\n    # s3 info\n    app.bucket, app.key = re.match(\"s3://(.+?)/(.+)\", config[\"dest_s3_dir\"]).groups()\n    app.key = os.path.join(app.key, app.job_id)\n\n    # loading model\n    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    app.preprocess = transforms.Compose(\n        [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize]\n    )\n    app.labels = requests.get(\n        \"https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt\"\n    ).text.split(\"\\n\")[1:]\n    app.model = torchvision.models.alexnet(pretrained=True).eval().to(app.device)\n\n    app.ready = True\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    if app.ready:\n        return PlainTextResponse(\"ok\")\n    return PlainTextResponse(\"service unavailable\", status_code=status.HTTP_503_SERVICE_UNAVAILABLE)\n\n\n@app.post(\"/\")\ndef handle_batch(image_urls: List[str]):\n    tensor_list = []\n\n    # download and preprocess each image\n    for image_url in image_urls:\n        if image_url.startswith(\"s3://\"):\n            bucket, image_key = re.match(\"s3://(.+?)/(.+)\", image_url).groups()\n            image_bytes = s3.get_object(Bucket=bucket, Key=image_key)[\"Body\"].read()\n        elif image_url.startswith(\"http\"):\n            image_bytes = requests.get(image_url).content\n        else:\n            raise RuntimeError(f\"{image_url}: invalid image url\")\n\n        img_pil = Image.open(BytesIO(image_bytes))\n        tensor_list.append(app.preprocess(img_pil))\n\n    # classify the batch of images\n    img_tensor = torch.stack(tensor_list).to(app.device)\n    with torch.no_grad():\n        prediction = app.model(img_tensor)\n    _, indices = prediction.max(1)\n\n    # extract predicted classes\n    results = [\n        {\"url\": image_urls[i], \"class\": app.labels[class_idx]}\n        for i, class_idx in enumerate(indices)\n    ]\n    json_output = json.dumps(results)\n\n    # save results\n    prediction_id = uuid.uuid4()\n    s3.put_object(Bucket=app.bucket, Key=f\"{app.key}/{prediction_id}.json\", Body=json_output)\n\n\n@app.post(\"/on-job-complete\")\ndef on_job_complete():\n    all_results = []\n\n    # aggregate all classifications\n    paginator = s3.get_paginator(\"list_objects_v2\")\n    for page in paginator.paginate(Bucket=app.bucket, Prefix=app.key):\n        if \"Contents\" not in page:\n            continue\n        for obj in page[\"Contents\"]:\n            body = s3.get_object(Bucket=app.bucket, Key=obj[\"Key\"])[\"Body\"]\n            all_results += json.loads(body.read().decode(\"utf8\"))\n\n    # save single file containing aggregated classifications\n    s3.put_object(\n        Bucket=app.bucket,\n        Key=os.path.join(app.key, \"aggregated_results.json\"),\n        Body=json.dumps(all_results),\n    )\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/app/requirements-cpu.txt",
    "content": "uvicorn[standard]\nfastapi\nrequests\nboto3==1.17.*\n-f https://download.pytorch.org/whl/torch_stable.html\ntorch==1.7.1+cpu\ntorchvision==0.8.2+cpu\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/app/requirements-gpu.txt",
    "content": "uvicorn[standard]==0.16.0\nsentencepiece==0.1.94\nfastapi\nrequests\nboto3==1.17.*\ntorch==1.10.2\ntorchvision==0.8.*\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"batch-image-classifier-alexnet-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/build-gpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-gpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"batch-image-classifier-alexnet-gpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/cortex_cpu.yaml",
    "content": "- name: image-classifier-alexnet\n  kind: BatchAPI\n  pod:\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/batch-image-classifier-alexnet-cpu:latest\n      command: [\"uvicorn\", \"--workers\", \"1\", \"--host\", \"0.0.0.0\", \"--port\", \"$(CORTEX_PORT)\", \"main:app\"]\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 1\n        mem: 2Gi\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/cortex_gpu.yaml",
    "content": "- name: image-classifier-alexnet\n  kind: BatchAPI\n  pod:\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/batch-image-classifier-alexnet-gpu:latest\n      command: [\"uvicorn\", \"--workers\", \"1\", \"--host\", \"0.0.0.0\", \"--port\", \"$(CORTEX_PORT)\", \"main:app\"]\n      env:\n        TARGET_DEVICE: \"cuda\"\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        gpu: 1\n        mem: 512Mi\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements-cpu.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/gpu.Dockerfile",
    "content": "FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu18.04\n\nRUN apt-get update \\\n    && apt-get install -y \\\n        python3 \\\n        python3-pip \\\n        pkg-config \\\n        build-essential \\\n        git \\\n        cmake \\\n        libjpeg8-dev \\\n        zlib1g-dev \\\n    && apt-get clean -qq && rm -rf /var/lib/apt/lists/*\n\nENV LC_ALL=C.UTF-8 LANG=C.UTF-8\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements-gpu.txt /app/requirements.txt\nRUN pip3 install \\\n    --no-cache-dir \\\n    --extra-index-url https://download.pytorch.org/whl/cu113 \\\n    -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/sample.json",
    "content": "[\n    \"https://i.imgur.com/PzXprwl.jpg\",\n    \"https://i.imgur.com/a5djbnv.jpeg\",\n    \"https://i.imgur.com/jDimNTZ.jpg\",\n    \"https://i.imgur.com/WqeovVj.jpg\"\n]\n"
  },
  {
    "path": "test/apis/batch/image-classifier-alexnet/submit.py",
    "content": "\"\"\"\nTypical usage example:\n\n    python submit.py <cortex-env> <dest-s3-dir>\n\"\"\"\n\nfrom typing import List\n\nimport sys\nimport json\nimport requests\nimport cortex\n\n\ndef main():\n    # parse args\n    if len(sys.argv) != 3:\n        print(\"usage: python submit.py <cortex-env> <dest-s3-dir>\")\n        sys.exit(1)\n    env_name = sys.argv[1]\n    dest_s3_dir = sys.argv[2]\n\n    # read sample file\n    with open(\"sample.json\") as f:\n        sample_items: List[str] = json.load(f)\n\n    # get batch endpoint\n    cx = cortex.client(env_name)\n    batch_endpoint = cx.get_api(\"image-classifier-alexnet\")[\"endpoint\"]\n\n    # submit job\n    job_spec = {\n        \"workers\": 1,\n        \"item_list\": {\"items\": sample_items, \"batch_size\": 1},\n        \"config\": {\"dest_s3_dir\": dest_s3_dir},\n    }\n    response = requests.post(batch_endpoint, json=job_spec)\n    print(json.dumps(response.json(), indent=2))\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "test/apis/batch/sum/app/main.py",
    "content": "import os\nimport boto3\nimport json\nimport re\n\nfrom typing import List\nfrom fastapi import FastAPI, status\nfrom fastapi.responses import PlainTextResponse\n\napp = FastAPI()\napp.ready = False\napp.numbers_list = []\ns3 = boto3.client(\"s3\")\n\n\n@app.on_event(\"startup\")\ndef startup():\n    # read job spec\n    with open(\"/cortex/spec/job.json\", \"r\") as f:\n        job_spec = json.load(f)\n    print(json.dumps(job_spec, indent=2))\n\n    # get metadata\n    config = job_spec[\"config\"]\n    job_id = job_spec[\"job_id\"]\n    if len(config.get(\"dest_s3_dir\", \"\")) == 0:\n        raise Exception(\"'dest_s3_dir' field was not provided in job submission\")\n\n    # s3 info\n    app.bucket, app.key = re.match(\"s3://(.+?)/(.+)\", config[\"dest_s3_dir\"]).groups()\n    app.key = os.path.join(app.key, job_id)\n\n    app.ready = True\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    if app.ready:\n        return PlainTextResponse(\"ok\")\n    return PlainTextResponse(\"service unavailable\", status_code=status.HTTP_503_SERVICE_UNAVAILABLE)\n\n\n@app.post(\"/\")\ndef handle_batch(batches: List[List[int]]):\n    for numbers_list in batches:\n        app.numbers_list.append(sum(numbers_list))\n\n\n@app.post(\"/on-job-complete\")\ndef on_job_complete():\n    # this is only intended to work if 1 worker is used (since on-job-complete runs once across all workers)\n    json_output = json.dumps(app.numbers_list)\n    s3.put_object(Bucket=app.bucket, Key=f\"{app.key}.json\", Body=json_output)\n"
  },
  {
    "path": "test/apis/batch/sum/app/requirements.txt",
    "content": "uvicorn[standard]\nfastapi\nboto3==1.17.*\n"
  },
  {
    "path": "test/apis/batch/sum/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"batch-sum-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/batch/sum/cortex_cpu.yaml",
    "content": "# this API is only meant to run with 1-worker jobs\n\n- name: sum\n  kind: BatchAPI\n  pod:\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/batch-sum-cpu:latest\n      command: [\"uvicorn\", \"--workers\", \"1\", \"--host\", \"0.0.0.0\", \"--port\", \"$(CORTEX_PORT)\", \"main:app\"]\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 256Mi\n"
  },
  {
    "path": "test/apis/batch/sum/cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/batch/sum/sample.json",
    "content": "[\n    [1, 2, 67, -2, 43],\n    [9, 0, 1, 0, 0, -1]\n]\n"
  },
  {
    "path": "test/apis/batch/sum/sample_generator.py",
    "content": "from typing import List\nfrom random import sample\n\nRANGE = 10 ** 12\nLENGTH = 5\n\n\ndef generate_sample() -> List[int]:\n    return sample(range(RANGE), LENGTH)\n"
  },
  {
    "path": "test/apis/batch/sum/submit.py",
    "content": "\"\"\"\nTypical usage example:\n\n    python submit.py <cortex-env> <dest-s3-dir>\n\"\"\"\n\nfrom typing import List\n\nimport sys\nimport json\nimport requests\nimport cortex\n\n\ndef main():\n    # parse args\n    if len(sys.argv) != 3:\n        print(\"usage: python submit.py <cortex-env> <dest-s3-dir>\")\n        sys.exit(1)\n    env_name = sys.argv[1]\n    dest_s3_dir = sys.argv[2]\n\n    # read sample file\n    with open(\"sample.json\") as f:\n        sample_items: List[str] = json.load(f)\n\n    # get batch endpoint\n    cx = cortex.client(env_name)\n    batch_endpoint = cx.get_api(\"sum\")[\"endpoint\"]\n\n    # submit job\n    job_spec = {\n        \"workers\": 1,\n        \"item_list\": {\"items\": sample_items, \"batch_size\": 1},\n        \"config\": {\"dest_s3_dir\": dest_s3_dir},\n    }\n    response = requests.post(batch_endpoint, json=job_spec)\n    print(json.dumps(response.json(), indent=2))\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "test/apis/realtime/hello-world/app/main.py",
    "content": "import os\n\nfrom fastapi import FastAPI\nfrom fastapi.responses import PlainTextResponse\n\napp = FastAPI()\napp.response_str = os.getenv(\"RESPONSE\", \"hello world\")\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    return PlainTextResponse(\"ok\")\n\n\n@app.post(\"/\")\ndef post_handler():\n    return PlainTextResponse(app.response_str)\n"
  },
  {
    "path": "test/apis/realtime/hello-world/app/requirements.txt",
    "content": "uvicorn[standard]\nfastapi\n"
  },
  {
    "path": "test/apis/realtime/hello-world/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-hello-world-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/hello-world/cortex_cpu.yaml",
    "content": "- name: hello-world\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 128Mi\n"
  },
  {
    "path": "test/apis/realtime/hello-world/cortex_cpu_arm64.yaml",
    "content": "- name: hello-world\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    containers:\n    - name: api\n      image: infrastructureascode/hello-world\n      readiness_probe:\n        http_get:\n          path: \"/health\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 128Mi\n"
  },
  {
    "path": "test/apis/realtime/hello-world/cortex_scale_to_zero.yaml",
    "content": "- name: hello-world\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    max_queue_length: 999\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n        timeout_seconds: 3\n      compute:\n        cpu: 200m\n        mem: 128Mi\n  autoscaling:\n    min_replicas: 0\n    max_replicas: 1\n    downscale_stabilization_period: 30s\n"
  },
  {
    "path": "test/apis/realtime/hello-world/cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/realtime/hello-world/sample.json",
    "content": "{}\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/README.md",
    "content": "# Running an example\n\n## CPU/GPU\n\nGet the image classifier endpoint:\n\n```bash\ncortex get image-classifier-resnet50\n```\n\n```bash\npython client.py http://<lb-id>.elb.<cluster-region>.amazonaws.com/image-classifier-resnet50\n```\n\nOr alternatively:\n\n```bash\ncurl \"http://<lb-id>.elb.<cluster-region>.amazonaws.com/image-classifier-resnet50/v1/models/resnet50:predict\" -X POST -H \"Content-type: application/json\" -d @sample.json\n```\n\n## Inferentia\n\n### HTTP\n\nGet the image classifier endpoint:\n\n```bash\ncortex get image-classifier-resnet50\n```\n\n```bash\npython client_inf.py http://<lb-id>.elb.<cluster-region>.amazonaws.com/image-classifier-resnet50\n```\n\n### gRPC\n\nThe Inferentia examples were inspired by [this tutorial](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-deploy/tutorials/k8s_rn50_demo.html).\n\nThis guide shows how to exec into a pod, check the Neuron runtime, and make inferences using gRPC for testing purposes (exposing gRPC endpoints outside of the cluster is not currently supported by Cortex, but is on the roadmap).\n\n#### Making an inference\n\nExec into the TensorFlow Serving pod:\n\n```bash\nkubectl exec api-image-classifier-resnet50-5b6df59b9b-p4s2h -c api -it -- /bin/bash\n```\n\nCreate this `tensorflow-model-server-infer.py` with this contents (e.g. using `vim tensorflow-model-server-infer.py`):\n\n```python\nimport numpy as np\nimport grpc\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications.resnet50 import preprocess_input\nfrom tensorflow_serving.apis import predict_pb2\nfrom tensorflow_serving.apis import prediction_service_pb2_grpc\nfrom tensorflow.keras.applications.resnet50 import decode_predictions\n\nif __name__ == '__main__':\n    channel = grpc.insecure_channel('localhost:8500')\n    stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)\n    img_file = tf.keras.utils.get_file(\n        \"./kitten_small.jpg\",\n        \"https://raw.githubusercontent.com/awslabs/mxnet-model-server/master/docs/images/kitten_small.jpg\")\n    img = image.load_img(img_file, target_size=(224, 224))\n    img_array = preprocess_input(image.img_to_array(img)[None, ...])\n    request = predict_pb2.PredictRequest()\n    request.model_spec.name = 'resnet50_neuron'\n    request.inputs['input'].CopyFrom(\n        tf.make_tensor_proto(img_array, shape=img_array.shape))\n    result = stub.Predict(request)\n    prediction = tf.make_ndarray(result.outputs['output'])\n    print(decode_predictions(prediction))\n```\n\nRun an inference:\n\n```bash\npython tensorflow-model-server-infer.py\n```\n\n#### Inspecting the neuron runtime\n\nExec into the TensorFlow Serving pod (the `rtd` pod will also work for the example which uses the neuron-rtd sidecar):\n\n```bash\nkubectl exec api-image-classifier-resnet50-5b6df59b9b-p4s2h -c api -it -- /bin/bash\n```\n\nInstall dependencies:\n\n```bash\napt-get update && apt-get install -y aws-neuron-dkms aws-neuron-runtime-base aws-neuron-runtime aws-neuron-tools\nPATH=\"/opt/aws/neuron/bin:${PATH}\"\n```\n\nRun CLI commands (described [here](https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-tools/basic.html)):\n\n```bash\nneuron-ls\nneuron-cli list-model\nneuron-top\n```\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-image-classifier-resnet50-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/build-gpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-gpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-image-classifier-resnet50-gpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/build-neuron-rtd.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-neuron-rtd.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"neuron-rtd\"\n\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-east-1.amazonaws.com\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/build-neuron-tf-serving.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-neuron-tf-serving.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"neuron-tf-serving\"\n\naws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-east-1.amazonaws.com\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/client.py",
    "content": "\"\"\"A client that performs inferences on a ResNet model using the REST API.\n\nThe client downloads a test image of a cat, queries the server over the REST API\nwith the test image repeatedly and measures how long it takes to respond.\n\nThe client expects a TensorFlow Serving ModelServer running a ResNet SavedModel\nfrom:\n\nhttps://github.com/tensorflow/models/tree/master/official/resnet#pre-trained-model\n\nThe SavedModel must be one that can take JPEG images as inputs.\n\nTypical usage example:\n\n    python client.py <http://host:port>\n\"\"\"\n\nimport sys\nimport base64\nimport requests\n\n# the image URL is the location of the image we should send to the server\nIMAGE_URL = \"https://tensorflow.org/images/blogs/serving/cat.jpg\"\n\n\ndef main():\n    # parse arg\n    if len(sys.argv) != 2:\n        print(\"usage: python client.py <http://host:port>\")\n        sys.exit(1)\n    address = sys.argv[1]\n    server_url = f\"{address}/v1/models/resnet50:predict\"\n\n    # download labels\n    labels = requests.get(\n        \"https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt\"\n    ).text.split(\"\\n\")[1:]\n\n    # download the image\n    dl_request = requests.get(IMAGE_URL, stream=True)\n    dl_request.raise_for_status()\n\n    # compose a JSON Predict request (send JPEG image in base64).\n    jpeg_bytes = base64.b64encode(dl_request.content).decode(\"utf-8\")\n    predict_request = '{\"instances\" : [{\"b64\": \"%s\"}]}' % jpeg_bytes\n\n    # send few requests to warm-up the model.\n    for _ in range(3):\n        response = requests.post(server_url, data=predict_request)\n        response.raise_for_status()\n\n    # send few actual requests and report average latency.\n    total_time = 0\n    num_requests = 10\n    for _ in range(num_requests):\n        response = requests.post(server_url, data=predict_request)\n        response.raise_for_status()\n        total_time += response.elapsed.total_seconds()\n        prediction = labels[response.json()[\"predictions\"][0][\"classes\"]]\n\n    print(\n        \"Prediction class: {}, avg latency: {} ms\".format(\n            prediction, (total_time * 1000) / num_requests\n        )\n    )\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/client_inf.py",
    "content": "\"\"\"A client that performs inferences on a ResNet model using the REST API.\n\nThe client downloads a test image of a cat, queries the server over the REST API\nwith the test image repeatedly and measures how long it takes to respond.\n\nThe client expects a TensorFlow Serving ModelServer running a ResNet SavedModel\nfrom:\n\nhttps://github.com/tensorflow/models/tree/master/official/resnet#pre-trained-model\n\nThe SavedModel must be one that can take JPEG images as inputs.\n\nTypical usage example:\n\n    python client.py <http://host:port>\n\"\"\"\n\nimport sys\nimport json\n\nimport io\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.applications import resnet50\nfrom PIL import Image\nimport requests\nimport numpy as np\n\n# the image URL is the location of the image we should send to the server\nIMAGE_URL = \"https://tensorflow.org/images/blogs/serving/cat.jpg\"\n\n\ndef main():\n    # parse arg\n    if len(sys.argv) != 2:\n        print(\"usage: python client.py <http://host:port>\")\n        sys.exit(1)\n    address = sys.argv[1]\n    server_url = f\"{address}/v1/models/resnet50_neuron:predict\"\n\n    # download labels\n    labels = requests.get(\n        \"https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt\"\n    ).text.split(\"\\n\")[1:]\n\n    # download the image\n    response = requests.get(IMAGE_URL, stream=True)\n    img = Image.open(io.BytesIO(response.content))\n    img = img.resize((224, 224))\n\n    # process the image\n    img_arr = image.img_to_array(img)\n    img_arr2 = np.expand_dims(img_arr, axis=0)\n    img_arr3 = resnet50.preprocess_input(np.repeat(img_arr2, 1, axis=0))\n    img_list = img_arr3.tolist()\n    request_payload = {\"signature_name\": \"serving_default\", \"inputs\": img_list}\n\n    # send few requests to warm-up the model.\n    for _ in range(3):\n        response = requests.post(\n            server_url,\n            data=json.dumps(request_payload),\n            headers={\"content-type\": \"application/json\"},\n        )\n        response.raise_for_status()\n\n    # send few actual requests and report average latency.\n    total_time = 0\n    num_requests = 10\n    for _ in range(num_requests):\n        response = requests.post(\n            server_url,\n            data=json.dumps(request_payload),\n            headers={\"content-type\": \"application/json\"},\n        )\n        response.raise_for_status()\n        total_time += response.elapsed.total_seconds()\n\n        label_idx = np.argmax(response.json()[\"outputs\"][0])\n        prediction = labels[label_idx]\n\n    print(\n        \"Prediction class: {}, avg latency: {} ms\".format(\n            prediction, (total_time * 1000) / num_requests\n        )\n    )\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/cortex_cpu.yaml",
    "content": "- name: image-classifier-resnet50\n  kind: RealtimeAPI\n  pod:\n    port: 8501\n    max_concurrency: 8\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-image-classifier-resnet50-cpu:latest\n      readiness_probe:\n        exec:\n          command: [\"tfs_model_status_probe\", \"-addr\", \"localhost:8500\", \"-model-name\", \"resnet50\"]\n      compute:\n        cpu: 1\n        mem: 2Gi\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/cortex_gpu.yaml",
    "content": "- name: image-classifier-resnet50\n  kind: RealtimeAPI\n  pod:\n    port: 8501\n    max_concurrency: 8\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-image-classifier-resnet50-gpu:latest\n      readiness_probe:\n        exec:\n          command: [\"tfs_model_status_probe\", \"-addr\", \"localhost:8500\", \"-model-name\", \"resnet50\"]\n      compute:\n        cpu: 200m\n        gpu: 1\n        mem: 512Mi\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/cortex_inf.yaml",
    "content": "- name: image-classifier-resnet50\n  kind: RealtimeAPI\n  pod:\n    port: 9000\n    max_concurrency: 8\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/neuron-tf-serving:latest\n      command: [\"/usr/local/bin/entrypoint.sh\"]\n      args:\n        - --port=8500\n        - --rest_api_port=9000\n        - --model_name=resnet50_neuron\n        - --model_base_path=s3://cortex-examples/resnet50_neuron/\n      env:\n        AWS_REGION: us-west-2\n        S3_USE_HTTPS: \"1\"\n        S3_VERIFY_SSL: \"0\"\n        S3_ENDPOINT: s3.us-west-2.amazonaws.com\n        AWS_LOG_LEVEL: \"3\"\n      compute:\n        inf: 1\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/cortex_inf_rtd.yaml",
    "content": "- name: image-classifier-resnet50\n  kind: RealtimeAPI\n  pod:\n    port: 9000\n    max_concurrency: 8\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/neuron-tf-serving:latest\n      command: [\"/usr/local/bin/tensorflow_model_server_neuron\"]\n      args:\n        - --port=8500\n        - --rest_api_port=9000\n        - --model_name=resnet50_neuron\n        - --model_base_path=s3://cortex-examples/resnet50_neuron/\n      env:\n        AWS_REGION: us-west-2\n        S3_USE_HTTPS: \"1\"\n        S3_VERIFY_SSL: \"0\"\n        S3_ENDPOINT: s3.us-west-2.amazonaws.com\n        AWS_LOG_LEVEL: \"3\"\n        NEURON_RTD_ADDRESS: unix:/mnt/neuron.sock\n    - name: rtd\n      image: quay.io/cortexlabs-test/neuron-rtd:latest\n      command: [\"neuron-rtd\", \"-g\", \"$(NEURON_RTD_ADDRESS)\", \"--log-console\"]\n      compute:\n        inf: 1\n      env:\n        NEURON_RTD_ADDRESS: unix:/mnt/neuron.sock\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/cpu.Dockerfile",
    "content": "FROM tensorflow/serving:2.3.0\n\nRUN apt-get update -qq && apt-get install -y --no-install-recommends -q \\\n        wget \\\n    && apt-get clean -qq && rm -rf /var/lib/apt/lists/*\n\nRUN TFS_PROBE_VERSION=1.0.1 \\\n    && wget -qO /bin/tfs_model_status_probe https://github.com/codycollier/tfs-model-status-probe/releases/download/v${TFS_PROBE_VERSION}/tfs_model_status_probe_${TFS_PROBE_VERSION}_linux_amd64 \\\n    && chmod +x /bin/tfs_model_status_probe\n\nRUN mkdir -p /model/resnet50/ \\\n    && wget -qO- http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz | \\\n    tar --strip-components=2 -C /model/resnet50 -xvz\n\nENV CORTEX_PORT 8501\n\nENTRYPOINT tensorflow_model_server --rest_api_port=$CORTEX_PORT --rest_api_num_threads=8 --model_name=\"resnet50\" --model_base_path=\"/model/resnet50\"\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/gpu.Dockerfile",
    "content": "FROM tensorflow/serving:2.8.2-gpu\n\nRUN apt-get update -qq && apt-get install -y --no-install-recommends -q \\\n        wget \\\n    && apt-get clean -qq && rm -rf /var/lib/apt/lists/*\n\nRUN TFS_PROBE_VERSION=1.0.1 \\\n    && wget -qO /bin/tfs_model_status_probe https://github.com/codycollier/tfs-model-status-probe/releases/download/v${TFS_PROBE_VERSION}/tfs_model_status_probe_${TFS_PROBE_VERSION}_linux_amd64 \\\n    && chmod +x /bin/tfs_model_status_probe\n\nRUN mkdir -p /model/resnet50/ \\\n    && wget -qO- http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz | \\\n    tar --strip-components=2 -C /model/resnet50 -xvz\n\nENV CORTEX_PORT 8501\n\nENTRYPOINT tensorflow_model_server --rest_api_port=$CORTEX_PORT --rest_api_num_threads=8 --model_name=\"resnet50\" --model_base_path=\"/model/resnet50\"\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/neuron-rtd.Dockerfile",
    "content": "# LIST VERSIONS: aws ecr list-images --region us-east-1 --registry-id 790709498068 --repository-name neuron-rtd\n\nFROM 790709498068.dkr.ecr.us-east-1.amazonaws.com/neuron-rtd:1.5.0.0\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/neuron-tf-serving.Dockerfile",
    "content": "# LIST VERSIONS: aws ecr list-images --region us-east-1 --registry-id 763104351884 --repository-name tensorflow-inference-neuron\n\nFROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/tensorflow-inference-neuron:1.15.5-neuron-py37-ubuntu18.04-v1.3\n"
  },
  {
    "path": "test/apis/realtime/image-classifier-resnet50/sample.json",
    "content": "{\"instances\": [{\"b64\": \"/9j/4AAQSkZJRgABAQAASABIAAD/4QBYRXhpZgAATU0AKgAAAAgAAgESAAMAAAABAAEAAIdpAAQAAAABAAAAJgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAB8qADAAQAAAABAAAC0AAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgC0AHyAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAwMDAwMDBQMDBQgFBQUICggICAgKDQoKCgoKDRANDQ0NDQ0QEBAQEBAQEBMTExMTExYWFhYWGRkZGRkZGRkZGf/bAEMBBAQEBgYGCwYGCxoSDxIaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGhoaGv/dAAQAIP/aAAwDAQACEQMRAD8A/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAphkQHBPWs7Vr5LG2aZ2CADqelfM3iT4xi0vXtIiQAcHaCxz9elceKxsKHxHVh8JOt8J9VLIjZ2nOKfkV8i6V8b4knRL2TAPXcMY/Kvb/AA/4+0fVYg8UwZ26DPSs6GZUqul7F1sDVp6tHpdFU7W9huVzG24eo6VcrvTT1RxtWCiiimIKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopuD3oAdkVl32pR2ti94CPlBYA9wOuPwHFc/4v15NHsRzhp38gexbBB/Lj8a+aPiH8W4reF47YM7iR2WNTk7DwFbHQDnP1rzsZmFOgmnud2FwM6zVloeqal8Wb20AkhgjdWJBGGyp4ODz6Zrn5fjjqILeTbwELgc56/8AfX6V8Z6h4y1fW5XK+YkR9Rhc+u3/ABrn5by+d1zcyZHXIxn8hXzM85xDekrH0MMopJaxPvzSfjeJ5fL1KKNOM5QHqO3U8+le36Vrdhq8CTWkqy7gCdhyBkZxn2zX4/XOtX1vOWW4kVsjGCAK+hPg38aDp1/FpWqShI5mAB7474PGM4ruwWcT5kqzuu5x4zKYqPNTWp+itFZul6pbarbLc2Z3Ie46frjNaVfURkmro+daadmFFFFMQUUUUAFFFFABRRRQAUUUUAf/0f1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA4/xtDI/h68aIciNjnr2r8utQv531CRrmYxMWPJYknnoF7nH1xX6y6pBHPYzRydGUjn3Ffkx4306TSfEt3DJIUjikYOTgMQW4A47181nsWnGZ9Fkck+aJZj1CRiY7aJ5PlyzyyYzjuccL+Jq9p2q6pA3m2l2I3B4CMxH8h+dYlnkxCSVSsQ5CHvju3cn0/lW2keoIu52FujDqw+Zh6hBjj/eP4GvmozbZ9BOmtj3LwZ8aNS0t1steAZV4DKePxr6s8MePNH123VracSuwyQO31r85vsqy5gjk3Njuo499qrx+JzWhomuax4Vu0ubGYFMglfu5/4CTXq4XNa1F2lqjy8TllOqrx0Z+pCOHUMO9Pr53+HnxbsNZjWC/kw+Bwa99gv7S4UNFIG3dPxr6vDYunWjzQZ8zXw06UuWSLdFFFdRzhRRRQAUUUUAFFFFABRRRQAUUUUAFFFGaACikzmloAa7pGpdzgDqTXLa14ostNRkDZfkfQ/5IrD8ba69hauiHBfIX6pn+or5V8ZeOJrW22pIXuZR8q9SCTliR+tePmOZ+x9yO56eCy91tXsT/FD4jy3M32dJMTh2AABbGM44Hfnqa+ZWubi9uDNdF23MfmYDBPf5T0xV6WC5vZ5JxcebI3UyDGS3Tv07ZqKJ7QpJBe+bbmNgkgJyFbOMjPI/zmvjqs51ZOTZ9hRoRpQ5USh/Kb94+z0dfusPTjpWZe3kYXEcy7v9ruPqOP0FW5rKaydWeQz2sp271OQM8ZYdRn6cGubuLNhMd4G8NnrjnoQQeDms+SyNVqZdyp37ojkEHK9SOf5Vzhml068W4jJVkbPHGD6iupLRxn7m3HYjke1c5evBNHjvmrpysxTjc/TL4AePm8QeHbeEy5dBiRCAACOMhgO/v+dfT6uGAI71+SHwO1+5stYexkuxbFhuUkkKSp54APX0NfqV4V1JdT0uGYAkgAFuxPtnkfSvr8nxTnD2T6Hx+a4b2dTmXU6eiiivbPICiiigAooooAKKKKACiiigD//S/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigCC5iE0JQnFfmv+0J4fl07xS9zApWOUhg20s2f5Amv0vPSvl79ojwi2qaA2oR5DQDPy9ffHvXk5xQ56N10PVyiv7Ouk+p8RaFKZSGUDcvCknIGO/px+prrBHa3MwMjltzcsG+8fQcZ/wA+p48z0+5fTfOtlOZIuCc9MkKFX/PauwtdRt4Ll45kCBBtCgbpAmdqr14Z/wA+TniviIxaZ9pNX2N+NJL2V47MMtsh+YJ8qlvfux9yfoO9Wl0y2CmVNgGcEhdxOOvPP88UttN9uzPOSYrcfJbxjChv9oggMfY8Dqa0IbD7SRc34aTLAJFEOOO+TyRnjI6npXbGHMjkk7GGI7vT5vtGns6MDleAP0zXq/hv4p6lb+RFNKUkiK5DcZ2nIOPQ+1ctDopkcrdxxwq3IQ8yH6ryR/wIk+wqld6GmP3TrGQQV4BK/iD39KpRnT1g7GM1CppJH2L4T+KNnf7LW/YRyM4BJPGDx1/WvXbXULW8j82Bww27vwr80rXUr/S5fKmw4HI7AjvjPevYNA8e6lBEpilYRbSrZ7Dtn6GvVwudSh7tZXPIxOUp+9TPtqivDdL+J4ljTzhyuC3uB1x/Ot6++JWmQGRYX3so+THfJ6/lXsxzTDOPNznlPA1k+XlPVKK8Yt/ilar5T3KkCQHcPQr0H41txfEzRzaLM5+dskr6c/4VdPMMPP4ZEywVaO8T0yivMW+KGjZUxqSGIHX1qhf/ABU0y2nMYIKhSMj+/jj8Kc8ww8VdzQRwdZuyieu0V4Jc/F+1D7IFLDGfyOKzZvjI0pWRIsIeMZ6H/wCtXLLOsIvtm8crxD+yfRpIHBPWjINfM6/FG/vWhKpho95Y5659PTAqW1+JWsW0i+eodYwfYc5Az+FT/beH8ynlVc+k6QjNfPTfFe+a4K+ViIALx16/4V0Nj8VLctJ9tjKc/KB2Hv8AWrhnGFk7cxnLLa8Vex7KBiqOo6hbabavdXLbVQE1wl58R9Hghyr5YxhsDrls4H4da8N8b+P7i6hlW4YqnJRRz2AyfypYrNqNOD5HdlYfLqtSSUlZGZ8SviFDcRsYBtCOzxjOT8x6/QE8V81Sy3erXr3quxZWxgLgbu+Wz3PrV67nuNdvJWY7Vc5ZsYCrxhSzFRnn7q+tdlpuhTmBIuYLhMKjkAowPOGA6D3zXyM3Urz5p7s+upU4UIcqONWynnjYMuZAM/NwSAc9cDp7ipbyx+2Wkd9kTb8RS9x8vC7vRsYB9cda7e50OaH968DQujZUlSQrezDsagbTZpnOQrrMMSJ1U+uVPOPQg5FbwoeRLrLdHm8dkIYJbG53NbnBIccqp4PPp/LrXIalbPZ3DWty2UJIVsg5HY5Feo39lLHGEVpIymcB1J+XoMZ+8OOevFcNr1g1xBHKEQZG3IUEe3XGQR+NTUpWNYVNTz28MxbfuJA7/TiuPmkKuyKeSTj2rq7u3uY2KBc44+XpWDHbEy7nxjmudRsbPU6bwHdG38TWrruBzggduMEmv1C+DF/LcWjISHC4B24GPr3/ADr8r/D8LQ65bvuGwMDk4GB6/j2r9K/gzaTiU3UROw8Ntxlc9m/pXrZTJrEK3Y8TN4r2ep9Q0Ui7sfNyaWvsT5IKKKKACiiigAooooAKKKKAP//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArm/FmkR6zodxZv1ZDg/4V0lNdd6MvqMVFSCnFxfUqEnGSkj8f/F2hS+HNcubMsEkaXIB5IAPUDux7ZrNtWkH+qbY0kgz/E7ueg49SfwAr6T/AGivBH9mXY1vywFcEs69c+nqa+dvDTQXN5bxheAS20nbt/vM30A/OvgsTRlCq0z9AwtZVaKmj0S3vLWxaHQ5ys05BeTcV2KiDJkdR1GegPYZPpWzavJf3X7p5pJJMFtmRJkj5Qo528dAMBRySSad4e8OWc1vPeXETAXZDyuW+fDEFRkA47bVAJ79a9M0v+z9PtfskSLawAliu3YxyernJJJ68kk9zXXSo3V5M5q1VR2V2c/a6HevIdyeUH6gEuxx/fbOAB2A789aZc6SY02i4YOpxmPA5/HNdW+rW8qljF/o4bHHG4kdgOWJ9P6VgXN9eSJhLTyEH3Y1KbiPQ9Pxxj8a1lGFvdOVTm3qcldaSxVvMJcDoXbJz9SBWXZXkukuAGDd/LOAduMHBXIIre1OPWGbJjVARyGdePbHeuD1a0ZFErSYxzsXJ5Hc9h9RXBVp9UdlN3VmdvbawJleS1PlvHnfHnOR2K9609N12OZgCPmJPH5153o199kuEMhyoBHI5I6j8Aenaqs0503UmhaQssjZXnpk7v8A9Vc3mVKNtD3KW8gnRuh24P6cn9awLi/lSXybT5s+/Q1TsUM9ivk/MXbj8OT+FaenaIrYuI2yAH3HPJOf6VnOHOKKUdytObhnSKFjgthsfn+FP8iWFkE55UYck/3uldlBFaQHeuCxJI47471z+oRG4eaViFVSBnHH/wCvrzU1KSUUy4TV7WOVUSQCUoCzb8H/AHTip4Z1Nx5bkKWclfQL1/QVfk8hbBVkz5s4Ue55Y/4Uy5tWEalX8oAFR77yeOPQCsFS6o3c11NbTtTtvLkkiZCisB1HDNyf8KfcXMUmYGbndycjB5yao+H7SEWRjH3gxY+vJ4HuaTUYoIvk4aSVhs/LJNdCbsYNLmNl7qKGJHj4XHGevPGapLcPLI6xEbR8xP8AeOM4/Ws77OLpke4yIyOcjHAOf1qzDdQSXQsLfDHPJ9Mdf0q4tt6ktJEmta6ukwiOMLJcnacP0QHA5xXmk+vRzFri/Ly7i2BnGQoyff0/DjioPF9/LNqsrQuFEQAweCQMgH8OTXnWoXi2phiacR7V643MQPuqB2GeSc8nsTSd5Ssb06aUbnsGmSSSItxdSLHGRjyiAx555Y4x9AK3YruRz5cdrgx9PMlXaR6ADJ5+grx3TtWluLtLSJpGj2/KOUwM8n5DuP4kn1Ir0drY3BEH2m3tQw/1cyI+Rj2yxJ9ya9CjHQ5q2j1OhXW2tI2W/KwAHOULTAD68kfyqI+J7KchRcx3IYk7RlHx9M9q5G802SHMGnOINgYnyZUDAdN2HC4H+6vHrXkWvapqKnypLiKeTJB2ljKdvcruBb6iutRdrr+v69TKNNSPoz+3NGv8wzE5OCBsJAx7jpz61gXvh/Tpo3Nmcbzltvzqx/2kJyD6EV8yNrF6wVDcSW/ozAx5/wCA9fx/OrVvql1A/wA7NvxjdnaxH1HUfnSkpdTWOHtrFnaeI/D89shkjkBzn7vLH/H8ea4J7dbaMgnB756/WtSTVJ4fmmlabfx8xz/kiueur0yEq/8AEOorhnG7OuN0rMvaO5l1OIICQrLkgDPXtX6l/BuK/ist0ixNGRgOOXHsSD2+lfm58P8ATln1qxVxyzMVXqXKgnGPUYr9TPhppj2OkRMzhgw3Kdmxh6qw45B9RXp5PC9e66Hh51P3LHqlFFFfWHyoUUUUAFFFFABRRRQAUUUUAf/U/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA83+J3hBPFvhqezUDzVUsp75HSvzh0zRprHxSumyr5cibotxUEljnJGRwBk5Jr9ZWUMpU9DXxD8evAw0vUW17TP3P2gYZunzH0Pb/wCvXz2d4e1q8fmfQZLi7N0X12POb/xvo2kXC6dYzbvJHGw/Jk9SXwcsxznsPWuevPGFvdsIJtszkjCCXaozjoGA/Mj6ZrzO1DbxBOxjRWB4xufHbkdOO/15NdJo+leJ9Vui5/dKAW3EAIvP3iT8zEA8KPxwK8SNSU9UfRewhBXZ041vUpZozY2kS2iZVXaRgm0dcAAPKxPXbhc9TXULq18BstoFVtvL9CPXAJ2gZ4yWPPSsx47LS2MWLjUrhFziMMQCP78uOT6IgAH6muJfFtwoWxs0sYF4KFtr8c/OcnH4kn8a6InPJJ7IsTm38wGSHa8g5JVdzfTHzY/AA1mTW7pCIleSRGPIZiu38+g9K1LWC581luctITk4c4YdOCeMenHPtTrmwulbcibQPlYpHgj+7yTyB34rOqnYSZwxRAmE+c9VOfTkr2zXUnRxrFxBeoMIzKzDqPlAOM/h+lXLbRYpmzcLhkOCQMZHXP41uSkROYoV2L1BH94Zz+vNeXVnym8feNG3uYNLtTASDtPy/jWnp2qCbCR8bwTjtnArza/mmBKseuR9R/8AWp2g6hcy35hGVVAQSeg61zKtJs1dFWbO81XUngkBhPrkY5ABxx+lJqF5K9qlsowZdvPf5jgViTKJpHM5wWT5eeucevYVrpIvmJLs3CMgDPP3ec/hVqV3YmySRjW7qdXs4JwzKu8/98L/APXq/NcRy3cUZbKld+PTOaclmtnJNkbnV2MfHODjI9qyrSOZdR+2yYC54HYg8H8qPhuh76o20uUsrgup3ALnA9c9/wADWet+txqEczfdQAjPoetYOtXpt7mNs4GMH3wcf04pLOVrmRJ8AbwFyOePvfn2qOdrYpQT1Z2mryzSWx8oHZIOo+nFcppV5FpP2m7vOJEUlWbrkgcV0lzdPFagDO1FLEfgAP5153rcAv4JZnk2JGhLhOCSeAg+tbueqMEtLHmWt3893qJjjcgznIUDcSc5UAe55yeABUR0hUjI1GF3zwsioRIT/sYzwO5OM9qZdaVeQOivCZbuddwWMnESMcBSw/iYDoOceldhpPhVrC3e41mMWkBA3BZGWSUsOR1DBR+v05r0KdLRMcqtkc/bamNIjWxtlmSCUnMsICuD6qzEkkd8rjsOma0P7WdZ49M1icTIqlw32hCxQ9CyNGdwPvyM1pS6Hf3iS6hYyqlo67VGI0TYv91eWIXuWbnrXOP4VttUVPmAgiG9jA2ckdxJwD14AyPYV1xgk9TGUkzvdP1Tw00iaV9vktcHPBUoB2wRkD2+7TtX8C2OrRC7s75Z8qSwOwMew3K0bkfUH8689/4Rm5uLOTTVaG/hGfLS5BhuIixzkSxg8Z/vDHeoYtI8W+Hbdbq8nlEFuxAZVacbCuWVigb046D8a3gu39fqZtdmZV14V1HTrloEI6FsR4lcD1OX3gfVMVjtaGzVpZJBKD0KqU/MN/StDVdZ8RWNmTfPb6jazcxMr7Jx7eW4Xkd8AE9a4aXV/td0oVTCwAypGD+NOdzopyfU6B4JWAdzkFQ351SSMySIHIALZ/AV1+nXUVpYxSSxrKOQc44H41mazZzG8NxbHdDtGwk889vY15dSerSOhHpXwaistb+LemQt8lpZW8i84wXZTnuB37mv1Y0WC3tYEhtSSrYIznIUDGTnnntmvzA/Zq0wf8JHqWsMq5tIQi7hn5pGx+PAxj3r9MvDVwJrfibzmJ+dlA5I498KOg/HHFfRZLFKnfqfK53L97yrojrKKKK908MKKKKACiiigAooooAKKKKAP//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAK8b+Nmiw6x4Ou0lG4hDt9jXslch42s3vdBuIkA+4SfpiuXGw5qE15HRhJ8taMvM/M3w5pVtBE1/rJZvJ4ZQpy/PyhfX8P5V0Fxr8d5GYEjZWX/V2sJB5HTzJCcE/wCyOB7VX8R2V7bWT2gDD5yAFbaCDy2cdP8ACvNrtDaQhbEhHJ+Z1Hf+6uece/U+1fBRqSi7H6AqaqK53hn1eaF4YpILEZBMSSh5WbOPmbDhRnk4Vjx8oHWrNtbLczeUZJ9WnQ/6qFGggAzjAbLSN/tO5rzBIbnUn+wm7MbjmZtwwB3XAIOAPTqeOgr6X+Hfhq2to47u2mVkyMtyXkx03N3Yf/Wx3r1MOubY5MTJU1dnR6J4T/0NDqlqLfI/1ceWVc9QW6tn1ySadfeH7axZltgie0WM7fqRn8K9MWa3iGRhAOuRgj3OOv5VzeszQGNWjQHPQriuzFKEaZ5FKrOU9TzEwyjdDJ8yqT+HuPp3FQG3eQHI6de45HWti6LS/v0BHPJPb61T89YSUbAyOCOf84r5mtFN3PZpt2OM1aJvJZh1Ukfh2NP8PKx8ySRQCxA9+oyao6vdCa58tMk87ucdOePwrpdKgNnaAznceCG7HP8ASuOK1OmWkR968ZkMZ+Xym2luvy4J/nTlnFtKNykEl8egz09ua57ULlpdQFuPus4BI5OScHPoKtIbi8iljAOYuFA9evHf1qkm3oTolqb32oTTWzZw0ikEf7QIPP1qtfI9vGFQFsKWH45IGPxpLVo474goQSVb6ZGD/wCPU/W3jVGlZsED5cHp3FW/MjrZHmt9ema4EUvEhJAHbJ6fqDXY6Y2YUAOVb5go65HX8K4XUvKub1rqFtz5JwO3A4H410miXJa3Dn5cJkL3z6VLWxq9jrr25RN6hs8bfXjI/WuUhs/tEjXtwB9nViFT+8x/ngUs140hECnLs3bng5q2NrQwo+T5QBKg9c54+projHqzmbsUkjaO5a4tTvu7gqMjkxqOMIF9up4rrLTSpLmNWuGfc5O0OFI/Uc+5zgdBz0qWJSHdPMDGeXcrxwBgD8+n09q6SwuBLsuCvlr0T+8wHoTyBXfRmc1S/Qzp/DEUtu1xf3DyTKQI0hXa68cBWGCPyrgPEWi3VjCUsZ3jveHEarnzQOPnAzggf3fvHrnmvclmhwGuQHC9IxyXJ7HP/wBYV5d4xb+1LecpMXbOEgsEyPlPCNJuTdz15Ar0lytaHPGcr2Z4fp+s6nsktLpEuIELYN0vkFW4GEZXRcnPYqT6VvweKYbRZ7LxNo84gVUJmhUncT0AAJB7DGeMHr1rkNe8B+I9de2lvYpWkhZjKGCyO5GNigoNg6c88VDDpniLSk/sy9jmSNidgh3s1u548wN8oJx2J55JxxVtNf1/X5G6SZJr7eF/EF3KtlfzQ26H5zDM2BIv8DwzAj5Rx8revXNefNZ6le3git5/tMY+6NzDGPUOFwPpmqOv6b4ntbhIDPIVA2RRxOWURk/fZ/VvQ5JPWruk2N/aWbwvM5dyCzbiQB6c1MnZXOinE0JGnhBgJOQe/tXo3hm0ku4Htb0gIvzKeB06/rwM1wGkaPPPqii5dnjjBJQcZOeM8c8V7Vodt5LF3G6V+igcIteZiLJnQ37p1PwKsZbHUNetHwcGE7RzySf5Dgmv0T8HuY9MjRsDPAAAHA47dq+KPhpo8lq2pXcW7N1NGuR97CDn8MnrX274TsWtNPiLDkqOTnP0yf6D8TX0uUp8iZ8jm81Ks3/Wx1tFFFe0eQFFFFABRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigArN1gE6ZcADOUNaVMlQSRtGejDFTNXi0OLs0z88vGVlc/aZUAQRyk7lPcH615RqOjyGDChgwPynHIx0zzz9a+r/H3hwW95NEqfeywJxjB9civD7/7TZrsVRDg/eYjGB1I718BVpuM2mffYTEKUE0ct4N+E19rFyl/ffJDncMFhyPY4yP5V9P2OmQaPbpAIlBUDnuQOnP/ANavC9E8SalARFFfROwzgyPhf/QT/jXfQa9qzw7tQVSOzRnK/h616FCtCENFqceMjVqT956HT3WpurMqsR7Hkfga527upZPnBJ6npVmZopFE44DDr15rEu3Cjfnp+ePauLFVm9bhQppFGa5kQtkfK/A56Vi38rpGVjGCuCe9SzSmRiScZ/DNZ91cMVBJ+YHaccH/ACa8mc+Y9GMbHEXErJfgOdpkJBPp2Ga9EtHd9PRZOrDcc84H/wCuvOdTtZ7md2QYCr25OPX8D60nh7xLPcf8S28wOMehOKmETSeqOg0h5H8RtDL8+1xJgf3CwBJ/OvSYdMe1vLdI03ASukh9QjHj8M5rndDhgi8Sw3T8R3KyWjccBiMrz/wEYr3HStOWZPLbq+JQf9vaAx/EivZwGEVSNzzcXieRnid3pstlrkk3mfuWYoR/dPpz+B/GuU8U3TNB6ZHr6V6F46dbXU5geEnCHPTDL0P9Pwrz3UtNudQs/LY4ffvHsrZx+o/WvNxELVWkdlGV4qTOTsrYzwQmOTa27nHOQxH8q6jyYEt3uIyBGgCrjj5geSf89ax4YJIE2YP7pOe3OT/LpUP2nMUaxL8jMWAHPAJz+FZJO5pJ3NolVVJARuZCp9RkY/PrWjaQbcM5wEQkY7EjjPuaxrhJ2hDRrgF+P93j+dbTyC3eEEgAHzGHqccCuqN2kYSLVvIZBlvkXCrgjOMgA/jitV7hkCSxnduAC5+uOf8ACsCBWnMSFuZCWznso5PsOa0fMY+SsRG4jIHXqeP0NbRRjIuyTkxuZwWydiqM5dj16f5x6VesoNPhVVvz9ocEnltsYPZQq4zgfhWXEhXYCcKg69yT7e9UrmS6t5d6IT2T6nk9a6YTcdjJxvoegPf26lS3k2qAYCooU4zxjqf0Fc1repaS0DxmYIeeMAtyepDAj865yLzZiFQh5DycEkD1J/8Ar8V5/wCInmgcwxzF5Dn5t2FXngLxyffpW3tW9yYUlc89+IPmxyhhcquTu2vguw6A4AJHtnGfSuU8PsLkEzyZwdp962da0eW4AjjnSVi/ziMlsE+pbGD6muet7SfRrloG+bzeVx7d/pTc7xstzvp6aM9U064toPkiABAH1J7Zr0PSrOTyxc3TMo4bkYGR+PWvNvCemSzyCeToSG3YBP4civbfszqYY1UmMkZyOw7V5zjedhVqtlZHtXw40hp7aETzmNHbeRk7j+A/rX13aQQwQokK7VAGM8nH1r5y8C6c8t8hi2QqiAAsM7R32jp+J+g5NfScQ2xqASeOp619tgKajTsj4rFzcptj6KKK7zkCiiigAooooAKKKKACiiigD//X/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAPKfiDp0NzEZM4dOlfKHivTmZXEIChucjk5/Gvr7xrcIITgD5e9eBXNtBcS/v1LDnv/AEr5HMYp1pJH0mXTcaabPn/RPCV3dXzNcR7UXHTaCQfUY/8Ar16la6YumL5MUxkjxnnqB6HqDXXeVEkey1QKMcN7/WsKfadyP949RXmzXKen7RzepWkdVUiPkeg6evSs2SP7WV2dR07H/wCvVi4gkHMeRnv9arWl3HFNtvwSgP8AnHFcsnzOzNYqyujEu7G6hzuXjpz6ensRXPXSSRgrjGBnvyM17W9lp2p2vmwsA6jB75x2OK881pIU3RNwQMH29/pSxGF5FzJ6F0K/O7MoaXYx3Nv5pXJIK9DjB9a8k1G2GneLolRDsZwv4k//AFq+idDt/L0/KL0z+R615zrOird649xHzLBJEyDvtLAZ/AmiNPReY4z95o6uKwk/cQIuZYphIo9SQT+lfTFjZbY45gu1imcemR/SuTsvDcb6nBdYG2PqMZHzL/8AXr1CO2VIxH/dGPwr6vLMG6cW5HzuPxSnZI+cfFWmLPrUcc3zKcqdwyOexrnLnSzbIsMg5RcZHoOnP1Fe3a9pGZvOKkrnt2/ya4PxDCFjYxjDFe/9K8fGYVwlKTPSw+J5oxijwnUdkczxIu3JOfoxzUWm6W0ske/O1R2HQY5znvz+vtVi5KRXCMVIIyST04zzXe+HrMXarwAOO2R+NeZShzTsejUfLC5CmmgwRxui713O2TkjP3R+Vc7PDFLcOWj3IoHDcLx1JPU5xx7V3OtXttaZsrI7mA5GOWPcn1/lXCCK8lmlMrhjkBUXJA+uOv0reslB2iY0m5K7IFZpZGf5VVvvED+HsBWnE6uJGjXHXpycYx1p5sblgzyFE42/Pldv0Azz60FZLYeXbOX3/edRjP8Au9h7miMhyQIRDtHKhRk45JPHAqtLatesvnDG4846Adl+vrWgkiWq8KC/dmO7HsoFTRN55DsvA5Abj8hWikZNHPXljLCBb2rAxkcgZ+b6nso9PxrDuvCd5qakxys+7KkopX6jcckD3HJ9q9CnEPE0cSs7nuSefZafG91G224+RjjIPX8AKnnaY13PKn+HtpC6iNN+zqAensM+vfvW7aeEoY49pgQ5xuyM/wAxXp0ckbMFEeAPoD9eelWGtzjMakg9iarfW4nVZwlhoFnAxMW1JFz2B5+hrbjsZVnUyNnBHTp+lSzsUzwAyk85B/wqG3vvOuY7flixA4/+tSh8SIm21c+o/h5DboTLkyucHaoyAAOCT0Htk17WpyAa8t8D2XlqsTsWWJRhc8A9yQPy5r1OvusOrQR8hWd5MKKKK3MgooooAKKKKACiiigAooooA//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKa5+U/0p1Nf7pHrQwPIvGkB+zs0ORg5OTzXkJaJE8x8jPYn9a9h8YoUh/dsx7c9PevEJpI8iEpsKnORXx2NdqzPpMGr00X5HMdqHDAHtz1z71zN0yZzL+Z4/Wr107Ngswb0A5rIlkAmUuCM9Aozn8zXmVp3dj0aUeo6GESMZfMDBeuWwB75I/lSXdudm5tkg7YGT+dOutesNLtztT7RNgEKCAB6lm+6oFeL+Ivj3oNlcyQxtDdzxn5vKVpEX6udqfln2pqMbcq1fkVzS3PdbKRIbc7pSoboCp25/L+tec+I5gbxRy3bjkfpXzndftPRxT7LiJSvIzCsikj3JO38hiuj0n4xeDvFUyQtcGGRgB8wxg1rXw9XkXuu3oKjUipN3PrLw9AF0yNV+YngZ9DU+l+E2u9Zhu3Xd5TgPnuByPzPFc94U1QwwLaTkbVI2kdCuOCD+Ne+aBGrW4mGCWPUdwK78Dho1Gr9DixWIlTu11Ny1tFjxgY4H51rEYFMjUdae5ABHoK+nSsj55u7MXUY1eMqehrxbxLhQcjgZB/CvXdRuQiNXh3izUFjZmBPPbqTntXh5tZxPWy6/MfP3iS/EFywzhcE57ACu48K6ysOmhwTGAPmY8du3px+NeY6+yWxn1G5OYUIK7j0HXHPB54r5v8AEnxR1R7trDSpCQn8QJ2qT1wP6mvCo4ac5e4e7Uqx5bM+1J9b0yS5Ms1/HFnKhWcA7h1ODk/nV7StX0GZALa7EjgckFd2fYEg498V+b0fjrxa12sFjI00ruMAKDlugA/wrpP+Fn+KtLums9bsofPhO1w0YRlI90wf1rteWVl7yVzk+uU/hufopE+mToxEm/Pdcbvw3Lj9TVG9u9Hj2xrJI5XtINwyP9wgV8Y2Xxe/drJfQSxsoBDwyFW+pz8rfQj8a9c8N/EfUvFkQV4vMQHIm3YbA6cco30IrCVGaTvE1hJN6M9be/lllyJMjsuCn6d66K0UvGC7Mm7nIAHH868vj1u1j+a4wpHLYUJU6+OLSBglpl29Sefzx1rzlLleh0yg2j2W1gtLL98AzOc43N/QVJK9tg7l2N3APLfXH8s15Gnji7aQbtoJHAyCR7nFbEHiKW9GHcIWHRVOfxOMfzrdVU9DB05J3OnS/EUhiBGOe3f8P8avxXSsDucrnuAST9BjAriWX5PO3YXH/LPPftzjn8atW+pRsnlCXjcQATnIHHP/AOupUmnqOUU0dVcp5illJB7bufzrK0G1kGsxyEkhTxsGD+tRtLbRqv2gk7uyOo/qa6Lw5brJqUY6AHIBJJA/CunDrmqRsc9V2gz6t8EbxbKGQru+bGPvH3PfH4CvRq4XwlZxwQiT5tzgdeM/hgH88/hXdV91S+FHyU9wooorQgKKKKACiiigAooooAKKKKAP/9H9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAqKVgqnNS1RvCQhpPYaPMvGE37rhTg8DsK+etUlMMjuCcLzkAnmvoDxax8n5evc9/oK8F1kGMSSK21hkgDknNfH5qrVHY+ky5+7Y8r1DXL2SVxZZ3DgK2efyBx+NQofFd+BGJYokyAxDHdj64/pUltaveXTLGzAk8vnHPsAea9d0pEjVWKscDBbA/XjivGoUFOWp7NSqoLRHlvijwrP8A8I0yzMzs+Nw5A54/zmvhPVrA2Wi3c7KN6yuv0IcqfyAGK/VbXtIj1TQrlLXhvLJXB6lecHqa/PnxxoYF5caPGnli93Tw9MOxP7xf94da9dxVBW6bnHCbrRfc+W9Y1PRbm10+LSjOZfs4a9aZEQC5LNlYdrNmMJtwzYYtngACqV3rdrNo9jpsFjDBc2sk0kl4hfz5vM27EfLbQsYX5QoBySSTVbWdIm0i+e2uFIwTjjH4VVt4TcSLBCuWbgDHNfUxrRac4bNfgeH7Np2e6P0G/Zh8ay+K9Pk8Ma45e7sFEsMjHl4iQMH1Kn9K++/CrNDD9mfPydPoTxX5P/BWZvCnxJ8MxEjN27wSgH+CRD1/ECv1k0oYKOo68V5uD5XNyhsb4xNQXMdyjAZx6VUuptqE5qVG+WsTVpmSAgd816knZHlxV2cPr2pPGvB5NeR6tDLqFyRuwqjn26/4V3mqt506555rH1S0WGylMY2syHn265z9ea8WvT57tnsUHyWSPir47619hthptp8rPmRgP7qD+XpXzHptqtrpM2oOvmsE3nPGSeTzXs3xavDqniy4UEyIuIx6Y6kD615RpEJkt59LkP7xASAehTtiudR5KN13VzumnzW8jzSHUJYrmO8G1pI3DgMoK/Kc4Kngj1HcVY1vXNR8Rarc61qjK1xdSNK+xFiQFyWIVEAVVGeFUAAcCnanpU+lXBEiHYeVbHY1TgilvJlgt1MjscADmvejVi4Xi9NzxnT967Wp6N4UtZtSsH/eiIFWUk8ZXByCfQ19ifBbwzBF4Tt5TFucRiU7x0LuSMjPPH5V8++EfD0sKw6JEP8ASZ1BkI5Ecf8AExHqei+pPtX6N+APCsei+G1S5hbdJ8zKMfKB90fUAV4Ln7VyUdm/wPUS9nFSZwkvhC3u5WkMeR1BGCgz6Z//AF1xureDdPthuhiecrnCkhFH5DNen63fxqzERkKp6ucDr0HTNcjJrGZvsu1W3Y3KcHj0AGa8ao4KVkd0JTtc8ourMRNssYVjY5zsO4D23HFZUV5fWcxWQtuHT5gT19icV7zfaakluWFkhBGdxHA9MDOa8p1p49PhkkW02c5O05J/76/pSdEuNZPQ6uwvWmjRpR5hA53nJH41biMD5SEqSpyVPHXtgf415dpHia3mfyYIPJz1YMOD7g8A/Su9h1iNgInnWTH96Rdw/LNZvTRiaOhY3NvGXhQomMcbf55zXY+CLmd9Sg4KZ7g5z+Irzldc0+NgId0znjBZiP0BFen+CtUN1qEZiVYsdTg9fxArqwlnVic2IuqbPsTwpBdiDzHhMUbfxEjc34dh9ea7ivPvCchlAWSYOV/hXJ/E+leg191T+E+QnuFFFFaEhRRRQAUUUUAFFFFABRRRQB//0v1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACoZoxIuKmooBHnviawtorKWaUc9vrXzn4g3xK46s2Tg4HPv3r601ezjuIw0kfmFfuj0z/WvnbxhpgSZlZPlYfd/wAa+Zzmm17yPcy2or2Z4DpVxGdUeyaWISMdwVVzge7cAfmc1ozXN5pF6YjIyK2MED5SPqKzpNIgsdZW4s1CEn5mAzj+f6YrotTto71BJICCvdh/9fvXz0oNw03R7vMlLyZ0ejeKnKm01FswNwGZgMH8eo9a8e+JHgCHURJdQJ9ps5X3hocmSGTu8Z7/AExzW1dWk0CEwKsnfHXP4VvaV4guLVRbztuTHzRvjb9ARjFaUsW2lGp06h7LlblDr0PirxF4VaVjDqFmNRU8CaEgOf8AejcrtP0JFYeneHLTThjStCnaUnBeYxxqD6El+BX6Fz2fgjVjuvbUI0hxnaCD+IGKuwfDrwJMyz2llE7j5i5Ab68dM1301zLlg9Oyb/I55ezT5pJ/cfDHgnwlqw8Z6d4jvmR/s0yuwj+4o6ABj1xnsOa/UXQZhNYxyDuq9K+XfFOmm21QfZbdUSE4wvAAH06/hX1R4aiEGj2qydREufrjn8q9LK6rnUd1a2hzZrCMaUXHqdQvMYx1xWBrOfL2ryK0ROVYqvPFZl5i4jIfo1e1PVHiQVnc8uuIZftrOQSAQM/WszxbdTQaTK6DjB5HOPpXX6jGEO7bjHX1rhtdt5L3TJIZG+RyowBggE/0/WvMrxtFpHpUmnOLZ8P+IPDM+satcy7uGcbRz1x2NcrdfDrUopvtjC4t3QfJMib1Ge5wMlfUYr6DsNLkXxHdaXKdht5sgHjj6fka+j9HtojpyNfxgov3tuDn9OteZSqTb5EexiHBK7Vz8210XVArWGo21vqHP31doz+TIDW5oXw91a6n8vRtOW2aT+PmVh7rwoB9zn6V+gV1Po1smYbVbl1ySCgDAE8HGMn69qoPr1vbWYWygigYnnK7frjI5NYVJ04XX+f5bGMUpaqL/A8v8B/Dix8JL9t1cqkmd7b23M7gdXbufQDAHau68S/EG28kWOlkxbcgnzC2c9RtX5fzNc/qcySsJ76XMg6AuCPqQv8AKsBIbSJPPt4DcEgYbb8uT2BbmvOq42esYM6VQi2pTIb7Ubq5jEkvzSDuw5x7AZ/MmptEtY7MC4chmc5Cqfm/EVQFvd3VyxkOxF6jAOOenr+dWr+5i06MzK6wxqPvTBgTn0xWNODfvMc5fZR15mSYGWdTx93c24fp0/OuQ16zt5o2a8yARxiQjj22jj8TVDS/EUd4+UkQM2MZUnOO4yMYrpp4ru8GQx5HHOAffAwRXbGSasc3K4s8UvdDWc+dZsIgTnEyqc+nzAg8/SrVho2oqMm3THTOQBj1BzXpF3pdykS+c25vYlwB6kHmmSzW9vAkUywXB/uSRl+fbjioku5upvoZljoMi7ZLqTYAMBQRn9OtezeCrZ7e5TycjaO/615rpkFu0vmrbR2xY56dvrgY/HNe3eG1MKggbzj1H9K3wME6qZy4ubUGj27wvrrC+WzVERRj7uMlvfuTXs6klQTXiHhKWOG7yyrvbAyw5H+78pGT6mvbYyCgI6e9faUG2tT5SokmPooorczCiiigAooooAKKKKACiiigD//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAI5V3IR3Irxjxnp0bltmWYjOa9rIzXMaxpC3VvI7tl3HX0HtXn5jh3Vp2R14SryTufEes2Est2FDnZnpjjjtUes6jp1hCiXVwVAHOR0Hpn/AArvPF2kz2F08XK9SCK8E8RW8kiMJGCDoDyTkemOc/TH1r4uUnFOJ9ZTSnaR0X9oW0A3xuXB5UxnqPfioIb61uX25ckHJG0nv2JAFcTp93qj2eLqFo4Y/uyNjc/bkc/1+tdboiRXimSAYQfebG0k+xwM1xuMuax06JXOu0/THusRwOAp6BnzjPsB1H1r2Xw94afSrUyR8u464JH4lu35D2rM8F6fYReXIRMzOMBucYPuGIr2eCzjZf3mTgY5Oa+qyzL0o+0lueDj8a78i2PLJ/Dh1K4V7lck8qMdh3P1Pau8hiSIR2yfKFHStmeFI4zjgjpXOfaAt2hA7f5FezSoRpXa6nnTrSq2T6G5thjQ4A9v6VkXTxmPeuOmc/Wpb2VlXCYySOPY1iXNztUkEHcfm7Be38q0lImEOpyWrSckAkfz/nXN+V9ttDEVww5/Kn6/eiPLyHksOO+O2fQVT0557q4SS3kGwfe759sDv9K85zvOx6Cg1C50vh/wJp9tC018guJ3eRw7gbgJCCV3d8dB3xW7NoqxM4t1+VhjB4+tb1ioSNS2V6fnWuio3TJ+tdP1eFrJHE8RO+rPD77w9eQXHmQLuLD+LnPGM4PPHf1rzzU4VhkxeReU6EjLIXUjPHfAz9BX1NeRRsCEC5HUEc15D4nlTzNlzbouPlEgYbwPUgbTj868LHZfGKcos9XCY1ydmjwm/klhQzRQMwOdoSNdpwepPWuZfxDdTstsYwijqCPmXHoWIH4ium1qaztGB84sjEgHC5/DKgiuTuLVp22wRGUdc7tpGeeg6/nXz86bjLU9hSTRv/bJbaEH7SuOuFAc59toGK8+11b7VJTPCAXTplXf8SckA/pXW2WmXBA89iR/dXgjPsf/AK9dFaaQQplSPJHOR8vH4E/pXRTTMHJJniNvdahYF47mEKc8tna36Cu60vWdRt1T908i5xw+7j24/pXeTabHIux7VAPVuSPwbIrKXwuZXzEvkr0ygUNj/eHP60Sg90UqkXuE1ydQVAeQTn/aX8QMit/TdJgVFe7m3dcCT5m/PH9auaT4TtbZ/PJdjnJJxn8a6wW6uBHERj0OK1hSk9zGdVLSJmWemws4+zKeec7q9B0WEwyKjgD6c1iWsFvEvzIQ3+fStvTnKSBuQB2r1MJTUZJnnYio5Kx32m3z2eoKs0m1X4AGBn8e34V7/p063NokqABSOMcivmWZBI8cx6gjI45r3nwzqUl1ZxiX5R0HGBx2HTOPYYHrX0OHlq0zxa0TrqKKK7DmCiiigAooooAKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACijpyaj81T9z5vpzQBJUcqb1Ipu6Y/dUL9T/Qf40hSYj5pAPoP8c0mNHkfjnw1LcQNdRgNIK+atR8OW89yJZ0JYnlR/WvuG6to54yjSySZ6gf/AFhXinijQGs7g3UUDbepJC8fzr5nMcvjGftY7dT3MDjXy+zZ4M+jozYkzsThU7n8O31NWrewRZFaCIjtjOE/kK71zZcyrD5hPoq8/iQKz3vNJhP/ACD8sDnIfbz9Oc/lXN9Xho7nV7eW1juvCumrHbiWYgluuzAH6V6IsqRKB0FefeFrp75SII5ERf7xXA/EAGu5XMQEcq7s+2a+jwyXIuU8bENubuWpgWTPSuLnZINRVXYbSCfxNdSZxKWReNvWvDPiJ4hufD2rRRDG25GYy3+z1FViKihHmZpgqDq1PZxPQr3UIWYDdgk8DPXFYl5eQxQku4AH3vXjjrXk58cGQZlGGTpj1PWuW1nxhcyxeXEcJzwOp/8ArV588bHc92nk9XRMteJ9Ua71FDbEKm4/e4HzDGTz/OvUvCdnYQRBrYqWflmBzyB1z/gK+LvEHxEnSX7LprxXN2x4UfPtwOWOz09K+qfgzb3Nv4UtW1An7RIN77jk5IyefXP0x0rDDTbnfuLMKPs6fL2PdI2JQZ4HWq8+rR28ywMj/OeMLkfy4qit+8kzQwI+U6tt4P0J61JFNrRkYzIqx/wkcnHvnvXq8x4HL3NJ7pGXjPPrXKaxBbX8DxnI4wGTBI9xnNWjpl3fD/Trggbs7UAXIznHU1g63Y6nEc6dcYRR9w/Ln8a5MQ3yu6NqSSejPNNW0vVbJ3CXTXELdFZeMehU8Vy7WkjPtNnsHfAwPy5FdPc3lyNxuZYgwJ48xzz68cViGYufkeDPfnP8xXzdRRvorHrxlKwyKwES5ZGAPOMFR+gpzTKM+VG4PuDUkMNzMR+8gAGeyd/wrZt9NuOu1MjnKhMH8qhR7Dcu5nW2JF/eRjjqc8/rWvHDC46KfXjB/MVOsMqDcVwfoMfhUbPMGG5VQ9N23GfrxWsVYhyuNKjA8uQg+mOv5VIkQdMrnI74qSOIOf3hGT3FX1gfdlX/AMa2jEzlIzhJIMCVmA+hre02SEMOSfqaqbvL+WQBj7Zq3Es24OB8p9RXVSVncxm7o7Zo1nth0yBXongeDAy6KCeN2WLN7AHov0A+teb6ZcsqeXMCVP6V3Hh/XrfSrr7LP9x/utnHJ7EngfXrXs0mrqTPNqReqR7TRVe2m86MOOnr0z9ParFeicIUUUUAFFFFABRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigBCQKQ7j04p2AKKAGeWv8AF831pdwHAGfpS7c9aWgYzDt32/qaaIh1PJ9+alqBpiWMcI3MOueg+v8AhSduoIkJVFyxAArC1a1+327xJGcMOrfKP6k/lW0kIB3uS7ep7fQdqSSUHcqY+Xqx6ConHmVpFRlZ3R88y6Z/Z9xJB5atzxkkj8uKoTWtwp3RmKL0wij9SP8AGvTNd0txIblUwGPU8Mf8B7VxU8e0kdTXluioLlPSVXm1JdCe+iRlMoK+u3H+Gfyrak1S7hUgRbj6+v4Vx7PcQsohJDdgP5nsBV6LUruMbCPNbHJHU1rTq2XKROF3c3v7dBYpNCyYGc//AKq8k+Jnhm08Z6dhXMdxFny2BIK56ke/pXXXPibYVEsDBDxwOCa53VNesJi1ugdGQZYnj/PH4Ac1OIqRlBps1wynCanFHxmkuteGrWbStWleaS2P7uST7zJnGCe+PWuR1zX5NX0maxWUxmQbXKnBI7j8a978bJo2twrFJukaQlUYAjb7k8cCvmiLTLDS/FUcV7KZLZmJG7uU5BNeGopyufZUcdenaS1PUfh18ONNtbJLqeIok+Nx7EA5x6g5HXNfWelXrxR+d5LCNR0UAuw/+vXgtprUjuHtYWKxxqVGVCtnnp0H6V3Np4yvliRI4DuYgbEC4Hbk5/lWlOryy5pHi4pSqHqsfiDxDdxILHTtqscFpHwVA9VI5NWZW8Ru4InWNOpHUflXmv8AwlHiKdlaxljhA+8r8tj2we1XHvtZuSv22+2oRjKgEc9+1dX1uLR57oNPZHoBXVJjie9TylGdqjaSfqSax7m80u0Yu11uc/7W7n0xxXFTwWkWTd3TTKnUhiSPwrOub2NxsSITQcfP/EPr3/Aiuari/IuNE0bu4srqfM8Ko3Y7SufzK/oTWfPZ2G7OGjPX723j/gagf+PUkM9vGuxHMat64Cn+aH8QKtq6oAjDaGPBX5Mn6cofwxXA5c250JWIY7SWPDRSfIe7pkf99LuFaEcN1gFQHHrG2f5H+lLDZlPmhOc/3fkcfh0P4GtJGdMNJhwOpxhh9e4qoxFKRVW08wgSnBPcirrWpiTdksvcZ/l2NTfasx/u2ORyVb5hVJrlFb5Djn8PyrZKKM7tiHaBmL8j2/GhAr8kEZPQ9KWUoy+bGwYfTkexqh9p2/Iw3c8EcY/DpTvYRr+fEo8tMbvftWlZOXx5nJrnhH5uGKhsfhVqK4aNsFCuPxropy1uZTid5BIQAQoWr88giEd5tG5SDnk59q5OzvpZBtAb8Oa7CyMVxAYXZskcgivTpy5lY45qz1PbPCurrq2nrIiFNvB9Pw/wrqK8I8H6kdH1c2LysI5zgf3dw6ce9e6qwdQw716NCfNHXc4asLS0HUUUVsZBRRRQAUUUUAFFFFAH/9b9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAoopCM8HpQBGwaQ7VO1e57n6U9VVFCqMAU6msSTtH40hkcjFsgHao6n+gqP5VUPJ8qj7q/wD1vWkZgSoQbsfdUd/c+1DqsSm4mOWHf69lHvUsoo34M0ewqSW6IPvH6nsK8d1YixuDEcb888cD6V7OEMcL3F1+7BGW57emf85rw7xfqUElypX5MHoPT0rixklGPMzqwsXKXKilLOoTbGMs361kTXT2wKxncT99vX2HsP1NSefuGd2MelUZW5yOQK8+dRvY7lCxmaneX7jELhdvLHGT/uj+tecarquqQOZHUMmCPlHJrvrmRzgY9cdhz3rFmjjdSjLkH2rgrTk3udVKy6HgOsz6zdwzSRQiJpF2lu+D2UV51c+CdQeaK5vXDXEozjrjjpn1GOfSvqqXSo5c7F5Jqj/Z0MDtLMu9lGBx0H90VnGdjrjVPnHSdG1y3uJEs5CXBCqh+6wxlRz6811VrJrke6aIjkD5WH+fpXoj6couwuBkLuz/ALdTNZJNllGOc8+//wBfrUTqXLc0zjYZdaxHM6gB+GHQ5z1/KurspblY1jP3R69j6/SrkcDQoUl+7nH0I6VnXOqRwqQq4J4weP1FYuXYzbNln8va4b5W6Z52nuvuPT2qi17bl2MJ8uSPqB/Me1csNVe7kZW/1UgGO+G7H8DTLUXPnb3Pzp07/h/ntUO5J18U8c5ZWG3d1I6fXH+Fa1t5tuQmeG5x1U4rmoy6tvg+63+SK1ILiYy7GGOMAduKcUJs66O4idv+eZHUdVIPXg1YaaSEgSZZBx6jHseufxrmnRjgqSCR19D7+1XIZpyOchuOR7eorVSZm0XC+9iYGzjt/hUWS3z55HFJEke7cp2P1xVpBC52PhvQjgj/ABpqIFZXYNhBtJ64rShthLgnj3Ax+YpVsWjA3nK9iCCKvoiqo2sHz+YreEO5nKXYlhthF941rxW3ngDBOe9QWtszYOc/pW3BDsIHeu+lA5akhINOSP7oPHatKImDDBSv41LGCRgnP1qfZhcHIFd0YpbHNJ33I5EimdLnJDKeq9a928P6gb2wjMhJbHUjGR9eQf8AOQK8LQRjMY6n1PFeg+C9QVZWsmXY3UAHr749fXH4iuik7S9TCqrxPUaKB0orsOUKKKKACiiigAooooA//9f9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooADVWZ2yIIxl25PoB6mppH2DIGSeAPU01EESlnPzNyx/z2pPsNDo4xGOuSep9apGWMyNczsFjiyFz0z3P9B+Nc34k8WW+jQMF3PIeBhTx6nPtXzP4s+J+r3ebG1l8kvgKAv3B+fWvPxeYUaCtI7cNgqlbVHuHirxnaws0TS7Y4xnaCBk9s/4V8+3WsNrGoNdE/JuwB1yfXA615Fd6xf3twz+Y+1iAoJC+wzk9+pP1rq9Gv4XQOju0ScEqMZ+nTOf0/Svl8Vmcq8lfRH0GHwCox8z1uwjijjEW1pJX+bLdAB3OeM/n9KvtFztUdB26fmawdF+23mREggSQ7nLdWC8AZPQevb0Ga63y1jwu7e3oBgf48V6mH96CaOKtpIwJLPqcVnS2y8jH5V1jx7ztH5VDJaYHA5q5UbmSmcXLAVXEYAz3NZ0tqSp3HiuzltEU5K7jWNdwuynoormnRsaxqHE3SxRnccZFQx27SKZApAPTtWs9mJZxHGC5PftXQLphWIBuPSsI0WzV1LHBzwHaQe45/wAa4XWIhGA5OdxIA+nWvXr2w2qSDxXnGuWbSTC1jYKz4LH+6o6/nWcoa2LjK5SstKJhQ4wT0+lbdvpJIG0YwME/TpXY6fou+OOVRhWAAz6f/XrSmsBbHbjKk801h3a7JdRbHGafppTNu68A5H49a05rEK6PGvCnk100lhmQTJxjg4rQSyQxgN1atY0OhDqnLpY+egHQ9anEAtyEmQD0btW7Havb53DoP61PJapdQYPKtVqiS6hmNYQ3CAgDNKNFglXLEq3qKnhtLmzwo+ZfQ1rxM7KGIrWNNPdESk+jMAaVJBxFcZHcEVehsn3AsM+4wK3YlDkbhiriWidfu1tGguhlKozPigRCOTWjGoKgKc49RUiRFM7SM1MmBw4x9OK6YRsZNlmGMkDB5q0Y5FXpuHoDg1HDk/dbcP1q6FLDDHB/KuiK0MJGU4YHK7gB2atTTpFjuY2mJUZGGB5U9iKhljfHzZYeoNQxMw4PGOmapB0PfrKfzoVJIY46qeKuVz3h68S5skyVLAAAqc8fjyPpXQ13Rd1c4mrMKKKKYgooooAKKKKAP//Q/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooqrcXlvaoXmcKB6nFJtLcaTexapGZUUs5wB1JrzHXviZpOkq3luGI9AWyfoBXh+vfGnVb9HisIQir13KT+YJrzMTm+Go6OV35HfQy2vV2VkfS2o+KNI01Gmup1UqOBkcD/GvD/FXxngBez04cgffJ4H0x1rwG/8Q6lq05uLoZ2jOMlAT24H51nwuCw/djnPIyR6/ebNfOYvP6tT3aWiPcw+TU4e9U1Oj1DXr3UZPt158yIM4PUntgH8K465u2kEr3DyFnYqNuOvU+/tV6a5txH5kqlt5zjPYdPzzmsLUb22gIjSInYdoA4AY9T6V406jk9WepCCWyM2Qrcb1hBfGVXryz9Tx7Aiuo0aWZFS3x0+6vRj7nsBXMKzGzVSAqszMwHUjA6nsK3vD7xQzM0Z3SMcZHRe2fr6VKKlsexaLqUwQ2cIIUY3lfvu3pu7D6c/SuutBc3WSHCx5424xj0XH6nmuDsLJUaMNlA3ylR1Oe3HPPf19a9HilihiSCABCgxjoEHfJHf/wDUK+iwMpNWm9jxsUkneK3NeC1VFAxz70+SFB93k+tEO5UAf5e54/n7+1TMwAGOvpXuRSseTK9zGnthjJ61iz2bSfKB/hXViAk7n6mq9wiouAMfzrOVJPcqM7HNW1gsR+Ufj61cls3K8DNbNtbNjzHH4VNLEzdeB6UKirA6mpwl7p/loWOXduFUdz7VwmtaUtnCsAG+5u3VCfQE817Q1sFDXMvHYZ7CuYi00XetRTSDhDkVzVMOrqxtTq23NK3sFht1UjGwAfSm3tsiWzDGWxxn1rqGhVR83T0rPmtvOJZunat5UrKyMVUu7mJBZgICef8AOaYkQW4eMjiM5B9mreWDbGo6Fev0FUJUAnbjnA/WodOyRalcrMmWw2BtP5ioo7dYgVHCnt71flRmUnutKIgycjg8fSly6hfQorHuO09quJDGORz9KcIjuJPBFTLEcF1OR3FUoktiLEpGTz9P60pG0ZGRj8aa4OcZwaZuZDg8eo7VexJZQ7scA4q6qJ3z+FU4wfoTV2OQ4wwx+taxfciRMiRjBP5irOExnPHrUajseM+lSHK+/wChrVGbK7oDnaA30ODVOMAOQDz6E1bkIIJBGfcVm/vDKMqM9ip/pSbGj0Dwrqht5vs8oAQnB3Njk9COMc9DzzXqgOa8a8PJMmoBWyC/r0b2I6H/ADg17BAgSIKo2jsPT2rqot21OaqknoS0UUVsZBRRRQAUUUUAf//R/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACmPJHEpeRgoHUmsfWdfsNEt2nu3AwOB618x+Nfirf6p5lpYr5MI7jPzV5mOzWjhlZ6y7HdhMBUrvTbuexeKfijoOjBreKYSSDrt5r5o8SfEnXtcmkj092WIepwfy61xrW97cM11cJ52exPr7mswxmGfYgRPZTz+OetfHYzNcRX0k7LyPqMLltGjtqy9CLy5YyXMjeZ6nJ+mBmnXFpjJkDFR0y/PT0q7DGJkxg78YzuwPxwavwrbRZV9uS3Uct/XFeeo6bna3ZnN+RJHEAm2IH5mySWPp+lEFsDG0mNxGR3Ix3+g9TW3L++y4jOGJ5f07Yz/hVKSM7WEzgonqcjI9ccfhSUNbj5uhRnd2VQo2+WvRMcn/erjbrbkzSssarkFiSx9Sff866m7lcIdiMcqAM/KB6/h+Nci0aSF2275cfKF5Vcep6D6CmwSHGI3Wz+GCMk+WcjHTBf1b0BNXtIuo0kIU7Qp4749/rnp7/SqcqPgvIwVd3zFccsewx1J/SqVr5oljjjb5Q2QvU8dzn/AD6VSYmj6S0ScR2P2ll/ePwOcnGOpP8An8a2NMvLqWQRpgKDkDHft16n0rgNAvWuLVUkwiqOSTknPcn0z1I+grprS6hhm+0EkAdz94A/3ey7vXsPevWw9bWOuh5tWlvoekfaGgAhyWYdhzg/zJ7f54uRkx7WuPvt91B/WuMgvpDulXCJEMsx+4mexPVm9h+PpXRWAPmYQEuBlyx59h7Dua96jX5tjyqtKy1N7qOeT3pIrXzH8yUfQU63AlYqhyBWmgUERiu+KvqcUtCMQoo6VHJEuNxFaOwd6gkTNaNGdzBuIjJy3QdB70y1sSkwbHOOa2vIy3ParEaAAn1qFBN3K57KyMtoNxIIzWfcMY8ZHFdDInU/lWPcW/mgpnjOamcdNBxfcz92XaMDoM59qaLPcd/8Qq8lu6Mrt34qcJj5ug71nyX3L5uxkSRkZGODUcce1ijDjofoRxWsyK/H51D5Q6N1X+lLlHzFFojkEdB0P9DUghIO5O/UVaVMNg9DzUwXAx6UKInIymjRhtXgjsf6VEsJU5XtWmYBISfTv7VKIVPXgjvRyXDmKMZJGCuQaspCD8yHj0zzT/JZCQg+o/qKN5X5iMr3x1qkrbibvsOVWA55H608uQNrDingq+Np696hfcowfzFWQU5WUg7QRj/PSswXLb9u5XA9cZFWrqcpk7d304P+FZTyQu4mkj2jPBweP6j+VZSlqaxR6P4c+d0aRsoCDkE8duR6V69GNqgV4v4entjMgDbS3Gc8H+ley2+fKXJzxXfRehx1tyeiiitjEKKKKACiiigD/9L9UqKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAriPFPjjSvDsRRpFknP8ACG+79az/AB34zg0KxeC3IaZhj2H/ANevknVNTv8AV7kM6Ha3TjP4/WvnM2zr2T9jQ1fV9j2cuyz2v7ypsdF4m8TXviC4eZWwh6Dt/wDXrjxbys3zISAeDj/CplgW3JaWRiwxxn1/lTvtZjbaIjyPvHpXyE5OUuab1Pp6cVFcsEV57G4kQebnBzgMP554FUzYK75j4Zf8/Sqt/qEpl8rpgdeh/PkVY09S0XmeTuychiMk596zTTexrZpFlNMypZnMzr/tAD/CpY7GdJDIWWLGcbckir6Rkk7js9hjGPqxwKoXUyBfLRTI2T1fj9MA1o4pamd29ChlhON++bHO4kk59AOBUTshnUC3Z5emAMbR7iqa3Lgnc6x5PVMbhj/PrWvbIJyWZHbsC7bQfXnv+FELNFS0Od1Zd8YEo8sHnbgMf0P86wigUrDAC2QNzOcAH0CL1/H9a7DUbezjzJIFGOOrHHscn9BiuNuJZ7htkA8iJBgt32k9AB0z6Dk9zS2Y1sZjxSzXAi8whE3fe5b3OewH/wBarVrttr8IyiQt8scXJLMO745Cjr7/AErRKLbJvVQjYBO/kADpn1PsKyrGSRNQ807laRtpZx8/94sfQkc+1VEJHeWU7wzNHcAeYTlx1yMcL6D3rqbebzWMrD92gLqnZn/vH2WuMkASdZB845Az0Yn+fuTXRwXWYE89tmOScE7iOnHoOo7cVUJO5lJaHc6VK9xNClx8zDLIpwPm7yOOw/u/4129k32pdlkc2zH5nHWQ57e1eTvfbx/Z8LFJrsjz5By6oeFQH+8Rx+fpmvXdJdbYx21suNgCDP3UVcfmfU9M8V9Bl81LQ8jGQtqdZGv2ZBbxD5z+g9T71owxqg45J6msuGRTIVjJxnk9yR1P/wBatJJB/qxwF619DCx4syxj0pCnc0BxjI79KQuMhRWtzKxGUydtEhCY9qsDHWqsnzHPpSY0MkJxjvVPBaQ46VeAHSmiPBz61L1GUGySB+NI3y+4JyKshVZmI7GoJOAcfwmoKKO3DFenWnkg49e5/lTXDMSV+opURs4btUFDcgkL0owQ3XkfqKkeIHGOpJpwUN97jAp2AIwN3y8VIwXqOD6UnTgilLbxtxzVIlkTqGHHIH5g1ErqSQetSHI5H5VVkAchkJ3e3X8qljQ+RAvK8A1SkeSPljkepqXzWYfKfriovm+8mSO+P6j/AAqG+xaRVkYMpypx1yvI/KsiSIEF4JDGTnjPH5VsMARlBjPpx+orOvFmEZZDz9A2azki4mjpc6uO6uvccV79o00stojO24FQQSMH/wCvXzNpl4qzBWbBY7fYEfWvonwnPHPpMZQ/MvDD0NdeEncwxMbK509FFFdpxBRRRQAUUUUAf//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigArl/FHiW18O2LXEzLuI+UE9/pV3XddsNBspLu+lEYVSR3r4k8W+L7zxRqcmCWTPAJ7dif8K8POM0WHj7On8T/AAPUy3L3XlzS+FEniPxJLr109xLymTySAB9KwYb4sdiJkdyeuPxrO4+UzsPb0AHXA7D3rTgMKxh1xjj5cZz747/yr4fmbd5M+uUFFWS0NJo5powbceXGvBYABR+I4qg2mRiPzbiVkx1zgZ/rV5LiV8NyFQ98YH4dKytSvk25D7snqSBgfjVSUbXFHm2Mq5Fqv3jnHTnH8+taMDGYBidyjgbnIT8h/wDXrDQwSXSKxDhjwqLliP8APvXbWUFrDs3Y2ngLgM3P0JwKmmm2XN2QsEdsIt7RcjPJG0cHsCSf0rEvruMErtVQT/EefyP+FdzKhEe23iRFxwAOBn/PfmuZvbJlTaQuWPoP51rVg0rIypzu9TlLVfOuBIFRu67hwM988fov411SDYE2yKzN1252jHv3/OobXSbYSEySqMH5gMH9MY/PFbUMUJPmwBpPSV+B+A6fgOKdOOmo6kjltTtB5LlW+Zz8zHrjrxx0964wKHukhtznbk7VGW56sx7fj+Ar03VdPlnQuANgPLkkk8flXmV/JLF/olsdqEktjgE+5HJ/PjvWdRWlsXTd0VrphGCARLMCQBnhTkAe3A6CseeNpr6HyiW28swJwf8AZHY5PU/StG1s4mVsjOB8zDgENyQB2z3PJIqxIuJFRywOcKFBOF9SP0AJGT146i8hs6mzXcE+0Zc4HA6D/ZHrnuatCOe5lM0mdxbYBngEckfQDg/lU0I2ugUbEQBVycnheWPvzz6Us8saiOyQ/O2AAOe+efbPp1qktLGXU1fDiifUUnIyuSy+vPGT6E9B7fWvX4LlZHaK0O1Y8B5QMgHqFT1xnJ9WxXjenLHA0kMb7dp3O2eSPqOnp711theTW0W24OyMMSFHGWOcZ+g5Nelg63s9GcmJpc+p65pu5VMmMKq8kc4Hb6k960IrhFQb/lJ5I/z6VzVpfxiyRJX3STp5jgcEID8qgds55+uK2bdSz/apgFQkdeOnT8B1/D2r6alUTirHhVIWbubMUkjAyuNuecHsKsJwCx7Vh296t3KJfuxgblH+yP4z/T86sS38aYij+Z2BYD8M7j7V0KorXMJQd7GtvBJx3/pTJHVBj0qhbykRbmOWbGCe59celSRFXdpCc5OBVc1yHEshiy7iKfIdv4YpjSBRgdqqtKWUu3fHHtTuKw1jsVmH8RqIyDbgDkkflRPKqw575/Wos8YIxgVDZSRYhVc8c44qVo8AnHSooSFBPbBP5VYLDaSeOtUiWVyAOvSmOB1B5pk0yquCeCKqeeSwH8VS5IpJlsPkevtTCw/h6e/amFgenDVF5nzfz/8A1UmwsK7lTkDj+VQlwwJHOKSSZVqjLh8unbrt6j6iolItInlUsNykn19RVYybcEHPcEcYoVpWG+JskenWk3LMN5O1uhPSouUkKbnzFOQC3TPr9ayZpwARICh/T65qe5SWP5miyR3H/wBbism4u4dm2XKqffB/A1nOdty4x7Ebv5c4fuec9z9cdf1r3fwCUezMgIycjKk/kR0+hr5ze4jZtsE2/oQOjcfoa95+Gt2TC8Hl43c54z/n+VaYOd52JxMfcPWaKKK9c8sKKKKACiiigD//1P1SooooAKKKKACiiigAooooAKKKKACiiigAqC4nit4WlncRooyWJwAPXNPlljgjaWVgqKMknoBXyX8V/iDJeTS2FlKWt0yFUHhj0zgYz+NedmWYwwlPmereyOzBYOWInyrY5v4neOBq+rHTLKTz4g3XkL9fevN4woXbjbzz/tH3/wAKq6ZZTXI+07QsshPzOedvfA6DFakgit8kMGIwAwHA9ev86/PalSVSbqT6n2tKlGnFQj0GCJ9yMo+6OAe/19vatCB7VWMkwZwOw6k/lwPQVyj3xhk2QqZJJOdzDP8AOtC2v7mOEtNJub+6nQfjjr9BUQavoaSTOlaXz0xEghTHHGW/M9K5S83zTPFb89ieBgd8kZ4/WrzzXSxia+bYp+7CAN7E+pPT271mRaq8JVZrf5c8RoAQfXPBzj15rSWpES9p+m20lwH2+aSPuoSB6dckmvQ9Mso7cgGOG0D/AHfl8yQ/QAnmud0m5M/zSM1onoACcegxgA/rXbabKxO63ilKk/eLIg9TgjJP511Yanqn/X+ZhXm7W/r/ACNKa1WO3BkWWRSON+I1/pXFX6Pknaqgduo/Wu1lBmLeWgZmxjLF2AHuK5fUrEoSZsDnnHHP61ri46aGeHlrqYUUzg+QoQ4+YAYAHueBWxHIk8mLu4ds9ewx6ADt+nvVKKFUJWEcn/ZJx79q1rOxiE4e6GXYbiM5OB3bsK5qSkb1LFTVI0uItqgsgIwvb0/z3NcLqGmQq77tzseQTwAOnAHH9PavWpRCwAhiDLjA4647nP8A9asOezklLfZxg4JkfPIA9D/hVVqV3cmlVsrHlaW00CyNCBu/vMcbT7Z/njNVrKB5J5BayMzhvmKjJ49z+lbuqWy6a73DuDkdevHooH6mjTFk+zmRotkLnJA6yE92Pb2Fc67M3fcmNvLDbRPM/QYYZJGeu0HuP7x78Uy5doCGU4lGW47DtVq7nSNVjOJjGOeyj0VR9fzrN02P7Y7ySKNjOxc9SQvbd6MevsMCmmr2Ia0udHo1ukUQ8+TdJI4bHUe31J/zxmuxihju5P3TBtuQMnhc8lm9yBxXBpI0c+wOB/IZxk59cdvoPWvSNJsiLMQvkCQ5+bjK/wB5v89K7aGr5bHNW0XMRWrTxyrISSJGDFh1KqcKPZQefeukuNd88nI/chgqqT97bxk+2f0FU7wWywkwg4cdT7cAH+eKyEjDPukBYrztx0GOPpnrnsK7FUqU/cizm5Yz95o7aO/URGGNd807ZIPAA64+n9Kj04ZRmllLKxO+TvIerH2UdB7VgfvUiAwFONueg3HqPwHWte2XESoH4GAM8c+p/wAK7oVnJq/Q5J00k7HSi6Z9qRL878Aeg/8ArD+tX/OSJc8bYzx7k9K5kSrAi4yA2eT1IHU/jWfJc3F5KinKjeWx+GBmuv6zy77nN7C5082oqkIkByd2wf7TDqfoDViGYuxAPAA59BjNc0sMl1KigYjiLfQAdTWm7MEkVeDJgfRT1P5YrWFVvVmcqaWiJSzzSqDwuQfwq68iqxz0HP4AVSZuQg4IHP4/4Uk0qDnOdxx+FUpWJcbloXBS2dm6rj8z1p3nko4Y/d4/Mf41itdJG5VjlTgAevc1QnvjuMinK4Ofp7VPt0h+yOiYLNGI88/4io4gUYbhlen0rDg1RAEd2wcfnjity1uoZVAyCcU41Iy2FKDiWZBwADn0P0qpJMSwDdexp8kioD3U/wA/8ayppt5AUg81cpWJjEkeQsenTt/WjGRkcEfp+NSqoIy3U9D6fjUUjPEQ2M1HmV5AAW5Iww7jjNRyMXyG4cd+n59jUu+J1DJ39DVaZ2C7vvr0NDGZ1xNMg2n5R6jlf8/jWJPdW8wZZGBK9z8w/wCBd1+tatw3lbp4MOvcY6ewwePxrjtSuEu13wKFk5zkd/8AeGCP+BCuOtOx0U43K93LHDMqsuwnldp3D8j/AEr6H+FZZ4WZSNuM7SORnuPY18nbszCPJGD91s5H4HtX2D8MbM2+mJIVChlA4ORnr07GtstvKdyMdpCx6vRRRXvnihRRRQAUUUUAf//V/VKiiigAooooAKKKKACiiigAooooAKKCcVRv7pLW0knkfywo69efYetTKSim2NK7scD8RvEMdhpU2nwMDcOuSD0Vff3PavjO5DXV4XX5znJY9hXpnj3X4ry+Nra52gksWOWJ75rgIdyoEjPLHJ4r88zPFvE13J7I+yy7DKjS82CWowfMJ257jlj7+w9KwtVmVmJVuE4OBhR/ia3p51C+WDhh6HoP5Cse8SPYrL0XgADA/Adz715stdD0Y9zjUju7qbYVCKD0IOT9ea3XkMMaRI5WV+gXBx74AGTUsoeCNSx3buoX5QPfn+v5VRub2O32x26s8jDO8dh+X60r9EUacFooQSzRH/enfGD6n39qvqoK/Iy7W7r8xI/3R/U1kxzmFUZ4cZxt3AFvqAentnFbNlZXZ+afJMnO3IwB+FapENm7psd4oJVdg7Fhlj7egruNPg1FmWS8xEpIOXbIwPpXKW0H2cDdIqEH7qDcfxxwK63TnuQd22QDozuy5Gen3uB+Wa9DC6OzOKu7rQ3J1iMZd23Bhnp1+ijgfrXJ3lsWJkVfl9TwOa6tp7KAMrXDyyMOcHIz/vAD9K5TUWmkIMabQejHr+Ga6cXa1zHD3uY2145MGTYf9kY/KtbelhAsYGHc5yRzk9yT39MA+1Yq5SQsGBxwWHcn0PJrV0uCdpfNQEksSZGOMY64znGPWuCje9kdlTa7NJIQGSa/YgMCQhPp/sjOPqcmor6RXi8uBfkBzycfi39B19avratNJznYeWfucehPb/P0r3youLaBguTgnr+tdU4NRZzRkrnj2vPtm2YUk8sTnHX27ewp1le3KhI3YiTaZArcEJ03t/dX09eg9m+JGto5z5fDZPJG5sZ4IXpz6msjTFAmkeSJnJKuEI3NIw4VpOzbf4F+6OpzXlq3Mz0Ps3NW6lR0UIScg/OeOvUgfyqjbyyXDP8AZgyQWwwenXPAPr6n8q27izmlZ57puGB4zjGTyc+tUYrR5JVghP7odQBgAdTj0HqepqVfmHpY7mzs0ljinRR8pAAPGT1J98dq6eyE16v2gMy2+7AOfmfb1I9F4x9K422lmkKE/LEq4H8PA6k/Xkn0AxXplu8cemx7U4kwEUdSn07Z4HP6V7OFpqVzy8RNozmlVsmZPkJHlr+mT9T+lTyXHkKhiXdJI2WLenqfYDp+FTSwxoXnuyAU6KP4fQD3Oaw55JHnyUwRwF9O5/8Ar1rNunuZx940EuY1uxO/711+SCMdAc9QP5k966S3eFBsOHlbsPfjk/h2/wAa4GO5a3L4UtI/DOf5fl2HA+tb2m3MjI0mcg/eYHBJ/ur6ADjPQCtMNXV7E1qTtc27hlEyoxDsxIZh047D2FRwARysWPKqWYn+HJ4z7k9vSssalHHKwTBAA3N/Cg7KvqavytnyrVx88rbio6/ifYdK6YyjJ3MHFrQ14JGkjwnCuRj/AHBx+bNmr7N5Nu079Wxj6DpWatyqcnBxgceo4AHsP6GsbVtWkkim+baiZC+2B1/PmuqVWMI3Zzqm5SsX5dR8uN5V+Y8n3JPA/rVFr9pGdiTj7qjucdT+NcpFeyXN3HDED5MYUMP9rbwPwGPzNT3Bk8wJFyvVj6+34nrXnvEuWx1exUdzVEsu8SEH584z+VSkqIGiA5zjHuetLaIfsyzzZ3McAn070WrRsdo5JYE/yNaRX4mTZQj0q5kYKx4Q5H9a2o7KW0G8E5U5rWhlh8tmHPWlkYzAKvrj866YUYxV0YyqN7kIlaVDnPv/AIirMcMUgEg6t1x3I7/WlRFi6jkdakJjHzIcA10xXcyfkTBABjOaqt8pKHlfTqRSGZlPTg96Z56FsP1HT2/Gq5kJJkDwgBnXp3wcj6/55qi0kUQ8wnA9c8VduZBGdwYr7j/PNc3qE8gRpvLKnu0XRh7jpWFSSjsaQTZNctA585Hw7dcAEke471zl/wCWqsWIViOu3bn1yOQa5DUtVgik2TqYXB4LLtBP1U4NZa6+88ZimQFccFCTx9Tz+BrzamJTdmdsKLRbeOOfUoCrDaXAODxn19q+5vBlsIdHhVWJwBkNg/qOtfn7p8xfXEJBeLIwUIJ//WK+/PAhm/sSIyAqCOAR6ccEZGP5V7GWdjhzDZHcUUUV7J5AUUUUAFFFFAH/1v1SooooAKKKKACiiigAooooAKKKKAEIryz4oayumaQEyFZ84OefoP6mvSr29ttPtmurt9kadT/Qe9fGfxM8Y3HiHUHkiXbBb7ggPQKO59zXh55jI0qDpp+8z1Mrwzq1VK2iOEleB55L68O4t1Hp6AemamF0ot3mlATrgf41gWazT7JSpkZ+V3HAX3NWblI2iEpzjooH8XqfpXwvMz6/l6Fa6kiWMSEjHXc3Cgn27j61k/aJLwiGDdIzY+bGT9R6/wBKrTNczXBd9rZ6L6A9zWpZ7IlYZIQk5K/JuPueTj2BqEaNFpbVyhjchWPqQ2B6kf8A6vpWPd3kiSeQn3vpzn1P/wBc1XuNXlMn2S0G1RyNi8D8f69agWOdtimRd7dEHX6k8CnzXBRtuT29zEj+Y4adv4mYjb+A/qa7S0lkMcRR/J3dY40zKfrkYH61woL2jGV5QwHIAAAX6Dp9c10ejSSXLtKx3I5GQqE59B1yfzxVQepM0el6SbVY96AKqcfvCSc9Dwvf8a6gSQvtdIBxwCRtUfRRnr71y1k1yUUpG0Y6buARj6A4/Ct5NsW4PMAw5bqzj8zgV61GVlY86qrs12u4o9yQqpYew4H5cfjXK6jeSysWk+c9ieg/PGalnvjErRxRsN38TEAnNc7cISxMzYZf4eWPPr1x/Os8TWclZFUaSTuyncPICPLJJJwPXn07/lXU6XblYjNdSZhTkgnClh2OOuPQcZ61z1rE7S7ijFh2/iOffnA+nNdTBHPdyCW+A8qPHA+WNR6Z749Kyw0Nb9TWvLSxoxzSTIbm5XZABiJO7HscfyzWReSs2ZAOTngdB7D29TWvc3Cld7H5UAHA7dgB6n8653UbyRUYY2MB84HVF7AnsT6Ct8RNJWuY0otu9jzfUfMF1IW+aQ/OPRewwO5+vStLS/IhjYsd7knJY9SOMKOpA9fWs+9jR75Zp8hA28qM/NjoD3wPzNbCRF5Nyx5eUABQOQnUZ9AOpFefHV3R2y2sJeM9zjaCyLj2FZxE4jMUL73c8gdFGffHbk9810N5gcOwHXAXHygdTjpnHGT0rOEEsifuBsGMnAJ4/vHPUnoPepcHcFJWMyC68p40dyURtpPUuxOAB69PoBXoOi6yby2WcMFklO2MDkRoOB9WOD/M8V5lPaSrMcMd2SBn+AHgnP0/M1v6M8to6YUoFGEUjpnAGffHQds5NdGGruDsZVqSkj1W22uQ7NiOIE5OT9Tk9WJ7/lVO+Rty4B3uQqoBzj3NMaeSRktkf5wdzMOg44/+tU7vDEmVPlxL95+5HoD7160mpKx5yTTuY9xaTRSBCCGJ2oi8n8/5n61ZVLiGBbQKQwGWx79gP5Vu6dbsxM8KFGYZLt1Vew9QT/L3NbkdnHbqZIh88hA3t0/AfyFOlhL+8hVMRbQ5e1toNPi+03n3oxvWP0PYt6n+Z59K0LaZrWEXt4S1zMP3aD7w3cD8z/nilu7cNOI1b5Ac7j3Pc8/kPz7VjTyBkeZTjnJY8t0wAPw6f/XrRv2e3Qz+PU0Li6LSpYQsvmRgs5HQf/qrImYXqmyGGQKXZvUe59/6Vz8EspaSaQ7FlGCc87SeBn3/AJVuWCNb2MjyHL3ICqPQHP64rBVXUfkaOCgi3a2iCItH1Dlif1q0pTgMByCT9OgH4mq5vEjXy1/jyOOgUcH8TWZJKxjAHLE7ifbt/jWt4wWhi7yeptXMzSCO3U8DA49e9VS7WUSugxuIY59zgf402FsKhbqcGo5/NuI4oz3G7/vnpVt316k26G5bTxeWWQ8Nz+nFMi1ZmuHUfd4IPr7VnCMjYIjjKge2arrhJtvG0g5/r+NU6slYnkTOplvHnUPCfnHH1FVLXVI5TtfhumPUj+tc5NeGzlUZzFMOG9GHY1KphmQy7uTjd7HsT/jV/WG2T7JJHYLfREbE+Y91Pf6VRvL2ONcEfKc8ntXHSyTWsg3HhvuntUUmqzPmFzz0wx4P0Yf1pvFNqzF7HXQ1pNVSJC6uJkHOAfmUVkN4gtWBe3bYw4Kt3z79KxpkFxJvwUcenIP8qiuNPSZcg4fHOPlb/wCvWDqyextGEepXvtdtGR1eNm6/d5x9UbI/EcfSuammVQLjdhW6EKB+Py8fhVu90h4o/NUhWXkNjDY/rXGSmcHyXcbs/eAAz7Gud3clc6Eklod34Js9OvdVzcEFWb/dI9wfUdcd+3Nff/h6yistPjSBmKMq8Mc4OOoPoa+Hfhr4fvbzW7Z7GULKG5BGCQPzBFfeenQCC1ji2hdo6DpX0uWx0bPFzCWqRfooor1DzQooooAKKKKAP//X/VKiiigAooooAKKKKACiiigAooqnqFytpZyXD4CoCSW6Af1+lTKSim2NK7seIfFjxHJBALcv5UXIAB5c+v0r5Sc/2gzahetts4/uL/FKw6Z9h+prvviH4hOu6rJKXymdo4xwOOBXJJFBNJE33ghwq9Bnv/8Arr84x2I9vXlUvc+3wND2VFREsgWjae4Tg8HsMf3cfz9TTdR3SKCi+WG+7nk/gB+grWuLwISjISF6Dp+H9Sa52aRppnmuHLMw2fL0GeSAOw7Dua5GlsdavuYc9rcy4itxlW++R0z9e59uAO9W0svLjLSOHb0HOMerdB/KtuO1eWBYbdFVDxycE/gO3vV1dGhto1NwfOkbOFx8qj6dM/Wj2T3Y/aLY4iXfMuIIjj+HB2qR67jyR78ZqqiiPdKxHHBK8L/30eT9c/hXY3kMYG3diMnBA5LH0z6fp71nx2kEf72T92qHCrgZ/XpUcrvZFqWhg2+nJP8A6Tdx+ZHxjzDtXHYBOTj64z6V1emRkoGct6YxtH0XPIH5U2KREYlysRUfKoIZiT3Jx19eKlhe6k+feuwngtuJwfQYz+NCdnoJ6nW217PHEqIqh2OFznj0Ax1P6VuWM8iQGSZ1iVsbj0JJ6ALyxrJ00KCqru7bn+7ge3Uk1uSG3WIICsaAZ4wpP1J6fqT616lFWXNc4Ku9rFSa73MSkZcjOSflH9SP51z9zdzyuI7YKg9cd++B/wDXrXuBDJGCh2qBgdPzP/1zWU62yHAOR3PQH+p/zzWFZyelzWmkugtpckSGL77gYJ4O3146D3JrsrHddp5znaqjK57gd8Ht79TXI2sSykuzeXAnLM2ACB2A4AFdRa3TX6iOzj/dH+I8Z2+nr7D8cCt8K+jMq/kW5EH2Yyg5ZjkYAzn1HtXJXjSRt5eOhyecjPqSepr0SW0McH7wheP++cD+f9TXn+sRtFG0pXoMKDnH19zRjabjZsnDTT0PO9YkleTNsTuUgk4xtUn+Zx1POBXX2jrNZx4cRjIUEDJJHU4HU/oK4yd5I5HtY873+aR2P3PTkdz2A+tbmjXBugLSBfKigQZPUgHnkn+LA6du9cVJnZUWhtP5EgcRfMyjaSBwD2X3Pc44HfNPiiEcJ8sjL/ebOT+Z6n+VRxSRfZmMI226sQ2PvSE/wg9ceuOTSW264uPtCKQFO0eik9FUe3860nZSsZK9iyLHIWGJMySEHHUnHbnsO5NSz6e0GHfLHPBHAwep/PpWnZRP5Zhjb945O4ryce59P0ropLNfljPMjDGCc7V9T05rqp4bmjcwnWcWc9Z/NwQRCvUDguew/H9K6eyt7e9cXkwUxRDCZ+5kd1HcD17npVSa2hiVlwAgyDz1x1A+veo4tVTzI7KBDLcSsAqjhFC8kn0Ue1dlGKg+WZy1G5K8TqYZYolM92NqHlIx1Oe7e57DtUcl1b5+037bdnQDAVQOgHqfp9KzZ7iEgAyKVRhlUHVvc/zrCmuNy/a3IJ3fIT0A7ED+v5etdk6/IjnjS5jYvrtHcrIfLXALf7K44GOpPtXL3lzBP80g8u1Q5255YdyT79B7fhWXcaqqK8u7zHc9evzN3J9T+grKfzLmNY3YkDLMPTP82P6Z9q8ytieZ6HZToWL9u51B8sAAz/IMdcf4dAK2JrrdOI41+UHCjvxx/M1Db3DW+Z1RVaNSsX91MYG4/wAgPasCXU3RAIQS4DOCep2tx+ZNKDtEUldm7aypKZZclgAwX8Op/E1ppBmP5upxk/lXN2EzwWsKT/fbCnt0yxx+JArqY2b/AFOPuxoPxHJ/wropWa1MJ6MnX/VxepHP51a8j59g6Lx+eRVFMCdd3GP5VpW8wZC5PpXTGzMZFecbLdcdQP5f5zWPJMjSSleAc+5BH+FX9Rn2qyp2IIxWHHMkjEg4Y+vtWNWWti4LS5LFgqYLrHlSdGHQH1Hp71WYTWjlVOdvB+lXJkj2lwSufvr/AF/z25rEu2dR945XofasZaFrUmkvxgxvjb3HpjuKrSXKAYlXPo4/kaotcK/zPjf0PofQ57GqsjsrbofuE4I9D9KnnZXKaguo4wHVxn+7nB/D/Cl/tOwvgbecmOQcBtuMfWssfZn+8SCfamyx2RGXAZh68H8D1q1JhYz9UaeKN0Ys8f8AeXJBA9Qf6GuLjuPOuzBwQDwTnP455r0m3SMoRbz4LfwtyP1rPs9Nd9SPmRo27PQrkn1HQ1rCKbuDlpY9l+DXmm6J8rzI8gA7gQG7Yz0b05GelfXMX3ehH1rw34T6F/Z8LzMWV5eDkDa4H3WHHUchh9DXui9K+owUWqSueBi5XqOw6iiius5QooooAKKKKAP/0P1SooooAKKKKACiiigAooooAK8k+K+tPY6T9jjbb5vLegA9fqeAK9Zd1jQuxwAMmvkH4neJF1rWXsoW/dQt8xz1I9P6V4ueYn2eHcE9ZaHpZXQdSsn0R5XDEbt3cnapPLH19BWrFZmBo8DbxlR32+v4+9Z1vI10+6VTHDGcBVHXnp9T1J/CtG7uXAMiLyRwcEhQO1fDxStdn1zvsYWoyRRJvI+d8hQT2Hc1iDb5qlyxLfKicZbPU+2fXH0qO4e5uZ23sdqdAD2HTJ/pVrT0EbGTcWkfPz9So9j/AC9fpWSd2bWsjrUc2y759qM2FKry2R0XPapLy7f/AI94+XUZPPOT249uvpT7dQjRhPvL6/eJ7/T3NUrlmjbEimMscYUZPXoP6mt5uy0MUrszRsO4AGaX8cJ7f4k/QVTuYZJcJE5e4cY6AhM/XCgn1NK90tu+PLO92+SJR949ief896s6bDwZJQJp+vllsopP95gACfYCsormNG7ai6bpTR4e6myP4v8Alo5P+9wPy4rSmZEHlwRMkeOSx6j3ParUM8jB2hdEA43ffbJ7D/AVHeWSyKZplYAfxMR19eSBn8Kpx090nm11KcGpW8JI+YyAcKmWCn19/wAeBW5ZSPqDeZJHlV/ikPyqfXA44rkY4o4v3qDKnkdBu+vrUqfbp33Tsqxrnhhx9MHj9CadKo1o0E4Jq6OwvpklAijke5ZeNkY2x/8AAm6fhyT6VmOsm7E0aqB0AGMn37/nU9lJbxoFLSXEzdgdq5/22+6o/U9BVqeSExlY8FVyrMDgZ/uqR/QH0rqlHmXMYRfLoUGV5SomfCKeExgH/gPUk+9dxozKAqwgvKMgjsi9/YZ+uTXGpbp5gUyrbLwTg5kIPUeorpH1FYLZLLTozEmcAEHdIfUDOSPyHv1NbYW0XzMzr+8uVHUieJ0MEJzI5ySTk+5PoPT9K5rVkVlzKSFPy56Zx2A61q6OIYbYyy/vHfkhMHJPAyRx/QdqoayplLqxwdo4XIGD0APX/Hk9K6sSnKnc5qTSnY8g1C6SKdo7VNu05z6E+/8AePcnoBxRo8YLtHcDCcE88sTjH06Z+gFbWsWcdjE+EDP3Xoo471zGmvNaTSS6gTGZQCU4LkHgDHbeeMdcDHrXjRVnZnqN3jdHXLOkirEOY0yeOmfr6f55phuwkDTTZC/diVeM5POPc+tZdxcQRMJr5jtQZWBepz3b+np7k0Wn2q6uxf3UYATKwxE4UNjqT64/IcdabkSonf6PeGBlghVVdFyVPPPq2e3p6mus3RWkDuWLSkjOP7zdvXPtXD6LbSBhsPznlnxgFj7df89q60RRx+XHEd7ZIjz6/wAUje/YDrXp4Wo+Q4MRFcxkXMsrFsv83IJ7L6//AK6fZFlJNqvloeGb+JvRfUknk/lW1Par5YVPmZflwOgJPOSO/sOlRC12DOduzjeRwueoQd29TVqnJSIc48tjMuLuC3jCIgRFyBuOScnnPuTXHanqSySGAPlgOSeka/QdT6Dqe+BWxqbbInmhXZuB8sHsOm8/0/SuIttKnvW8yZsqTu92+vp7dvrXHWqu/KdNKCtcvxXGVV7cbyPlXA3ZJ7Dtn+83rwK27m4W0hEVuge4KgADnLNxyfrz9BUNtBcF8wrtWNcIF4VR9e5xyT0rd0zSohKzu2doLSueigD/AAq6NJt2RFSaRkyWs12qWaEiCMAtjqzcd/oP1rUGlJh7+cdcH2AHOB7dq6W3todu8jb5zBR+PP8ALH51Xvb6OOLy0G4K+MZ7AZOfxr0FQUY3kcUqrbsjiol83UZEI4gx+f8AFXXsQrMCRuBA/HG4/lXNmAWrGYHOSQ3uWGf61K85aWd0PzIMgepY/N/h+FZwfKOSubsVvKEE754BH+fwqhbzlbeRSc7QCPwrW0u+ju4pLLO1mHyg9MkZH4Hp7E1y0rGFSGU+XKvB6445FaTaSUkZq7bTK1xdurMRnswx6/8A6qmjKyjzVAAPzY+tczBcyB/JkG7b8p9cA8flW9aiRBuHTv8A41xqd2bNWRff5kwGPHr1x6fhXN3cjKXjyQcA/QjjI9Qe4rWvJhBtkPAJwSOx7f8A1q5bULxGbcSA69e34g1U2KJW+1fN5brg4P4j2P8ASrcRcjZgkHgg+n171ioGmYlgAc8EdD71tWpmhA3KSPUc1ES2WI7OZe/X8aklsnkXBchvQ9KvwSiRQc81caQOgQgMB271vGKaM22cva2VxCzsdgI5BIOD+NaNmZJrgeYysD0PQD0PTNaBkjEbFAeuCcjj8f8AECqdoqxys4dtp6g9Pr/9euqnFInmufSPwr1m+mZtLnl3Bfuq3PA9D3+uele9r0rxr4aaN5WlebMDvLBlLegPY+o7g+1eyJ92vpsKmqaueDiWnN2HUUUV0nOFFFFABRRRQB//0f1SooooAKKKKACiiigAooqnqFwbWzkmVSzAHAHc0AeS/EjxnJp9rNZ2UoRBhHZfvZPVQfp6fU4r5QvLszMSSQZicHPT3z/n861fiB4guNR1yVnbKQkqqj7o9cVzFraNfFVlysaqCzZ+6o7Accnv6V8DnOI9piWl0PscsoKnQTfU17ONZjHawgtGo+Zzxx359zVzUS8lv5EZ+VxxxwFHcfXoo/GmWim8k+XEcCAgD68Zpbx2MJRAS8hC7h/CMfzx/OvMXwnf1OQleK1IJ+Ux8c9Ccd/XHU+pxWhp9u8WLiXJkfBjD4JIz94j1OeB2piWttJMnngnB4Uc59AT6nvXXWcKp/pNwBtOAo4JZufzH8zSpwvqVOehJbxGGI9cnl5DyeOuP5DH1NZGpSHmNdyMBzkcge4/xrqriOWMhM7ZpOeeNq+v+ArBuUUny4gTHzliPmkOeT7Ln861qQstTKnK7OLMDeW8odkB43McO2OoUDt61Kl1KAsFp8zYAZuh2/0H61auYJJZGI3YHykjjOP4R7fSorayRcxldqnjAJH16Vx3aOrQ37C5kjZlC5EYyXPA49McADtn8q1kuXu8yRR7woyDtGFz7k9fSsa3tLoFVEgSPg7QCT7cf1rdmsrifYkjbUX+Hdg59cD5R+OTW9NyaMJ2TOWvjFGN9yfKBPGWyze/Hb2rOkuJCAI0IU8cnLY7jpgfzrsJ7Lyj5sUkaMOM9W/MnrXMXluxnKSzE9ehCjJ9TjP5VMo2LjK4y0W6d/tFwCIlyFG4genHYe7cn0rprfUBEWzCTsGBJwiRr1wnp9eprBtvkkCF0IU44GQo92Of0q1PeQ4AEgk9MqQPw4J/KtKc+XqTOPN0NZrmaf5bZPLDnICcf99OwyT9M49q1LdoYQIY1a4uHHT7qkdySSW2n1PWsaOO6QiTCu+35VJyRnr8vv3P8qYZ5UkAjUSscBneT5cjrkLgH8TXVCp1ZhKPRHo8d1b2iCCL95IxBZ/4dwHYDAIQdO2T3qhctOEaRHBlYjLnnDnsM9wMf/WArnY7+eaQ28D7fMYGSXAUH0RM+vUnHA9K0SiX2YIpP9HgwksycBjnLqhPOexP4Cu7n542RycnK7syriG3SP5j8395uTnufc15pqlwI73Fqryt0B6HPTJPbjJ/ya9DvJDc3jwx/JDCPmxxgdhnt05PX8a5DULS3CtN8wIA3AdW54AHbJ4HevNqQtsd1OXcq21tGzxyzttDsPl7se+Py49K7CDEtxF+62KgyFyMAD6/nz+tYDJHBHFeSN1Unt8qjjI9uw9akhklv5TAquUfkID8xGeGkb37AY4rnirOzNZbXR6BaahExzF8+T26nsM+3oByTW5aFnm6jzOjNyQo/ugDjjv71iaXZRwqIY8s7cOy9AO6r6e7eldrDZBVESYVQOdvyjHYfT/Jr2cPSbSPLrTSbK8bLM0iL8sacHGAfx+vXAqLULiHyVVOpOPTKjsPTJ696aZ433eRjykOOOAxPue2OSaw55FlleVjlD93Ax8vTIz0BP51rUqcsbIyhG8tTA1V4pAZZ3AReWycgnoAo7+gH6VRikYgI0OE5Y7unH971Y+g4Axn0qldNLeXf7vJSM5G0cE9sZ7Due/auz03So8RrMTJO5Bcg52gdF9M55wOAeSeBXnUKbqTbR3VZKEbMLW2ubtWlkULsOQG6bj0z9OuO/5VYvENtZR6bacCZsyMerH39s8kfSt+YJH/AKPapiKBTubr+A9WJ/KsS1ia7nZn6IpQe2PmP54xXr+y5Vyrc81zvqyxDO8stv5fMaTMFPqF7/iRx7ViyxhHkicniMZP+0Tmunkt0hiiEY5JOPrjoKzpYftE8uOCXP5ADAoqQdrExkrmbJEZVLHlZFP1DoeP5Cs64UQ7ZVBO8duue9dTFbeQ2YxlT0/4FVDUIRDumCDyznPbafX25rKdPS5SnrY465nlhc3Fq5DKMj+nWoLq/e+QzqRGZjv44CydTx2B/wAamv2EM4mxjaMMOzDrnHt3/OsC4ge3dxDzDN8yZ6euP6VySbWhsknqWYzK0hM2cg9T1H410lq0oQA88ZU96xrCNpRvIw46+49a2yuyIkcKDhh02k8g/j2pQXUJvoZepzqkbMccjGCOMehHpn8q4WWQyP5bghTypB5HqPcV0mtXDSRnGHzwynr9RXFWbyhzESWGejc/lSm9QitDqrJNoC5BrooQF5/yawrRcDArWWbbjdwaIBJF54kxmPg+3Iqk8jxtsmOM9CKcJwDt6Z/Wonl3EJIA4PY8H8K6ImbJfMHlt5jK47FRhufb/A0unNEuY3zsY9eR+RHf+dZsUUTSbYZGX+8hOf068fpXqWheCZZ0inkJMbkE4O5WX1VgTyO4NdtGnKWxEpxivePojwEWj0aC3EizIqjDA54x9AcexGR06V6IvSuS8N6bFptqlvEchBXWr0r6ekrRSPn6rvJtC0UUVoZhRRRQAUUUUAf/0v1SooooAKKKKACiikBoAWuP8c+ILTw94fubq4+ZmQqijuTXXswRS7cAV8zfG7xC5gjsIuDJwq9Tjux9PYVz4qsqVKVR9DfD0nUqKCPle/vWvtTZiCzSMWx2/GuktYRFGlk5Yyzctk87f/risG2gS2mluJV8xoyAFyAAx5JJ9fQV0lnLI0MmoS8yHgDHc8KP6n2Ffms5OcnKW7PuklGKii5cOVjIh+XadqgYxuPH546VKZY7ezMikbgu1e+49yPWsFboGTaxJI4X8eCePUd6vrMbiRZHj3RxHaoz1yeB9KUZDcS3CFESyzrjI5HTgds+nrioBqDxXAeQAOwBjU9Qo7+gyeB6CqOp3bwAIG3zO3Poo9MdPpWVYXAuLmSRCCzMQCWyBt6k+yjj6/jTUtbAo6XZ6Bb7ERrq7b963zOSOQSOuAfThRV1bNpJFkkG6QgBIVzu/wCBY6Y9P61zum3KvO00LkBeQzHnPeQj/wBB9Pwrozq8EMWzb5UZ+bv5sg6DOPmC4+n866lKLV5GElJP3SebThsHmhccYUHj6DHpUUejIR5r7trc7Y1GCPqThRU9jN9pIk28cYQjk46cdABVu4LqdxOTnOCcn8ugqnGDXNYjmkna5QQyW6F44gFPH49ueM/yrPngvbxyZHKqP7uefx/wrWkkiEiiSUPJgZI6Y9qlN1byhlWViEGG24GP8B+prJxUnZsvma1SObXSZPMLLgN0LHnr6ZqnLoJdjufdjjPH6Zrt0lswoQLkAHvlmPqT6UoOX3eXz6noPYCl9Xi9Lh7aSOEHhtAn15J/+sOtL/Y5iUbIzx0yelejfZlnUeccBfbA/Cs+fS7EhvOTCg59M+9a/Uo9CfrT6nGRpd2oJUtGx67QAv4kDJ+lVgss5XzXL5YZLqVAHbHYnPrXWtZW8oZIfMZvQE4A9SenFZz20w3IqkMMfNjOMdBik6DiwVVMzn0mVGUQRkhSxZskHnvuPQU/+04tPgisYm8pYyQi4LHdwSxx3Azj3xWluug3kD7pIXc/AAbsqKD1NU7y2iuVEafM+ck4HY8DA6fnWi93WJN76SKxlW30/ZCAXdsbm5+Y9OnGQOe+OvWuei3tFGJFLIGZ+vzOegwPX+VQ3FzexzC0SMCLIVnXcwAz8xHHXAwoHU1Umu3+3FrYbCSI1B/hxyQR2AHU+2KG+axSVjoBp8uppL5qjzdyqFAG2MDgBccEgcDsK6W3sY7WKO0tkBMjckHJb3dvT2Hauc8OXUqyRqFIVizru5Zs8bz9e34V391IplisrfYspA3yDog7gDueM9h+FKnTi9QnN7GzZJDFEUc7iv3seg6DjufTsKszySyRqk7bIyMt2P0z7CoIhBBGJERiCCY0zlmJPBOOme1QX5nRlg489wBs7IPc/TrmvSu1A4GryIrm481fKhjLRqAAq9y3RfbPc9cVjaiHjUxjEspOCR9xpO4x/dQfyrbdks7ZDGSTkhSerE/eb8uB6CuV1KVijEfLGqdfb09+etcuKlaGptQjeWhgx7llJuCBEhySvGSPQ9z9Onau2sbmQRBmxHkfKO/1C+gH4k9eK8ut7xZ7wBBuMXJzn5fy6cf4D1rq4dR8sfZ45A8xzJIzcBF6gsew7geg+grDB1LG2Jhc7WWZrXTyZhzKScHkhVBbn+Z9Sa1PD9u72yPMMNMCdvoME/yxXF6fcvqzbiT5bncS393+EfiFyf1rv0nENr9oj/hXao75cZz+Ve1QkpPm6I8yrFpcvUbcGNntnyAu4n8zVGQpbt5yjgtnPoR/+qlmZWmhgUfL8uP6fzP5VUtJftAls5fvoSV98cEVUppuxCjoTSqYbh4cjZIPkJ6YbkfUetYlxdXMRkDp8ycbG7/Q/wD6waS8uWMaR5wQTtPoR1H9RWZqd2H09jKN8ZXa/OGVv4WH8vbFc8pp3sWo9zzjVtagmmjaNGhkUlW2nKnHRh+tTWd0JgIpeVPb0PqK52QR3N0ZRw/cdN3v9a6Cxg2YGMgH6GvNcm2dVtDrbFDAB3XPB9P/AK1WNQmWPaR/GNp9CBUFozRxAxncoHQ1k3crNI0XQKNyfzre9omdrsxNScncrdV4NY9rEWYMeoOD/wDXrQuHdi0q8n+YH/1qkgVPMZk4Dc+3NYPUtFiNzEwJFX1kBXL4OOtV3RWDoOoPT69qzpJ2jYSA4I59iKa0HY2WMU0nkhgkn8Of4vb6+lZs9zNA54wO6E4H1B9ayL+SObKplNp+Rwc7f9k+3pTvOmv4fs10w81R8r98/X+frWkX0FY39Cv2utURJEEjA4+YDdt/r+dfc3hPQkt9KhkUKCR8wA/mPUfrX50+EtYu9M8SkyRf8e55XOAR6iv0T8DeJpta02KR4Aq4HK8fp0/I19Fljjszysw5t0dlBbiJiq1oAYpAMEn1p1e2eS2FFFFAgooooAKKKKAP/9P9UqKKKACiiigBGPFNHWhjTlFAFTULiK1s5bmf7kalj+FfAfxB1+41vXLnUOQM7UGOg/zivrz4j67JpujTlFAjVSCzdCxHAA7/AMq+E7ktcSl5mxvJdmPfvj2zXzPEeJtTjRXU9/JKF5Oo+gqw5hiidtrOMtn8yfqT1rppIoY7WO3jOcfNj+p/nXKWTrfzmXd/rMqMDoAeceg4/Oupjie8uSsYIjVQFzycHrmvkI7n0kjHthIDNNcdByxPp0wAK1EnIt/MIB8z5gFHGOgAH8hVqdEZhBHnyVJLseASOcn8eB+PepX/ANTlRsVcDn/a749T2x0qoxtcTdzgNZuvIjKkkySHaxHUdiR79lH41zkd7JEzR2KABUAOeTn+FeOvqavakBJfCAne3IO7ooHr7mrOh6VJcRrcyKdhO5COM9uAP0NZxXY2ubelx3MURMjkAY3HoS56jP49AOK63TolmmI2GSXILHsMdMnoFHpVI6ftQRyEb8HGScIO4Udz6n14rrLC2ihtltoSBHgOzYyW/DuPTPGfWrhGTlqZTkkjSW8jgVg77io+b+n0HoPSueu9QklJEC4X6457kkce2K0roQrEIU+VTz6fUn1J7+9ZBga6JAUlegA7+vtgdzWlWcn7qM6cVuyk05RfKU5ycsRxk+gI5xWna28skQByijBA7fUg4yfrVq1sUjOM8+wz+H/6q3I4fL+ZkIxjGRz/APrpQot6sc6iWiKkVsY4yxOB6nqfb/69W4ZCq4AOTwOwH+NE2xzhOq+nAAqJCm7BO4jr/hW0dHoYvUuu0zjKybE/U/5/KrUWnGch5GBA654Az+pqmG2MGAJY++MVKsxLbd+eeABwP6mumFRX94wlHTQ6SEoV8lACv5Dj0FRXEAkYE4RU6E+vr/hVOKT5sSuNvQ9s4qF7/wA9ysAwpH1IA7D0r0FVi42Zy+zaehlT6Wwn3RYJY/w/5/Wsm6W1gdllQggHaT+Wf144rsZJP3TDbhjkj1wB/WqqRxzkRLErepPOAeT+XGKwlRjfQ0VR9ThNQ0aC6wtlGAoYAsCfu9jx3rzoadJa3l0LxgLaTo7nAVSfmUD3zxyc17y6RJI8cciqqK25R2BzzXEa54dnvLYSIm5guQSMncjBh+n86xqUuXVG0Kt9Gc7JNNFOXTFvEoT5h2UjhB/ujH8q6rSLy3c7xhIA+WZj1J4C89fXr/SvOru+MlmIMM8rMXynOEGOv+8x7elWNLmQyImoEiPAEcWcu5YjJI5CjsT36etYuVp3Rvy3jZntWj3ltfTztYOZQjFTKB1YcMFP91emRgZ4FaEsSyyGJCFXq5B4AHPJ9+pH59qxoLmNbFbS22oCOFU4yM47D1zjP19hkzahJPdCyiICAEyHJ2qB1ye+P58V2SrxjFI5FScm2X729E06wQN8ozlzxtUdcfU8Vw+t3yysIYc+VnCjoBjoW9enAqa81iK2t5mgDFjlVz3AHAHuep9OlcRrcsz2kUMrkbAGnKnB+b7qg/3mP5CvMr1uY7qVHlFOrKqvFZjIQk7hwN36Akfz+lXtNV54I7dyFRiZJjnmSRui+4UVRtod0Rto1WNYiEwvKhiMke+3ufU11EFvEsas/KxDcwAxx6E+/fHPaoptp2RU0rHciBEMemxkDKBS3f7o3tj36fQe9aGoawbi8MNqpCBnIx3JwB+HT8qxLWC5WNr+5Y+Y4YDt15Yn+Q9qktrabyludvQH8yf/AK9eoqsrWR57gt2WbzUJDMm07fLVSSPVEIX8Oc1lSX06FpEciUNuz7nn+f8AOp7q2ky8bckjae3+eBVDymhILkHcoGPXFQ5yb1DlikE96lyzGI/J98Z6gE8j/gJzVC8d7m1fZwcEHHPI7gdx6j8RWNcOLaUO3yIw+9z8pJ4P09aItQeIEgg4POP7w/kf0PWlGbb1FKPY5xICH3jBxzxzkV1dlh0GOGXg9/pTI7RZibi3GAxJxVu3tzu2jgiklqSy8zeUmE439u2fT/CsyYZlGeCBx/WtOeMyIEPDH+fY/j0qjcEsgkxyMcfzB/nWrRCOWlO07Dwe2aWBieP9lf1p2oKDMjg/5P8A+qktmDsX/hxj8qytqWWLmXaflP8ArBwffNY4Du6xn3GPUf8A1jVlFdkIl5A+b/HFakMAlf5QAcbh9euKSTZT0MaO0mD84PTPoR2NWLu0RLcSxg/StlDGAOMFePwP/wBeq13cCNmtZ0PI3Kw/iH+I/wA8dOmEEZOTNfwNo2hajchtYZombC+aB93/AHh1wPUV9oeA9Ht9EtPs1vOky5O0owKsp7qf5j/J+LPB9z511EI1WYCTbhiVDc9MjkV91eG7e1+zo8ds9szANgsHDAjqGHDfU8172Weh5ePVjsKKKDXtHlBRRRQAUUUUAFFFFAH/1P1SooooAKKKRulADepokcRxliQMetA61nazcw2mmz3NwcJGhY/hQB8sfHDxKtxcxaLasWSP53x3avAEUqgEgzJcj7oH3U/yK6fX74axqt3qdwCY1PA/kBXOQCbzzdHHI9Op6Ko9Bnp9K/Os1xHtcTKXQ+3y+j7OhFGxYW+LtbdYx8g6DvxhF/mTXaW1ulvAYBnzJMea/wDtHog9gOTWPoURt4nuTlpHyqnHfH7x/wAOg/GtdrlhIVgA4BBPYZ4yPUnkVz0kkrm1Rtuxk3SJ5qW0RO0nnA6he/PesjV74RRLFFIcDcR6n1Iz+Wfritq53NKQv7xlBXP19+1cFrd1H9tdVIY7tvHQ4HIA9BWNR20RrBXMOy05pJXku1BkuvmEZzlI+o3ehbqc9uBXfWhg3xiEnCDluhY+w7e3tWPpdheToYSfKeY5dtvJ/wBlc8n/AD0FdjpdrbQ5SD5mBxnGcEck++PyFEU3sOTSJ4bQs3mzjDSkKiDrtHQAdh+vU1p3VzHZps+8R1UdiOmfp6dqfFGE3SgfP15OD+J7fQVi6g7b2CAEk4+X29Pxq5e6tDJe8yhPeTSuEBAZhyT0UZ+vX2qytyY1WFgfQK3U+5/wxxWN9naBjO5HHOSen0q1YnNwrrxu6seW5+vbFc0Zu50OKsd7YlEtw9y+1nPReuB0A9zVq4AkO6X5c/dj74qtYlDGsyDcxOF3D5j6YXGPp+dX5kkiUqyDPQ5OT9M/zr1knyHnS0kY7s54j6evqfQChZRGmFGB6+tPkBdhHH7ZJ44FU36bj6kKP6/4Vxu61OhW6lgSbQ2OWb0q35626iPP7wjn2J/rWQd0SbjjceFBqZESNA8rbmyBwO5pRmwlFG9CiSMN5zxwKv2sI3FkAAXv24rm7OcCTk4znH4delasWoSSAbCQo6HpwOB+td1KtHqctSm+hvPOgBG0MxHf17VnX5aLMYf92U+nzGoAftBIAJQfxHpjBoS3eV1edwwGTj1Ax1z7V2+0ckc3Kosx/KDXe5pAEAAAJxuwOGY/WnfbJHstlxgSqVQ4J5LHOR7mr93oyyFQ2c4BJHBwRg5rNNoUJWQgYI9s5xjH5Vg+aLtY1XLI831iyXQr5r12xaPFkgdAOAo498VkgC0DTR7TNIeTzwW447ZwDj0HNem61YJqGjvaFR8q7CMY3Acj8f8ACvIJ/NtYorS5x52ZMMBk+ikL7D1rCcUtEdEHfU37XUVjk8iKUStn5VUZJxxyegH9B787j3EvkmHG9nwrbBkfTPfHc965XQ7SVkE+RHaxAhd5DM79C8jdCeuFHH8q9MtraMoiMegySRt7dh1+prn9jNo1dWKZwcsM0b+dMRhMhUUZycdPw71lPYzsRdTryzEoDyTI3G4+/Yeg4HevVjZRS/NCMjGN2OMe3rVOfQpkjLFCG24HGCq+3pn86x+r1FsjT28Op5g0Lxtb2Q5yT8o4475I7nufc4r0HR9PkvJYlnQCPjCjhQByT+Ap9hoMKzSTy/My8N7Z6IPc8V3EViYYCqDEknyAfTk/0FdmEw0pO7OfEV0lZBfwRuqwxfdC/Me2Rycew7/lWi1uFjgt0HBwcd8Djn8qgMsVvvR13BEA/wCAxjcx/ElR+NWbe6Zrwsw+baFHPfODj6Yr2FCN/U8tydjLu7dvNZh/ECx+vIrm7vDiMY68DHr0/mK6x5fJn8sgkjDHPp1rKvrdCpEYz8hcD15OcfhWU6Sd2hxn3OPmtlcESL8wyCvqO+Pf2rn7vS2RC9t8w9OzL2x3BHpXYSkHJYb84IYDJ9v/ANVZ5mSTfAevUj09x6g/5wa5/Zo1U2Z+hygSfZz/AMtPuj1Pp9cVrPb7H82M4Knn6HkH8e9c0++OQoGz/FGeuGHI59D0rSOpO8YnJ2sPvZ9Cev4HqPxoTVrMT3L8rkuCOGH6H/A1i3M/Lc8Nkj6jtVma7jJBGAcfl7fT0/Cudvb2JQN3RiPwPQUSYJFG6nLE5OM4wfekhBWBgeC3A+orKabzVz/zzYD+ecVsRI0jR56qcmsupZrQRhoVRhknPP1x/wDXqeGF45M+nf2/+tToomyFPp+grQWMnlumc/gf/r1rFENlN4OTkc8gjuQf88Vy+qNJcRfZN33TlW7o319DXYyFS2yU4I6H+VcpfSMJf3i/d7dCD9R69a1WhKO4+HPh5pZ4vnyCw8zBw3JxuGfQ1956ZaNb2kQfAkAAfHCsRxux2J618b/DaNZdVt5I8ZXGVzgsp64zkfnX2nbkeUAoxjseD/h+XFfQ5ZFcjZ5OPleSRaooor1DzgooooAKKKKACiiigD//1f1SooooAKa1OqNjigBVry34s6kbTw3JCr7d/wB76elemeaFPNfOHxv1pY5YNNTJ3fMw9SOg/OubF1VToym+iOjDUnOrGJ82XrLFahpu5yV/ln6020tJruQZ+RV5yOxx7d8cDjvUl6pMyyuu5YiT9ZO/1A6fWtqytjp8KichpZmDKvXaO5/E1+cW5pOTPuE7KyNgZtLYIgZndR8oGDtzwo9Af5CpIEJCgDcQTuYcAtj+QGcVn3jzF5JlkDPgxR9ueN7/AJYUfjUjyLZ2oRGX92m12PAyBliPT0rRPqRYjuZwd0UJ2b3JAUfdwMDP071yY01BcmVySVwoJ4GO/PqOp9zir0+rKEzD94puOATjJ+Ue+euPSs8yz3UYe4U4TGyMDGT6n6Ht6msZtM1imbqeSQtvb7meZcE8Agdznn0/zjFdbCIkUW1qqrFEAmTnk9cDucck1wltJ9mDK0oaSTLOeNoHufQen5da6Oyu/wB0FjAXJyvHJB6sQO59KuE1YicOp0O8sCWIQDkcdDjjP8+KpvblV84DzC4+UYP4E/XtWdbyqeJd0yL8xAOFwegz/k10clzGYV8wqCc/KDwMDoT7Dr+VaxSktTJ3i9DnWsHZA0sak9gemc8fh/OpLawCEzNtIHB9Sc9O9WVmW4YzEYEhwD329MDHTPPPoPerUssCxRlHWIbQFwOBk4GB7cmpVOD94tzlsdVpMcfmLIx+aMdM8KB157k9z/kak8KyR4bCE84/ur/PmuUg1aKGF1tvljiIUEkZwo5Y+/p2Ga0ba6N9cKASFzudz1OACT7DsK9anOHLyI8+cJc3MTtpayfu0b5EGW9ye1Z02nNGPMc4RM8e/oK7GEbVEaqOcE47ZOcn8BWTdSjUJQkQCwofvew6nHrj+dOrhoJX6ip1pX8jh5IHch8cnp1wBnNNEZyFPPPH19a66SwR8sB+7Xt3JHr7VDLYpaRtLJ/rCceyj0zXnSwclqzrWIWxgGEqSQuSwx6YHens+0NHG27nBwMDP+f5VEzPOSqk89f6VdtrdjKI4gPlwST9ayjBt2RcpJK7LcIkDCMErgct3+gHpW5a+Woy3GB37/Wqa2uZjKzcbSP8/ia0444wxV+COT+HavUowaZw1JJof5jStuT5STg44I7/AKVz+owyRglQMFsgZ5B649xxW3KpWPeM5OOB6nGDVe+t4zYEXIwyYJ+jZH4VvUp8yaMoS5WjmHuEW1klJwyuFH+6QCD+GMVwnibT1EX2pc/JkZHdTg9uvX8q7bUGLjyOC23A/wBrB7/571QWGC8hudNuh86qpQ9zjKn8wRXnSTb5WdkXZcyPI4NQNrMlywVFt8qC3KoT3wOrt2A5xxnrXoVhqH2pVS6J2cM+eNxPQE9z6gdK8e8W2N/b3EUVuwTYSTv+6h9WxyQPTvx2rf8ADlwCiKztPLINxd/vBPUjsXP3VHb2rTVpWG0e7JqsEAzGQWwNuP0/D6VYmuWkQBHBduASc5J5Y4HpXnXnLBliDJK5x16exP8AMCqr69NajzUBeRfugD5VHrz3z3/KpliraMlUL6o9TR7ezjijRsuCWJb+AEZLn/aPX8aR7pVn8tDtSJfmz2UcufYlj/TtXlMHieKLcZ8mbGVj64zjBOe5PPPQVvpeieAWfnAIcNdSk5yV5Kj2HT65raGKi1ZETw7W51QYyBrqbHztmT02g7yo+p2j8K0o5cGKUjJjUZ+pJP8AhXKSa9DqV0NK0seYsP3s/wAUr4x+XP8AnNb9oXkgSNAT5khYt1yFyB/jW8JXehhOLS1NS5KedyMZKgk+hGf0zWZcoY5sDgxybfbBHFbc6rNeKWHyuCg+vT+YrOuCJIVlcfNIgV/95Bj+VdM1uc6exzE8Ij+UDKc8dx3x/hXF38JQfaICQU+brge/Pv8Al616DOhyDnjgEj1I+U/Q/oa4rVphA8jJ93B3pjt3IH8//wBVclSJtFmHJcIZME4zjGe27kfqKsK6Sgg9R1Hsev5Hke1cqshMnkEjHVGHIx1GPyyPTpWhBLnK9COQfT/61c9zSw27n8oBQ3Tgf0/UVz1/c+bIsYOQB29+RVvVZxl4xwd2Qaw4w9xOO2cD6ev61LRSNbTY2kjUn+I5z68YrsdPt/lMjDAJOKyra18lAFH3eAPb1rp7R9sXIx0wPr/k1cYgy2kONzegx+ozVmJM/Kf4ePwIp7KI4zu7/KfqcVWeYrIhBxuG38eK12M3qMlEcqmMj5gOPUjv+INcTqZlVVdPni6Er1A9/ofXpWzq98bZlnIJVz1U8g9Dj36GuR1CacS742EiMM8HHXr/APXHak5Iaiz6S+DEcFxcrJIvmIpAPGduf85/CvsSNdqhc5wK+RPgNbTQ3IkliYRzr8rg8Z/usOhB7e/evr4da+oy5fujwsa/3gtFFFd5xhRRRQAUUUUAFFFFAH//1v1SooooADVdzU5qEjNA0ZdxvZ1Re5r43+KmsR3/AI0mgR/N8giIEdAV6/lX1b4t1OTSdFvLyDiRUwh9Ce/4V8BJO02ozTM3JDZPU5bj86+fz+u4UVTXU9nJ6PNUc30NCRfNkXeu1FywB646rn8OTSw3qvMX+8ygE55HOcA/Tn8K5q7v55JHtI2O5s7sY4wevtj+ddlpmneXZeeY/nf5Uzxub/ADGfavjI3ex9Q7Jal6EhVE7jDRgYyPXkAe5PJ9AK5bVbzzSLbAKjkg5IznofXHX3OK6Zl224jwJJDwuf4mPBOPaqC2dnbbkuvmnds4HJwOQCR+Zq2m0kiU9blS0s5TEZ2By7ZCnP3iOB9AP1rOv1WAFbYF5B8u/OAMdST6gH+g55rsod08YSMgZByx45I5x7Ac5qlcWaNtULsRULAE9FB6kepNTKnpoNTszl4wtvbZmG/glFHRmHTPrjqauxFzmEFpJXKhznk5PCj69+nFMnidlyvWTAGP4RjilguoYsKAV3kksOCxOAcH0CjH8qiNky3qdAk5EaRbRtJ654wByc9TjoAKtyzqFZZBufLAY6Kq+g9s/ia5ZtR37REdqYU7sYAH8KqPp1PvV6G5nM0ixLltoLueQik7jge3Sqc29CeWw+e9uUCbAfmHyp/EAeMnPt+VQG9fzA8XzM0h2kDCJhQAefQVQvP9FtVSQtvmQ9OXIJzj9Rknr79KzL17vIiGUjVAu0DgEnJ57+5qLtbl2TO1iuUYQ2tucKr7mbrnnr79sev0rrdN1KPZ9jgP7mMEysx+ZznJGe1eQWlvfMrtDnfIoHmMcKoPUKOOg/8Ar12emxTpbpbI4CKRtVRwTnr79uTXRQrO+hlVpq2p6jLqEnklZHw8oG9h2LZyB7jgVLYy29vZM8SgeXuCb/4mPVm9h2HfFc1FbyyxI7El1J69Pr/npU5LyARLl9pCgLzluw/+vXpRryvexwunG1rm2dVAgKQ5+UHDHqWbqT7/AKCsvUZ3bZCSBtUFs+p/r/KoZoHZ3VTtWP7xzxgcY/E1QuJvPkB24Qc89x7moq1ZWsx04RvdGnaxtcr56oEjOFjz0LtnH1wAWPtVm2nt8stq26OIDLf32PTn3PSsf+0Zm0/ywuC25Y/UAjBP5VNprpZwurbdiOrMW/vLyB74ohUimrDlB2dzq/kWF5CQMFQCe+DgHHu2aQSqzrDHglmBPPY8/wCNcuL1JLcRS5QyEM5b72wZIH/1qjtb9o5zJ/E4KAemR8v6Vr9ZjdGToM6uO8jmSKVcETHBHTA6kn+VPupeMSYPmfkMDIH5VhCSFYWOPlAYD8v61RlvLweTIpIiUsT65/yK2WISWpk6V3oRXsc0Eu9VLRIRx3Hf/P5Vm38VygF3Zt+8jZWP0U8HHt0b6V11xJa3Ba5YABMCVewyN2f89Olc7KS6O8BIO3I56qzdPwrKtTT2ZpTk0efeMLCPUtLuNVRAxAzg9QU7H8q818Ms0ql4GZWlOXkPDY9F/uqB+J4r2i4T7MZCU/dy5WQEZAz3x6fyzXiWtTyaTqbadFE1tEAMyDI75AReS7H1GAPrWVJ3TizaXdHoj3iRx+VbsoYfLz1H19D/AJNYt7LcSkQQt5ca4y2cM5Hv1IH5VQhmuZFVreHyxx97AWMe5/vevXmtm20ia5kLvk5PJOcfmf1NcVZNvQ3ptJamRFZzSzYtkDtGdxbGVBHTjufTPGeea3bK1ugGWbfkkHJP4hf6101paxwxbLYHpnIGBx6Z6D3x9K0RC0KeacZwQFXls92J7eg/WiNBilW7EGhWc1vJNLF8rSNhSDymeD9WOTn0H416PYSwwZYHcvRD7kH8MccewrzaCVYoWmd8x4UDacAAnoPx49WJrQ+3CEQ2ksoZl3ySspyAwjPyj/dzj8a9HDT5EclaLmeiSt/oyyOCHiVX/Niao3bK0M0AyGQhk+vOf0qpLdC7W9dCVjG2NQfSMDJ/GoNUulY+dGflkAHP/XPNd0pqxx8juUZGDgM5IV8qw9GXn/64rjNfEnAb/WgbkYcZ9wR69CP/ANVdW10LmzMK/wCtfBH++oyP++hkfUVyl5cQzRspJ2scjjO1uh4/mP61zzaaNIo8zllEExEiFCT0x0J9D0wfb8qu292q+YSe24Z/M1X1RiZTC2OvI7Y9s9K566uGtpvLz/D09ewP58VzpGhqXLreSbx0OVP+NadhbgOZX4xyfqOlZFgFReeg5APqa0hOq/KxwOp9lHrVD1OqjnTaMfxevpV23uVkdN3CsR+AriTqG8sxyEXgDuSeP0FbunSljHn+EZP480ORXKb818JVC55wx/HNUJ5/PdihwGyR7EHP9aqqGwhHpn6nNV7wrBJlchTnGOxHFZOoWoGbfXsnzWspyj/MCPUe/oR+VZxleaIwyY3fwtjg46Ejsfcfypt8sm5ZegHHHTP9Pas64dIxGYiY2yMAZ9eeO2etFN800E1aJ+g/wg0K40zw/byMNpZRuRuV5AIIPb9a9wHIzjFcF8OruSfwrYNKyyYjUFl6jA/iHY+9d9X3GHilTSifKV23N3CkHNKaQdK3MRaKKKACiiigAooooA//1/1SooooAYTTadjNBGFOaBnhHxx1VrDw21rG21ZDlznknso/rXxg1xJp2mGcAfabkkrnny1PAbHrjkfnXvHx/wBbe51+00KIfu4hu25+8fc9h6+1fO9zcJNflpXLwqAWPYkfyB9PQV8Zn9bmrqC6H1GUU+WlzPqWNDtna5UPgSSHccjJUfwj6mvVpZ4ra38lGBIHl8/Tnp6dSfavPNH8yBmvSAGZSUyfuqemffHPtx3rRaWXy9jjaihVYA9R94qPrkZrwVLl2PXkr7kt3rEdtjyk3u5CRgnHA64+p/rUOnXLXJd3beSRuI/vMcsfZcDA9etcld3b3Fw/lgliAcAYAXoAD2J6k9667RrVoIIpWyUDBpDxyegA9R247A0J9ymtDtlnWO3DOAmW4HUknGc5/ICqJLCRpJyP3nLE/wAIA2qPzJqja3cd1qNsX/gaSVh14Rfkz+eT7mpZbuGByZEMiQhQqn+OTgj68nJHtitr30MbWZBNZgyuu0tsBA4+9k9Pof5CsR7VWlCA7jHkEnnp1zjgD2rqW/132CQ77mdcORxhn5cY68dKzlsJX85IGChyVD44GTxgdyOBzyfpWM6ZpGZk29vPdTLDChRCxZAerEdyccDp+H1ropWEDG3hbzJGHmSuOgA4AA9BwAO55NV5JIrSMx2/II2MSeXIH3c9vc9u3NWtNUCaTzHBlnG6RgOEQcAD0AHIpwp23FKdzPkjedTPtJ2ZIz94tnAH5+tNi0+UDCvkKQgZuckY3EDpgdB6mtuCW1mtmMI2RLn8FDf+hEn8z7VSM8mTg4YEAADhck4H5kAD2q3TiSpsox2ksl03XyIMrnA+YgZIHocnoK6uGOKxRFkLNK6gOOm0HovsO57msxw1nIHK7zDtBzzl3bKqPXnk464x0rMfVLmVnhQkNkK5OSSxyTz64H5mhOMA1kdJcalIVkMjeWnAIGOFHQe5YkYH+TswXxskG3AZQ2B3GAc8+p9f6V51FIzOrzsqlWDhScgFeh+gPJ/AVblvELRyGTbGpwQeyjk5xzlj1+oHrVQxLV2EqCeh3MlysNpGHbLSHlAPvORz+Cj+dULmQed5e8DqpOeODg49TngVzBv7ieRro5VdpEaD7/P8R9CevsBk9qt2qp59vNP8xR0AXsMf0HNN17oSpWN+2iCiTJ4QYG7JPvj/ABNVpZCw8vGQ5+cduPesyG8ODGxOWlHfAJAbOfpnP1+lS3NwnmLGMhQ29l/3xgZ/LJoc48ocruXXOZhcynIGFC98dPwFIkZNyEJ+UEAEeqgg/lUNu+yzM7H5o5CeehUghj+HatGOMH95H0jYkZ913Y+uRinFJ7EyYs1yY4ljGcyHn2CdMfzp5laSM4AwOWHvinSRq8zvngEY/wCBDPH65qjbPslltmPB4U/7eOv41V3ciysXfOW5SYKdjgE89c/56VTW6UwmL7syKQB2f/8AXUrnDjnG9Dg/SsogFjGemQfp7/nWilJbk2RqMEv7YkZLgc+vHr+HGa8/8a6Y13owvLSXyZrc7fMx8wU8YOf1/PNdUolsbhLtXG09cnGB6Z9+2atFEvFeaDarn5Xzna6443DkZ9+4rSMk/Uhq3oeGafC/2mKNZ2YRc4GQDjqc98nqRgdsmvRbMRnCTO0jAdAfkHru5/MZNec3Wl2WjavNEYZFnkbLLywUH/aJwAe2P1rsLWURJm7YliMKi4VVH48/pTlGzC522AY9u9lV+cZAaQ9hx0H41Vu7hncW1viOCPgkDqB/CvesOPUoU2oWwyjc5BJ2jsP/AK351iXmswxusikbE5AX7qqPfuSetZzmkrFRgy5q0txHGJYcq8cilVb7oK/dYj/YHI98Vk+HtQNvPDDeP5gVmMpP3cswO0Hjg9D7VQuNUdylpKTuciRyTngckD3HT/8AVWZO7xQqIVyzfIEPUmQ55H0JJ/AVjzO90bJaWZ6dp/iR5ILq5ly8jAr04LsQ5OPTofYGrn9ryTW8QZtwU5/EI2P0rzlp0hEwictlGjQ/3pSBuY/ifyGK07KVktWZmwscarj/AGgME/ka0+sPYh0kdQ99LHiROsWPxKc/qKx9UmKyvJCf9YN4HZhjkj39aoRXLbd8bff+cDryDz+YqK7ZY4yuSsed6Ec7fb6A1SqGbgjnLu9hkJD5BHKnqMHg/ke1cfLdmW/aOXrtx7HHU1b8QSzKJHYbCw+cDoT2YYrjtOvZbs+Y3EjfLz2rshHS5zdT022n/dCRuW71HLcMW2k4Xgn+n/1vzqDTYfNGF5X7o98dfzNbz6aZVJA/z3/wFZTvc2jZGXAXYgyYC9cfXn+QA/Gu0sl+cjuFBP1PJrn4rQ+YiuMAf0rq7GLAkbB52/zrJ6lstCFsKf7vFQ3dr5pKjvmumt7UMWDAY2qM0klgwY8dKHB2I5zj/sBeIxuu4Dhx3x2P/wBeud1GxEclq0fAjlQliOi56n1HrXpM8ZjKug57H19jXMaxcwWl9YBEyss444wMnpzXRh6dpJmNSbaZ+jXhSwt7fQrLywmREuGjPHIz19PY11NcF4DETaLH5TEhAFwccDqOhI6V3or7SlblVj5ip8TGsT0FOoorQgKKKKACiiigAooooA//0P1SooooAKrXtzFZ2kt3OwVIlLMT6CrNeHfHbxK2g+EpESXy2lBAA6k/0AqZyUYuTKhFykoo+LPiB4nl8QeNLzUpn3BiwXB6L0/lXNCdfKjXIDO2QB2Lcc/hz9K5eH7XfTlpPkErYLY6YOSf5VqySg3CxKPLCtgAd8cD8z/KvzvG1Pa1XM+3w1P2cFHsdwt0rWcbSchpMYJ5PPAHtxk+1Xp2e33SSkZAO845LuATgewrm7RlkuolBIitw7HsBjj+Z/Gtn7WWkJRdxDbUB5O9gAM/Qcn0xXGnobPczIjbLcorxlMk8E/xejY9O/oSRW3ealIGt7aBsmRjyOFCgYBGPTOBWNFFEsvmyYZmZsY5J56H3ycmqLXGdTjjhXccmJD/ALowSO2F6D3peha8zqLS9RpN8Z2Ql3iQDkkABdxPoOg/+tViS8D3CPjAjcHHXBI4HPoozn1Nc8R9kkgQnc5IbA+6mMYB9uhNTQoZZhCuWHzscHnAGM4HVjk/SmmJnT6VcPMW1O5yxI+X9e/5kmtG71Ke1AS3GBgFmA6Z7L6nqSeg/DFc/fzG3tYLW3AUxKFYHkBiw446gAfia43Vtakk1BbSGT91GhLdySTySfp/PFXzdBKNzsre4tzcwrt3gMQuD0Q7ecnux78cD0FR3msPAt3HFzPP930G5tqL+eSfYCsDS5fs8YuDlpZZG4zyS+cZ9gqfrgVo6UJb7ULmbZgA/KW9Bk5/X9fak5N7D5UtzYsbk2emJYRnLtJEu7/ZQ8n2zg/hV571BOZICdgkaU9hgfKuffAP0HNcvtMiXH2diQHUE923nPHoNo/WtWOyjEH2QNmRzCq54DBgpkz9cEfQ1UW3uS0jsCyJvknJULjaO+SpJJ96IIIluIncAKwMxXA7jcP8D7D3FZ+s3cD3EqdVKsxxnB3BT27cgCsObUZJdSzHtaONAqbgeqoO2cdc03JISi2jqZLONIVuJvlllIYDoQvXHsCSST7H2rLOmGaRLdkCR56kYxkjHHrjn6mpkkuLx4xdZ2KPm2nGWxnH0HTHrV9HkjKPMm+UsehyqySE5/CNM+2feqVOMkS5NMY0UUe1DGM+WSMHnaMd+5b196iljmaIuiYwVDNnG3OTxUEerCa5DodxJK7iflCsQCQBjJABA56kmqM3iBHvI7RT5ah8nsAu04+uOT9TQ1DYacjTmVUEZdNhyQqevXk/XPI+lRsXkjjuJeXI+YdFGD3Hes3V7qe4vrKNCyjylL9j8z5IA/vNxk9hW/qEcEaLA65YF3wvOAoAUH1JPQfnWcqd22ilKyQi3kf2RpOsS4Ue5PX8wM1b0++Ml3LA3y7hyR/f9/xFYUpit7gaI0gNyQp25yQ56E+mM1GrmG4cSEBZcLn0JySfwI4+tJOSYNJo6+S8hFu0+eARj229f51VmlaLy7xR8n3sjoCvOP6VzsF2JdNadASjFgV65wo4/L+talpdAQC0mOBK7AkjI3EEqfbpg1spX0Zk422Ogu4ooo4LiEAoPMI9GEg4H4HisQykFH/iUAk9yp/qD1o0m5a7sFguQQS6oy9vnXg47YYD86vJFHNHhRtdc5/E4P5E10O72MdtGT200N7C0SqGJ4x0B9QQensfWube1ezm8y3YgfdZTkEew7Z9qttDd2su6Pg+nrVuJhcEiT/WgZ2n+MegPqO3ftzSTUtHoxNcu2xxXjDThdWa6js882ylgwGSnHJx3x6H8DXkdvrl5cqFt3Aj45U8uR3PevosSQpMQBtLDH1/x/KvDtc0K2sdZla2McHmsTtQYJJ7+34HHpWsk3GwoPXUgWSaaIi4Jbn5ix2k8dAvp7mpGQPAqHnd6Hnj+XoK0rLRmMYZRgL05wo/Crn9jR7PNlDbUOFXHX3PTiuKUWdKkjh2kEkzNy7kY+U8AAjgH6/nV5b+OCK6l3bp3JHm54jyAoI/vEjp+dbsujSCM8bd3IHfHb86ypdA3XG5iWwMhQO/r7AetKLtuW2mW7gRyXgt41YRp5b/APAiMgfkMn8K2XikjBDcJLMxOfRlP6c020sJ2EcZfJyWdsevYfQCusTTvtMoLDCLg8dBk4xWnI3sZOSRhw6a5IyeoxnoAcZ/nV6ezZ8LjDMMqT0yR0/GurjsBGqk87k/Vev6VNPYEjC4II4Hr36+vpW1OizCdU+bvExnsdyFcBgec8A/3SO49K4TSPJWbzZMglvlQd/9709h/KvUfilaLFZkyOsco5Qn7sgHbgEgjv8AmO9eF6bdtCFJ+8xyr5GMexGc16VOPuq5yN6n0Lo8qR4jkByOo7fSu5ilikjCxpg968l8LTs4RZTkHoB29zXtNhGsiK2PkH60vZofMyibMNMvAHety0s8ruA69PpViaBBKsan5sZY/wBK1LeE4GemKj2SuPnZPDbhlZh69ParMsX7vdjoeauQqqBeOox+dQyOTGSpAZB+YFU4pEXbOXvVEQJGWA53YyMe4/nXmHiKC7l1nTJ7Z2VFnTzI15Df3SOD/SvSr+5CHkFS2dp9SPf1rz6/uEn1WxQxNkyg+ZEMY543YOefp1opu0kOS0P0l8ESWtxodvcQxiJ2RQ6jPUDrzzg12Ncv4OkSbQLWZTksgz65xzn+ddRX11L4EfOVfjYUUUVoZhRRRQAUUUUAFFFFAH//0f1SooooAa7BELHsM1+d/wAfPGr6z4m/seBibe1yzH+83b8B2r7116RlsZDvEaKCXJ9B2/Gvyu8b39vqnjS68pi0e85b+8c/1P6V5Wb1HHDtLqellcFKsm+hWtXjVo++O/b1P+fetaK2SYhsDcRvGemeQCfYZJrAtmaadGYck8enP/1s/nWtMzQQvsJ3uNue+M8Zr4SW59d0LNsRHuQPu83aQehCjnt69fyqaCaSCeRo1JkT7gbqWfjPHYcmsn7SBckwHBRgp9SVHGcfhWhHIliJr24P7x2bap/uouM/QDJ+posIa17HbRzzZLCF1iHr0y38jUtlZi41Se43FYrUCNRkAAjGSxPH3j19qwriaV7a1wuQoEhUf3+ijj1Y5+lGrahJaeHfsdjjzJpI42fOSxzhj+DE496IrUpnS395A8qxW/zLwMjku7Y2j6AAsT34JqxozCzt7m/mOVJ8vP8As9Sq+pPy5+prjZ5yT5YYgRy43DgdAMZ/2VAHuTXQzSmVdO0kfKqymR1z1IYEZ+pIqoq7Jehtagq2oMpfbID82MnDP2HuAPwrlU0+NJmLg5UF29yx+X8B2H0rU1rUIr5pmVv3dqyN3wWlUkn1PyhQM+tPZXuEMeQqSgHI45Bwx/ADNEo9UCl0C3t33xkEFXyFIGAONoA9TnOPWuuFsILW98k9I/KTHdyCCfw5NILO2fYsK4SKSNFPbaqEc/nioZL623/Z5B5ccAaVx03YB3A9gNxHHfmjSIXbMuS9huJ7extcAuY2l54xtx19l/x71uW95A08N0wAYqJE5HAC4UfiMn8RXExNu1CW4ZQpO9FXP3UwFXOPXGSfatu5cLClwxEawZVyenGTgD1wAB+FLm0G4khujLumY7Q0m0n1C9APQZ4Fbnh2w22wuXjUyIcKDzlupGT2Ud65yG3SdYrdv3cUCeY2WIVe+W7kKo/Emuim8QiOx3WgECuQIh0YRbQWfnoTnj25NOnFfFIVR/ZRrX11FAtyI3KeWQpkB+bAGW2D1B4Huc1WkmuJfMikPlgKECJ/AhA+QHuxY/Mx5OKbZ28V3YwJL+8MjGZ3P8UmcKB7Dv7CtRrdY9vIbYBhuox90cepJY/lW129EZaI5q+W00q03R8MoDKo7/X/AD04rmWjVpGuA2S3B7EEjJ698D9a27yYz3QjgG5pSfmbnbjHUfSoW06C3gnZ2LrGwV27l3+Yge+MZ/KsOVLVGil3MqfU7hLqC8kQmQtGx9Bx0H4DjtyPaug0nUJFV4bt9kkUSPnOfnwDJnPHB4HvXOTW7yyPcXHQ48lV/vKDk+nB4H5+lIlsB5Ue3ejxvuYchUQbv1YEZPX8aIKSKk0zes76OTXQkC+ZcJvJc4JC8Pk565zxn0rVvpbWOeG25MxBkIY8krDkqT6gHn3xXE2X/Et1Ge7blzEIxz/G4VyB9F/QVorEi3UNyGIdrUIGPVWkYHP1OefYV0X01Rk0bWniWGB1lGYbeUpxxk4BYkezAj6U+Kdpnls3JDofNUDnKruHX8SPwqCG5iksntmf5ZSW2+uOSfrt5NZVrPLJJNqVk20uZRtA+7tdXAAPHBP61FlZMfkdlYmOVkkjfi4UMvruTr/MGti3nlR2kbknGfr/AHh9R1rjtKuoHjubVV+zyAm5jx90K+DuUdge4/hOexFbegXjanpUjZ8m4gBVx1B2nKn8q1jHXQznsdmMXcQnYDcMh1759R/Os69tCuJYmIOePcjn8DVmzmItwxG18fgf/wBVX5AwifzADk546Eeo967VTUlqcTk4vQ42ZmmUtt+Ykkr79x7eo965fVRb3CeckoRo+WDg7l57j+Jc/lXeS2yyszQkFiM4PGR7/wCIrzzxOEClpFMbjgNggq2O+Oo9uhqYxa3K5kyrbyoApeUbc8eWDz9STn9BWst5Z8yZ34Hc9fw615O+sXlmTuEaK5x5o5T8Sc/TAwfUVs6detOi7JlfPBb7mPrx+nWsZuxvGNz0QOJ5PkVt2MklflXtx6YqUWaRjZMfLZz90AE8ep/X2+tcsk7EeRJPtRiOWJDN745yPTNdRaW+3JHTuTyx/wD11EbMJJofaiOOdSq52ZyegAHXPv8A44roLBbeQyMeDOQMf3QoJ/PGM1h3UQRPIiOS3b174H45P4VBayXVrdbTwNxkYnqA/B/SrjNReqJcbrQ6qzaO4td+SBuyufQ/Kf51N5iMfIz85LbNw4Yjqvsc5xWBBcSLEtpzhXGD7nIH4etaDL5kDNKfldt4P91ud35EHPtW0ZpmU4nnnxJ0T+0NGuHKkoqksDnKH+9x1HqD9eor4itb6W31l7aXywFbaHGBwPoQCT6Yr9GLx5ZrZ4F+aZVwAeQw7g+4/UV8HeLdO0+TxBcRW8c1nJG53ROqmPIPODwQD2P54ruoy0szmmrHqnhi4UbdnzF+2c/mew9a9v0icuidwOfqa+bvC96LVAiHDHgk/McfU5zntXvmg3ieSpkbJb05x7Z7n1o6jO4Qt5gYnLHOT710toEMXv0rmYdsuGU4ArXtZdqbl9P51LdgsbZJdNydCeRVK6Yqu6Pl0HHoR6Gn7iqK/Qd/w71BLOjofLP7wZxnv7fUfrWMmXFHF6jPtaSMqXhk54PIPYj0I6VxOs6Y1xf2ctqFLxzKfnO0k8dD0z9RXpNyY5laSNVD56fwn1+lcRqXmXM0FjHIqM8ihCWCkMDkDnrmoop86KqS90/RvwI27w5a7i28IMhsZB+o7V2dcn4LSVfD9qLmIwzKihx68da6w9K+zpfAj5ip8TEXpS0i9KWtCAooooAKKKKACiiigD//0v1SpG4BpaZKcRsfagDzP4i6pBpugXV5csFijQliemPp61+XcV2dR1e+1PaUjlclR/dXsPqRX2F+0b4rH2KLw3A/+vbMgB6qvavk6CCOO1YLgs5BwPzr5fPcWm1RXQ+iyjDtRdRmxY+UybeFI79hnr/hRqEq/ZnmVwFR8KSeuByfoOT9MetYAmNqArPwxHf+f61jeLdX+x6dDZr/AKwhnbHbd/kV81GDbse42Wbe5bzY51J27Wlz0ycBUH5ksaveW9/eSRSPnYPKBz8oLkFm/wCAiuUtbnyYo5JWztGFGdwbYM8/ienrTrTVGEQUcuwO8f7Z469uo/KtHBvYEzck1YXV1JKj5t7dnkC57RINn0GR3681k2089wIbE8rGYArdw7LuJ/EmsaAtboUOcSsFf18tF3H8xXQI/l7Jx8rTGS4Y46fKRHx6LxT5bBc1kZoYnSX5lbcR6LtJbk9Mkgk4rW0S5lWGS8lCs6xnb2wQck+5JIA+lZkNutzbHByzxsVPqCQrZ9sc+2TV95Y7VBIr7k8oSMByuA44H5k1CXQG7m1Zxq9w8IPzvKGYdz5MeAM+gOfzrQkuI/N+xKd0ShwD2+UrnPqSSfzrk7bUthW5jYiV45WlPTglRx6HkfhWva34tra4uJSG8uRNgbsqgZJ/4ESSKUkxI27fxFbyaemXILR4QHuyEhuPdhj8a4uzu7u+vZI3XfHIC8/spwVUe5PGff2qt4gAg1RZIZBHErM0Yz0V+qnnqD1p+h3kSRXLMdspjLDjum5v6jFJx3aNIvS522k2UjsLu7df3yY+Xpk7l49Bnv8AX2rpY7M305R8GKCTpj5flznPv0Jz9KxdPu3SK0W4yrlFJ4GMKDhc9M8kn6VbbWo/3KwKWW4YLtA6+azck98459Mj1p8q6GbbbK8qxXLywqS8SmSMr1BaNVzkDk8uBj1qhNb3VxqP2Bn3HBaR8AgNIfkUZ6gAZIxjp2rZ+1W6Myq3loTJukxhvmIJxj3LN74FW9Gt5dUu/tAh8mDeuQ3+tcLwqk9FGAMgcnuaI67Dlpqy60674ULbQAwxn+HoB9TzUV/qM3mBYxhArEDudufm/M4H0rQFtCLvzcZRd2COnHDHPoBwPc+1VUEZtZdWnOEYlIwByw6AD25zTcXFXIvzM5SW7fSza7TlicMO5PBOT+B/OqMl9MYIrN5CEhLtJIOhd1JY/wDAASP96m3cjteR3Z5RSRtBzg5I/kKuNYpbWqxXvMjNkj1Zzu2j6/yqL2jc0S1MdtQeWZ7qJNi4XA7IsascDtxwPc1clvbZL6CwkO3fbFpMtwFt4m2qPqQXPviqN0qLOlrbHLSMrk9eFy+OPr0+lZ1xayPercnKmENCPcurE/zxTjUG4GnaXCXG++lUCV2dVJ7B0WIMeewz+Aqnf6s4v47ja3lWjxSbXPJA8xGBA7NkN+NIllOyRKhO0Bdy56lDtP4nqfY1A2npNfxyTSfJMvmFjx8q/L09F71cat9EQ4pFB/FCQa2t0TmKQSMmOhZB5YPPQFR9KnPihdH1mO0i3fZ7gtMJABx5o8wfz21Cnhh7drm0nTe9uzKjYyAh52t6DOSD6HNSajpyafLFJqMXnCHy4HXjIDLlGHr1I/Cr5r6JC0Oo03W7K/WKJphDdqdqnoCrLkf8B5/MYrttE3Iwa3O12jZSPcHkfUEdK8ts/DsDnYrgLKSIZN3GCM4PoD+hq7aX2seHrqO51COQAsY5uhxJ/C/HYjr7j3q42vczl2R9AWdzujVJUAdS2cdyP5VorNjCKdw5wO+Pb3x0riNJ1cX1tHcnBZgQ5H98DqPY10FvMkn7gkAj7p9h/hXdCRwzjqRTuPvRkr7dseo9K57VTbXlttvMusnAdOcHHRh/hWvcSbZN0q7gThsdj/eH+etcvqQ8qQ7SEL9Tj5W9MgfdPuKYkeXXOjQx3heznG5uCwfIdRx8w9e3OeamsNKAUmSYMp6BGJGT9BTdVmMV2Xb5MnP3TgH13Lg89/et7QrwzgsJlx36Fh+LDdz7VlVp31NqdRo3bLT0tYwxYFvX73+fwzXQ20VwVCdEXgdO/c4qK3MWP3Sl5D69B7k1pweY2CwVQOeTx/ke1ZRhYtyuI8cdpt/5aMwIZj1wOw9KR1jYvI+GG0Ej3IyFH0x+lWHg3ZZk+VcDP64xVJnYHzScOBwOuFPBz6kmnJISGK0RjmLEAKQAewIBA/Dv+FRpfNZzRxFt+5nkK9iQPnx+h965W7vCjpbnJVhk4PocAfTqPxqlJdSDVLZyfLDuQP8AeBOD7Dt9KwVW2xr7M7o6pDcRp5a4IAAPXcPT345XP0r5q+M3hZpL1dbsGVknBZl3gNuHXhsZHrglh6GvfNOCyItxJGTFgElf4SOuPp/KqXirRNK8S6TcaNqkRaN18yKZGwSQOGBHRh3HX0rtoVWndnLVh0R8leFjj5rpGAQ8Andk/wDAe1e9eHpnnkTA4GMA9AP8a8gtNAbRbtre6kynARgCwkH++OvuDzXqmhzC3ZUyO24qOnooz3rsbT2MD2O0mXZheff+ZrZicIMgZ6AVxtpcB1VTxz0Hb/69dJZ/vThTjAz+NZSZSRfutQjjDFThh0B6GuevNUB+eDIzw6N1Vh2+nv2rVurVbiPyzgcYFY8lgdrEjLgYPcNx0PofQ1xVJSudEErGVcamzqZmyG/hZeuff1/EVzEl8mrajaKYhcAyqSUHTB6Fex9D0zWpqsU9pCbiHIUY+Vxxj/636VzmhW0kvjnTryx4LOpZVOOfY/41eGm3NJk1orlbR+pngsxt4etXhkMibAATnI46EHuK6usbQhB/ZsMkKeWHUEj3x7cVsZ+bFfcQVoo+Un8TFHSiijIqyQooHSigAooooAKKKKAP/9P9Uqq3zbLOZ+mFJq1WRr84ttFvJyMhImOPXApMaPzF+L2pzXnjGbzmJCdMemf85rzqXUBaxGXglS2APU0eLtZOoeJry7LbwHIHcYFcZdXfnk28WcKPmPqc9q+Exq568mfZYVctGKNSKSS5ZJbhslwAR+n51y2uXLXV6sjnKOyoM9wvzH8OK1ZLtYrctn1P0APHNYcJM88UzkMbdtuMZAAUkk/57VlFa3NmTzXM6L9kVczMNgyfu8bifrmt4RrBG1wSSHDvyeCfvk/gM1gI8cc0d8hD7D8xJ4JJJJPtkfjXQRt5+nxrL/HCwHQ4dwsfH6k+1OWiBFZMXNlLPOf3jbYlHQfOWzx9F/Wutgt1ubSdVPyrAGX/AGlWMqB9c1h26w3tx5MOQqYkBAzngKMZ69atS6iIXuGhUqQse1en316H6ECsHd6Is14d9gUgZg32fCY/2W2hzVKZZJbK4tl272jZG6922L+ozS27mRBdDLSThiS2OpAP9PyrI1C6ngjZEHzMoyR3yc5/E0kndDNm0lVyyyybVYR4VV3Eq2CSf0/KrzHz0l05fnWVVQufXIzx3OTXPRSMs1kD8rZAYHsvPBx6L+tdfdQrarAbNcyXCvlge4zhuenzHFTJPoVoUbqKTUYmIO8tIERvVSwUNjr1zmrUq+Vq0kKDYrqYuw+WX5RjHfpVmWwEbwshKqFyP++vu/gKzrmVxdy3yHcETeOORtbYmD2wcEDvQrX0Jvobeoa2I18lcERyLAORypIDMPrnAo0q9AS1vJG3C3QsPQswOP1ArktWjW3iBcAtGsarjHDjPOP724856YrZhmWGCJZ0VvLJkKk8DjgH0Axlj7YqHqrRNIpLVnY+H7aQx/2pfA/u0PlqTkAE4Vvd3JwB9T0r03Qre5CMj/uzHw5H3slTgD0Pf2+tcTokiu4ku222tgw27+styR97HpECMDH3iB2re1PxImn2UiQt++Q4X18xgfT06nNbRtFXZjO8nZF24kingNhZDCHcHbH8Kj7oz3J4H6964/XtRW5vILC1bMVquQqj70hyOvoP1NQ201/cWoijBXzfm3E8hen5H9fxrXt7BrYi3sU3Tycl/wCLHfk9Pr2rmnVlN2RrGEY6so6fpbxpEswwSRnPXHtRrZS92vEoI4RO3J+Tdn/dB/ya6me1t9Ph33MmXfnPbHqvfHbJ615vqWpCaby7RcwQKWyRhQTwB6kk/pR7OV7C9otxiX9vZa7bk5lW2LZf+HkYAP1I/Ks7U9RUXe2D5i0weNR6JkE5PryazRY31y7PduNvLAdBzxwPoTyaz7y7ZtSYWeWKjYG5xznOB7ZPJ+taKlshc/U6nT79WjIuAInilYnB6oMY/EdD7Yp11fW1i0V26iVMvbyoBygckhh+HB9a5fymhXy1YlFDE45LFzk/pjFWAsl1HOrghlKFieQu3g/XGce5FappENNlvRL+50a8NtfMZ7WeJV8wgnMeONw647eoqbxM7XCxwWh3hCV2Z3AqvIwe4Ixj0rOaAz2W1ly1uTHnPJjbp+IYfrWdPFeQ232mMMstowfj+JM7c/kcH6CplK70LUbbl2z1O5s5jcoBNaSkGSPuFI5A+ma9BsZYtQguLJ33q4XY5OcgD5c+9eaMFZ3YL8kygtjsT6egNdTprvpm4qTJE4GfXB/zilGre1yZwPRdGtWhRVU8SAblJx0GP5VsTyusBuBkSQnD49OxqlpTOpVh86EAnPUZ5z9Ks6g01tE00ZyrfLz1UnsfUGvQprS6OCo9R0t+l1CtxD82eGx2I9q5q71KEo0FywVSPlb6eo68Hr6VjSXr2lwQCE83GAeFPtnsfesLV9RlQNHeREg9HC8n8OQfcjrXRFXMmzC8QajNYy+TcRiWPkjnkr/eRx1x6HNO8P39tLN5Ts21cEDcBjPcZ5I/GuD1K9jlDRQuCgOdnPykdwDyM96zdK8RT2E4EbM7xnA3HsfqD3rSUNBKWp9X6O6XalwRj1Jzg/TpXRAlgP3iqMYyvJ+mfU+1ebeFdaW/VfPjXnAyWzz7gY/WvSLXy5NvlKqkd8/4f41ycupvcvGRXIt4eNh5P8/x7Vk3cayeZkggjPp1IwK1JNqxFd3AGT259K5LVr5lHlQ/Kq8uxz0P+eBWdSSS1Lpxu9DndRt4iBOTjYRjrxz1/M1mTKpkZsZxIMBuxAwcfnVs3quzgEhY0z9STgcdO9YwvHlFwYSciMHJ7kEgt9cYzXC0tzrSZ2djetaSOY2O1SSQvI2nk8ewOSPT6VfubiVAU2b4/vLkfKc84BHQ9fx9q4e0uZgI5weUO5+fXgkf7px+FdXGeEaCUb4+sZ6YPf02n34raE+hjONtTyrxfoKxXyavYwxrDNgOMnG/OckDgH0PFVbKRbd0EkgkbtsOV/Pua9d1ezj1DTZtjbJcFk29yozgZ4OOoHWvJLWyjvALpwN7DG8DYT67l6AjvivToybjqcU42bPQNJlWUoEPXv8AzruLZ9jAD2ya8x0w7ZlZcqq8fh613kcjSIET75APtn/9VEnYSR1cbJcJvUDzEGPY5rFvXuIV8+LGGGCvuO1WrKfCsHGM/wAhRqMZKOMbt/PpyOo/EVyzd9jaKs9TjdR1F2QzRIZIDw4HJj9eO4rn/DsVvD4u0+YOcbwQyjkjPtjJFXbxJbF9okOwk5OOqnsfpVHw8qw+PrJLfAB2bk59eCO3NZU5e8mFVWi0fqvo/OmwNuD5QHcBjPHWrFw5jKsKi0sqdPgKdNg6jHb0qxcR+ZGQOor79fCfJvclVty5pOajtzlMHtUhHNUIfRRRQIKKKKACiiigD//U/VKvO/i1qLaT8NfEGoocGCzlYH3xXoleQfH0kfBvxSR/z4yfzFZ1XaEn5F0leaR+R8ty0ss8khyXk+ZvyOPxNZ0LtFtixmQsTn0+XI/n+lYFzqDmWKPI65Hrk5/+tU8N0JSJmbhF5zxyBtxmvh2ru7Ps00lYdqFy5U26N8p3MT6gcD8OtQLcGO0ZtxDdzjkkkf061KHjuNsmMgDaR/suCP0JGaxLV5CZBIS2FXH1LA5PtTS0BsnivH+zgclpUBX2IPp/Ku9tJwbGKLIX94kYbHOeWJOexNcZDbwsZPKziJQqgf3Vdef1re06YXLjzgcRO0qgHGQi7Rj8aU0rBF6mvbmZLZFVS7yDA46bpFPGParIWRRNdxgne5PP90rkA/StVzFZ3drclduXGVz0UyADPpUN+/2dYooxtVy5b1ARgOfqAa5tb6G2ljbifc1pHE2SJZGOMD5UXCj8cnP0p4sVlHmTgMI4trY6/ujxx7g4+tZlrtj1GBN4KqyHg8DcDn8iK32uI4r3epULLA+4Y67cbv5VNgZnadaC4ypCmQKyrkd9uMfmQas2t4WuhHKwCBBuBOQGXaCPqzsSfQVEr/Z2eInO+aNx6jzcEj2wRgfWktWt4o5rqQHYzSsp4wcyfp93FOwrs3SZnaS0243F85PTORnH1X9azikiziGIbpZtuEHZf7x/E4H4ntV6W5iivZZQu0qDu56855/GqtgWea9vWbZPIzIhPAQ7eST6KvQfT1rOK6lXKD2yXBlu8HMPRe7Ox2oSPx4Hqa6+3tLe2htA43PJ8xjOAo2EAFvXBAPPfntVzTrC3ggWYR5OVZN3HEecOx7ckn16VB50t0d0eI4cZaZ+mOo49CTwo68E1LfYerNNNWkZEZUDbFEkeeDliTuOe5J3ewq7pmiTXzebdKJNmHZj0LEc59uAMdags9PLSB5ydijzCpA3tg43v6Adh6/jXZTW0klukfmG2Rh8qj7yg9WPq3ueB2pcrm/ITlylISRRE2tmPOuScE4woA7n0x2Fb8cX2OHfJ88koy24cuT0B9FHZe/8oYINO0wJ5KgEYCRk5I7739SeuPSuZ1jxMqhltjl2Jw7dz65/w6VqlGCuZ6ydiHWZIFR57iQFySSzHOcDsOgHpXDtOkQSGMGR3O47z0GPvEenpnr6VVvNSkvrsbnE5UYB6Rrj09frjH1po84/LYANITueRucehOf0H4msnUN40ieV4bdXSV2kmuCNzOcEL14XsMc+p+lV1shcrJJHiCDd87fdLL/BEo7DHLHqamgs5EbAJeVieW5Yk8l2z69v/wBVb+leFrm8YtKT5YJz689QPc9zS9q2U4RS1M+LS47kAQ58tQeV/iJ6tn25rp7Hww8kRgWPAZQSo6gZyAT2Pc13Wm+HpHWOKNAAhHAH5ZNej2vh62tIBA+C2Mt7nqSfr0FdNHB1KvoclXFRhseNjwrEcELiNRnH949vw9KyG8PvvZ3X5CCpHXKnr+Ve5XFjDzF0XHb+Z9sVnXNmokJiTauMc9Rgbj/hXQ8DYxWLufOraC1rKN4ym1s+69R+taFppA2CIfeJ43dMk525+vSvT302Oa4BXiN41U8dCfX8aYdJ+yfuZo96sQwI5zg9vcYrCODd9TSWJ6GPbWstqiSEbdgwp9j2I9P8in394Gt2C9CMPH3Hr+XUGt2aRY7c+W26PJOeuPUMO2K4y4kt5/nuNqgnC84yQDgj2/8A1V304cuhyTlzamLqFqHi8qZ1KHlXIypU/wB7vj17g153rE95pcR8+Iz2hyrRk7sEdCp6jjv3r0W4kNrF5hbzM8qGGM54wwz0/WvIfEt19qQ3Vk6hj8rJ82Vx1XjnHcccV0wRi2zzPWbuFroXNpIxQ8gH7yexB649azYpg8gkjTc/8SnGGB9ORTLx3LbJGUr2PQ59uKoLaNKRuGCOjY4/L/OK3toZ31PdfB94IWRbWOONu4RIy5/FmJH4CvoPSr6ZolaRdo9B6+7cZ/Cvj7w3BdBU5XOdy7wcbRnnAwSc9MntX0BoWoTK0UbsWJxztbn2BPb6VxVYWdzqhK6PWCkzrufKlugI5OO+OwFcdq9o5/1mfLJyRnBb/Ae/5V0MGqhiyq+EX7zDkgf09gOamkmt57YySjABPLdz2H0FcVanzLQ6aU3E8seGfZckHfI5AHoB/T0qGK3dEMEbY3KVPHYL/ia7iXT4PNZCwAwHY9Mgep9z/KiysIZEa5YD5lZ8ew46/SuT2bvY6PaKxy8VsSIpJQVJAGPXs36VryG3jlt5IQU4IUnkbckFGHdSQfp2rZ+zn5VH/PYEZ9MHANU5rPLrGV2iMuyg9wSSR+WDVqNkQ5XZLFDItpMjRllIBKA5OR02H1H8Pr0rg2v7G5M0kbKxJIPGCSPXOMN6g9/evQVjENtucHf/AB55yh6Njvjow/EV4p4mSex1czQcSORz13D2PRh2zwR0I7120HbQ5KivqdlpjwsS78DAAFdvaQvEolPJwT9Djj9K80055JTGZeApBI9T2r0/T5fPwgO1exPfHGfzraaIRupCJF2qORyfpVhXUw7cElevsRVOW5eI+YvPHT168fSmm7QvHLG2C/4cjqDXJKSTNlFtGFrVisTtNgPE/wAxzxj1B/oa8+Se1svENvOMgxMGUcH6hT1+v516zMgu9275WyTtbnOeM5rktY8NPdQO1uA04Gfu9x2ot1iRJdz9GPA+u22ueH7W5t3MnyAHI5Bx0NdlX55/Cb4vap4bePRNSLJbg7cDBxjjo3TH0r7w0HXLTXbFLu1fepHXj+nFfX5fjoV4K2587isNKlLyNZEKOfQ1NRRXoHIFFFFABRRRQAUUUUAf/9X9Uq8b/aDbb8F/FTeli/8AMV7JXjP7QpA+C3iokZH2J/8A0Jayr/w5ejNaH8SPqj8UsCQrLIfmUA/iSRip8bLfyhhi6g8fUnP51Ru38t/KTAKqf++c8fnVtd7SxIBwF3Z/2ck18a1pc+uuR29xNAquvXZlh7delWIpRFYGZBwqsCMdR24HYd/bmrgtle1N+gyNw+UdcY5A/TApjaf5ZXY/7iTmNhxtJ5APqCOtTdFalewjuokkmQn5TxnoykrnB/CtXTrpV1eFlb92QUfP4ZFXk024miTzLcxNCpIAzjk8nA459OhrGn05/PLw5R0JGMHn2/D1pOSlcdmjsYpYr3TILwvvH2grIQOQFYn9c9PaujlW3nvD90O4QbegG8vjI9Mr+deSaffLpbm2mJ8mQZYA8Z6E/hXbvq9rcR+bnfIVEcmP4thZsj3OeD+FRODQ4yudXeMtteRiMAEL3wORkjH4YP1qJpViuVWTc4wVB7ASD5vxxVbSryPVmtpdwd1lABYfeGNvPueDUrzi/tUGQsmGbcO7RZX+WK57NaM0Kj30iDznJLufvAcZRWK/pinm5OyLTnyy8NJtHqN5/NjwKz2YXVq4UZY4Ck56E7SR9eKcJZo7tDAuZpnGAOTgKvT36VdgTOwKyraz3cimS4uCcAYyOQQMdskfgDzXSaTFIfLmm24KlwM/u+T8zt3OSOPUDsKyra1iESQuwuFi4ZMnaWPPzFevPYdQOa0w5urd443Hlg5lcDnC8HGOFHYKDwcDrnHNJ32LSNqa4W9Tz3DGCU+XEucM+eCwA6A+vYZNWYLCOe8hMhyqkCBF5GQOSFGee+egA9ScULC0mvZkvZGG2TKxogx+7HBwD64wSeMZ9a72wFzGGby0abZkAfcRe248Z9amKuwbsixFbQQEGbCow3FM5Z2HTd7DtnjNc9f6xcTXAgsUM0jn5mXn6AHv7f5Nbi6feXqO17JuQ85xgN6HHH4cAeg71UurMW8vl24AB4GBzz1J/wD1VUlLoRFrqc/9pngWUykPI24MYwX6dVT15+8x69uK5S8Fxejy3QgHjYME8diegHsK9JGiLy052IqjgsRwPXp+VPRNPjP7mJVCg/M3T9OT61m6cpPU1VRLZHnWn+GbuWZfMGQecdh9a7W18PMh+zRDJH3sjoTxkj19P8K6KxmtlbI+Zx0JHAJ/uj+prrrCS3tcNKBtJzxxz7+taU8OpPVkVK8ktjE0zwT5Cq8ikE8lm659fqa66LSIodsMfCL1an3GvWyg7+/YnAH9a5e+8SJIfLj+c9hnAPue4/nXdy0KW2px3rVNz0OK40+zj+VgdvQDnJ96z5NZWY+UgJd+MdT789gPWvN5dZ3rs44HRDxUqakI0I4HHCjAX8T1NU8x6RWgvqS3bPQjNbpEsSuGcnc2Oe/c1TnvYmDRnJ3H/Dqa4GXVJDkLJ7nbwPpmqEsl9McE4UfhWU8fJ9Co4SK3Z293qNtGdqEFj06ADH865+fVVjk2EmUEk4XsT1xWG6+UdrSFn7nqfpjtVSdoo8Ce4K54CqOSPr/hXNLGT6HRHDw6k9xLNcv8+1dx+5nOf+Ajr+NS2+l2hYXT7rmZOuQNoIHGB0znv0HvUUKQKwZSF9MAkk+5NbkYZ49rvtxyBwf06fnRTru+rCdJWsjnbrToLuLM581yNohT7gP0GWJPqTXnOtfDzxBqkjl4QIW4WMLl8EdCAR17biSO1e8W6nAKOeOpOOf6Vu2ckFqdxXczdSzdz6mu+niPOxxTpeR8qaL8ANXu5Wn1XFpGMbUzuP0OOp/r7c16Db/A1rpUtVZY4cAMAPuqOdoI5JY9eg/SveZNajQ7FCn1OcD8+v5Cta01NXA+6sajnA2j9eTXdCtGT1kc0qUktjxuL4HWX2mCeZlEEPzGJBjewGF3NnoO/b0ro7L4QEv5z3HkhioO3OAi8iNM4wO2a9Q/t21UgMwZh0Hf8AOaB4hkQb3URgev+Fb3ofakZWrdEc7F8NrK2hWKGRlAOdoAyx7ZJ6AVg6v4I1wCQ2Xluq/6pCcRr7k8s5z24Fd6/iyBT1OT0zirdtrsVyQQoH15P61MoYWekWUpYiOrR4jN4RvLWHydTnEhUjcFGNxPOSc9KqRhVkcOAEhUcZGBgbjnHHGRxX0JKthcp/pWHJ4wB2rAl0DSbkNHDGFUggjGcA9a554JL4WbQxT+0jxW1xHCgky7Fywz/ECev5VZlSM3L2znEgODgjqqgEjPrmu81bw2Qmy2cABSucc47fl6Vx82iXE088sCkb2LbyeVyAD16dK5J0nDc6I1FLUwNRDhGlU+Y6jeFHXvnb69OR7e9eRfEPTL24toNX0xC7j/AFkQIOR1yAepx6c17e2mXLq8dz8qE5Djs3UEZ9/0rjvF+nvd6JLFZQEycNHngsQckexz0rKLd7oqSVjz7wdcNf2+Y2yMZAJzj25r0uySeGXDLhX4/P8A+vXE+ErN7Z1ab/XPyVIGRnn5sd8/j617Db2nmxZcZ9a73HmV0cvNysekalFY/dyAR1wG/wAKnTTlGQw8xcg5A9uD9Oa07ezCgbujZB9eO9b9npUkaCUEg4xx3Fc0sO5PYtV7I56DTAi72jyg564P+cfnWtb6bFcA8Bh78EfiK7uzsIgRsXAPr1rYaOKGMhVHFdlHAdWznqYq58teLfCLWeoC9tElR+zA7xntjGK9H8BeNPE3hZzbsxlRSN4PIAIJxknrivQJdOguJlaQYwc8U5tA014fJCYBbJPcnjP54xURwNSFV1KMrEyrxlHlmrnpWm/FuzneGK6h2hiA7dME4HA5J616np2q2WrQmezYsgOMkEV8oTeG5Ul8+ybYR6/pj0re8PazeeH7qNbmVtmegJwc9eK9Wjj60HautO5wVMNBq9M+o6KzNL1ODU7VZ4WzkVp17UZJq6PPatowooopiCiiigD/1v1Srxz9oLj4L+KiBkixc4+hBr2OvL/jVaLf/CnxLaMM+ZYyjHA5xx1rOsr05LyZpRdqkX5n4fW/lvdLLL8yr27ggZP1Hep7e4WJ0k++A3U9MZzj6dawZfPW9ltYx843A/8AAsD+ldHaxJCEjkAIjHz55ycZx+Zr42pHl0Z9dF3JFfz5HhhGFmBxj+FwN6n9BWojefaNCNskUpyQQdyE8gkDPGc8de4qhZ2Qe7FypLLHkAD0xW6lmj+VDEm1Ao3Y7+hz7Vi2tC0SadPqMUgtJJQyx/wBjkbufXkd+n4V22mCAkiSM3Ibgo4BAI9GIzx61hraQXMsHk5WZMqWH4A59a2BEkcmyMk7BknHUDgcjnJzWFWz1RcXbRlhtD0K/JTULUiM4OxJBkn1BOCPyrnZ/AGjeZ52l6jLCWHCSx7h6jDLj+VdfAkMgLs3mIFB5IJDdx0zUrFJCHiBOe5JA7Vkqko7S/r5lcqfQ5G18Gazpd3Hdx3UE8W8OVVip+XB6MPr3qWWwu7HVRI8OLfngNuJDFt3T/ezXfW9iZl3zTHccfcJ/XrWkdP06Ebxh5TxluT6cU3Xn9odo9DyeHSdb8tVaB23EEEj+FenA6V02m6Bd3eoJJEjMseN7DIy2OEBIHy55bua7FruNX8yQbm6Y+n9KedanCBY3Ck8A9Mf59qTrtiUexBcaLKqrbuCqkY2qwRQc9vTPcmpbexjRIracgQxsT5a/KjEdPcgc/jVEyfaD5c8o685yRXRWNva5Qt8zZySf8Kwu3sabbnQaZYqkKQxEIzAZYLwB6DP5V2ELW0Uaw26F1U9CcjPUliepNcNJqHkE7NpwMcNk/jzTv7VncEOfoq9fzrZTsjJxvqdlc3sa53EMzdk6+wrLkv1gH3PLLAcdT7+/wCPFcjLrEyt5UeUPY5/wrImkklBxOArehLN9KmVRvYqMF1N6810SziBGL7cZwPu+1RRJd3Tgs2xRxwMEfnWZCFUbly+3uy8/hWlHtX19ueTWDTb1NVJLY6i3MNquAwPbAI/VqdNqT/xFWx0VTx+J71ySXcAO1gMD8cmrS38Eg8uOFmZhj0H51abWxD13NiWa6u1DMQF7Y4x9arpbzZx8oHODjqfw5rMIULhgR7E8D6D1q4sicKXKj39u2KVrjv2NWGMKB8vTj5jgA/QVYWO3iHmTMu7/PassOF+dn2r2z1/AU19RtoiRbx4duNz9fwFV6CNQzRb1bGMdA3r9BVSaUuhZnwPZSf04zVSO6IBctuYdh/XGKryXdxdsVMhKjsSF/QVEikicTRoCqxsxzyWJBP4DpVhYc/vpESIn2x+p5rKlaS3BcThAPTB4/U1mnUo92ZZDMc8E9PyrNRbKudMRHIco+BnqoPb61rRzY+RTwefTmuNi1GSYgIMk/7PC+/Na9vjcGZ/mPGTWqViZHWwSDfkjfj8qu795ywXHYDnH9K5+3uY412liR7nithbyJQGJA710RMJJ3NSINkFVUe5GavMHcgTMX9c/KKwlv4ACzMTjnIzmpP7UjXBjJx/nrzWimkZ8rZuc4IjTBxg46VQl39T87f7RrMn1CaZQMkp9MVX+2vs2qAAevGTUTqI0jBmv8ztiUggdhWlb7jINik+3QVz8M2Puvitm3llTBGR74p0pK9yaiZ1kMT4ElywQdQBTZbyJAY43z2wOlYnnM6hnfOR3OP5VA0oiyHIVR/dGc/jXofWbKyOP2OupqNcN/rJnUAeo/pWJfatbEeWqbyP73SoJZIuGKs/sScfpWXLP2ZAg6cdf0rmqYh2NoUVco3N09xIHmJ29Aq/KPz4rMuWuLpCkZ27uMjqR+PNX55ogMGJn7DIP8qqmSeRsEGNe+BjP9a43Wex0qmjMs/D8UMomkcK3HCjHPue5rt7GGKE5B4HU9Kz7aNEUGNcY7t1/CrouEAwRlfbmuqlWkupz1Ka6HTWos5G3cKRzjsK3I5FTJWQdOB6ivPGuyn3F2g+poGsSRLsTDdhurtp4xLRnLPD3PVYb2MDAOW9+gq7HJHIclgTXi8OszRuFkfZnoByTXoGj6q8kfKkZ9RjNdtDFqbsYVcM46nSSxjdlTzU1uw6MaqyXJY7UGTjJqSF8oG6HvxxXVfXQ5nF2L7EYOKxdSiW4iKggHGcj1rSdg6lT0PXPFcbqOqraylFk46gdf1oqTVtRRi+h6F8L9anjv30qc5x0PPSvoavl34ZW8t34me8j+4ByexzX1EK9DAfwjhxNufQKKKK7TnCiiigD//X/VKuQ8f6Z/bPgvV9LyV+0W0i5HXpmuvqG4hW4geFxlXBBH1pSV00OLs7n4A6lpwsvEN9A4xIJSvPXjPJHXvVqCZZP3MSYXGGx3yDyfYYr3H9pzwNJ4G8fPeWybLbUMshAwAW4NeGW9s9pAtx91iMqOvOeBz6ivj8dTcarTPrMLUUqakjZ0nTpAxkAyvJZugBAzgew5rYHlvEwUjywkZcng4PJHsSOB9aoreqBNAgPlsTgE43LjYpP86rtfItvIjrhskBTwSwwV/HA6VyWbZvfQ6OG4TTwbfjdJtdTjhWBzz9ScGpbeeZ53VGG4/KRkY69B9a5qFQtv5spLMV+XPTgir5litysiAhjt69ee34VnKOpSZ0LMkUxtZQzdNuOmDgg4610cVuBB8xLKoJ+XjBrmXn8yHzIhhhjPfp/wDrq8t8YAGlIjfoR1BFQ43RV3c6aO+8tNsYHAHf171VuLxZQ23IbHXoM/5Fc5JJaeZINgBZQcgHngHp6Y9OlVVuITMPsxbAPOG4/X0qOW4zVa7lUjfISBg9OxqaO4jeTzcbT27k1T81Ng4UZ7E5+nT1pSkojWQgBccruHT881PIUpG9HfAv8iKoXuSTmpVuJT8rSAhv7prmG+0OoSKNRk565yP8KmiWQRn7VlVXqVPH0470vZhzHQx6nDbEnkdvUVeTVJZFV49wQewwf/11yi3ljHhVXhh/FjJFLJqSqmI3VcdB2/Ok4Bc64XoZ8yKQSTnIH6VIt3bQjIP3fTA/+vXCJqTNyVJ9cEf061YGoMjfMDk8cYx+gzS9kx3R38N9CQCZME9Tgk/nUjanbRAbQGHr3rikuJHwJHAJ5/8A15NS/b1hTfwcd6OUR2a6nG4yi49SQBikF3CxwhJyeey5/ma4OTUVY7mcAEZGCP8AGlS83/efjHqM80nF7jPRPMjAzv59sf8A16WG7tlP7xgceuTXCR3RZf3Qb645/WpV2kBpGOO45FS7oDuzqNqcsWC546c1Vl1G2hDFFyT0PGa4xrqONti9O1PSZCPl+VuvJGam9xnSvqyldhGB3xms6410QKVjXGfQdKz3fgbiGqi6zyHjhT69P0qlFMLkr65JM7eWjOT9MUxL2/3F1Qfj2/OodgQ4Y/MPSpY3QY3NjPbg0+XsPmLi6hdqCFySeflx/wDrq3DqV3ECX+UD1z3qAW/AZhuB6YIBpj28VwAIXBkB6NhsD6ip0KOis/EAjX55GP8Ad2LnP410MOvKyhihbjqxAFcFbRzwyhJpYzjtgqRWoJNp4dsY5xhhSvqJo7FNTVmJfYR6A/8A1qurrCkeWsDMO3GP5DNcdFJKy/6MRx0IVc8/72BU0l1eRcs0iknuVQZHrxg/nRcVjrptTm+UGPA6/e5/KrC30zfcjCAj+IiuPN1qBUtFIip33gH9RTre8Df8fLRPj+6x+lQyrHcR3B7kJtrVgutxUPnB6NXnkWr2aS7bhmxnjcrEfz/lWxDqFpMP9HZsdwoxn86uLa1IkrneC5k2hI2xg/Tj61SknmQ5YEA8jvWAl/BkRsyoexk5/rUxvHWM4kVsHtjj861U2zHlsWzPMQww34n+lZk7XjtySqnsWx+maZJeyAAiSPB/vc/qKozaiYifLn2n0HNS9S0TSRXGcMWcjryB/I1ArPFjcEUn/aOayJ9RkUbmkaQZ5UDms06lAmCylD7kfyqNizv4bg52uQTWgLlCu0/L+GR/OvObfXF3EBRjOck10Ca5A0eFaJT7kg1pGbInFHQyMjNtLY49OazpWAPy7vr2rObU1wFUJIT3XJ5/Gq5vJpSRJiMLk5Jxn0Ap8zZnYeshSUskjBie3+Sa2rLWZ7Z8yM2c8ev65rAaQsudzAew4/oKbvA+XnH+fQYq4VGtglFPc9as/ElvjPlszE9WfJ/GurtdajLkS/LkDvxzXz7BqHkTB0/x4rqNOuNS1Fxb2sbZkPPHbHAr1sNipy0POxFKMdT03UfEtrCTAP3jP8uBjj61zuleH9R8S3iiKErEWwWxzzXp/gv4UO6C81TguQ3PX8q9/wBN0XT9KiEVpEFx7V7FHBTqe9U0R5VTEqOkDE8I+F7fw9YLEq/P3Peuxoor2IxUVZHA227sKKKKoQUUUUAf/9D9UqKKKAPnn9oj4Uw/EjwkWt0zfWJ8yL1bHVfxr8mfFVje6JEbG8iZJ4iTgg5O1gv5cgV+9bAMpU96+Zviv8APD/j2GSaNBbXjDAdR755/GvLzDA+1anHdHpYHGezThLY/JqCYSWKo3zTI5Vj6gjgfgazDetJqCWrnd5hYpn+HCZBz+le1fEL4E+LPhvGXmjNzaqeJUGc5Bzmvne5tb4TRXKIwMZGTj2wa8T2Eoyakj2Y1YyinFndQ322KJD/BuBHXryPzxTGuLiRhbt8zOQ6joc+lc3pmpKE/0kFXQ4II64/zmrEVys80ckb5aMlg305IP9K53Bpu6Nk9NDsrfU4oZWKEKdpYAZPAPTHr7VrRahbupud2Y2IBZSCFz1yOo59RXmsupGNwMbQQuJPQj1/HvVoX4RpJG3bm5zjG4H1I61Dplcx3+54iAx3AsCCxIyPY017v7OC5KsoPJ749+K89fXWiKkDaoPKqSB+PX9K0INfNxlh5ZUfwhiGx9TU+yktWiudHVf2las48qUKT74B/Gte2myMRzEP1ySCvv24riIr63Zy0bKsigBlkOAR9eKb/AGtFHmGF1icHgbgQefUVEoN6IpSXU7WUynEhl74ABP8A7LU0eo2JRUuA+TwME446mvOl8RxxTNs+fP3ieuavf8JBp6hsAoXGQEUde+eafs5W1QcyPQBc2wLISVA5yeAR6ZbOaadRsY8fZ3B56bg3Hvj+Veet4p3bfP8AMeLjIJ5H0yTSW/iGzdtq26uOTlmIP5YIpOi+qFzpHof9rSA5MZfkngHgfhUT6tJCChhYAdCwOOffiuIutTec4hLc+j8D8gKgiuyx2zXGW6D5iRSVKw+c9FjmaUBt+09gDgAe+etWfnXAllOScfKw/lXn0WoR28nliVdx7gn+taX9oJtCySFj2OeB+FTKmxqR1StaqxKFsdATg9OtPaexOTuxtPb/AOt0rlXuFlVkTDE9BnrUAE2Vbyh0weDx9fSjl8xnapqD5Bi+YDv0/LPWpDdSTLtCdfVsGubs5TjMjjI471ea/YfK649Mc1MoAmaQWbhpdpK9+OB+FT+dOPmRxtPXoOn05rHa+i7ptI7H+dVzqUSy7GXgdSTxmlyeQXOnS6hT5SM5Hfn+dRTakqrtBA9Oa546jbsTFCp3MBznIGaoy3F1uIVPlwME9PzoVNBc6Zb63cYDfMcA5IAFSC+to3G9iM9cc4/IVgxXgMYLOmO6HOT9DiqrXwk3LHblj65/wFPkC51s98GTdG7H0HP+FRxX1xHh88n14P4HFc4t/LCuHmEeR0bqAPoahfWE3ZFyoJ4KhSf1o9n5Dudn/beonBLgYOPm6/1FSnWJh805Td6jGfxxXHQ6orPsYsxzjdWit3agExn5/TOefyqPZLsPmOmXXkBPJJ6DDZp8ut3ZAUlhkj+70/Ec1zpnkmwOwHHHf1qXzJIRtRGwR1PPP5CmqC7Cc/M1I9buoVMkJA2/3cH/AL6UH9RUkOuXskmFSN1b0wck+hIFcxJvkOZTGGx/HHz14w4pht3ucg7o89dihl/kDVfVm+hPt0up6BHfqoG9fLbGSudp49MjFaVlrthGAfOYHvgq2Pwry7yxb/u3LMB35x+I5qazaWR1jtI5N3YKN/P8/wA6f1GT2RLxUV1PZI/EUAO77a21uoKD8+DUp8T2A5EzSf7WzI/ka5XSvBvjzVF82y0maRGGQ5h2gn6k13OnfCD4l3i4a3MB9WwP5dvrVxy6p2/r7jCWNprdmTJ4hsZEMIbzR1yu0Y/A4rOkvPtGBZY3H1cKRj0r06P9njx+xBkmjIPrj8jxWsf2d/GyDe0ylem09PqK2WVVukTJ5hS7ngMguA2XkZGPZmyP0rJkYQvv8yMA8nLMT/8ArzXr+pfs8/EVZG8kRyIxzggdPpjBxXAX3wJ+Kcc5SLTiRnIZTjp70/7NrdUWsfSf2jn49StY5Pn4IPVGx/OurtdZteiStk9ScHFSx/Az4o3EqLDp+MgYD+/XkV7l4U/Zu8QPGr66UiY8sE5qo5VVl9kynj6a+0ePLd+fzHK2T6qDVi3t9Qdyke2XIz93JODX2FpnwH0q2UC5O84xXY6X8JtCsJA5UEDtiuuGR1Hucsszj0Pie20LXLqQi3tnOeflXH867HS/hZ4k1Jl86GRPqCK+6LPQdLsVCwQKMe1aqxxr91QK7qeQ0lrNnPPNqj+FHy1oHwRkQA3vy+tez6D8P9J0bayxgsOenevQaK9OjgqVL4UcNXE1KnxMRVCgKowBS0UV1nOFFFFABRRRQAUUUUAf/9H9UqKKKADg0wpn3oIpvzDpQBkar4e0vWrZ7PUYVmifqrDIry68+Avw5uIXh/s2Nd/cCvaPMI608MGqZQi90XGclsz8/wDxj+xZYanetNot2YImOdp7fSvFNe/Y58aaD582iSC7UrwOhz61+tuKjKVzywdJq1jojjaq1ufgb4g+Hvjrw3dtDeafLHtznKEj+VcM5urYtDfRtGDx3ytf0N3ejaXff8flrHLn+8oNeaeIPgV8MvEhZtQ0iIM3UoNtcs8rh9lnVHM39pH4QvdeTIyqdwHI3D+tU3vJGIKKAtfs9c/sffCOdy32Z1B7A9K5PUf2H/hrcvusppoAeozms1lzRr/aUD8ivtby/I+5x79q0YU81PLjBTnsOtfrRb/sSfDmJQr3EzevNb1v+x78OLZQkbSDHc8mn/Z7sH9oQPx9XTXceWqn8Rmta20O9YgQKxPptzX68W/7JHw6iffNJPIPTgV19j+z14E00AWlvnH98Amj+z5Pdk/2jHofj5D4e1GXbuteFAGSOv8AKmy+G5Oph2HPPHFftAfgv4ObBaziDAYyFAzVab4HeC5wQ9qv5VKyrXcf9prsfjbF4cuQCqoTnBPbn8qVvDl87hljIx6f/qr9hW+AHgY9LfGfSq8n7Png1j8qbah5U+jGszXY/ISfw7fMf9W3H1/woi0e+i5EbFvfp+NfruP2evBw6x5pp/Z68HdoRS/sp2sx/wBpo/Ia5tdQziCEqW5bv+RqktzqNtmNkbjGNwyB785r9err9nrww6lYYgPwrj7/APZg0e5B8squfaollTtoi45nHqfmPDeXdwC0uAx5yoH+TWtbi6kbLy8dgy5BP5ivvGb9kaFixjuAAfQVAP2Tnh4ScEVxzyytsonRHMaXVnxBKbhV3SDn24rlby/u0lO2F9g6hiMfgetfoev7LNx0eYEVK37Ktuy4lkBJ9qUMrrdYjlmVLoz8zpNQuXbZAjBE/PP14qzBrd+oXaMMp4LqOM/Wv0kX9k3TmPzMAK04P2SdDXhm49MV0/2bO1uUy/tGHc/M8yavMm8DoeNi4xSm71/zQ8wkHTlSVyB9MV+pUH7KnhmPAJOB2rbh/Zn8IwjBjz9aSyur2QnmVPuflVHqGoPhWWZzg8k9j9akt7C8mG8QM3OSST+FfrFD+zt4Lh5EA9627T4K+DrPG22Q49s01lE+rsS80h0R+V+n+Etfv2UWVg5z15NezeHfgx4wv1UpamLOM5NfpDpngXw9aYMUCjHTiuzt9Ms7dQIowMe1dVPKIfaZz1M0n9lHwhpX7OeuSov2namRzzXd2H7M1vsC3cufWvsIIo6CnV2wy+jHock8dVl1PlyD9mHwtjFw5I9hW/Zfs2fDy25lhaT8cV9CUV0KhTW0TB1pvdniEf7Pnw3jO4Wjk+7mu40r4deENGAFhp8SEd9oz+ddvRVeyh2Jc5dyrHZW0ShI4woHYCpfKjHapaTAqrIm435R0FOAFGBRimIQqh6gUbE9BUbA9qAGzSGSbFHQU6kHSlpiCiiigAooooAKKKKACiiigAooooAKKKKACiiigD//0v1SooooAKTFLRQAwqDTdvcVLRigCMFhT91LgUYFABkUdaTFGKAA7u1MLMO1P5paAGKxPan0UUAFFFFABRRRQAUUUUAFFFFABRRRQA3mkwafRQMZg1GYsnmp6KLBcgEIFOCYqWilYLkew00x5qaiiwXK/kg0eQvpViiiyC5EsSjoKlAxRRTEFGaKMUAJmlzSYNGDQMWikwaUUAFFFFAgooooATFGKWigdwooooEFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAH//T/VKiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooA//Z\"}]}\n"
  },
  {
    "path": "test/apis/realtime/multi-container/app/main.py",
    "content": "import base64\nimport requests\n\nfrom fastapi import FastAPI\nfrom pydantic import BaseModel\n\n\nclass Request(BaseModel):\n    image_url: str\n\n\napp = FastAPI()\napp.server_url = \"http://localhost:8501/v1/models/resnet50:predict\"\napp.labels = requests.get(\n    \"https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt\"\n).text.split(\"\\n\")[1:]\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    return \"ok\"\n\n\n@app.post(\"/\")\ndef text_generator(request: Request):\n    # download the image\n    dl_request = requests.get(request.image_url, stream=True)\n    dl_request.raise_for_status()\n\n    # compose a JSON Predict request (send JPEG image in base64).\n    jpeg_bytes = base64.b64encode(dl_request.content).decode(\"utf-8\")\n    predict_request = '{\"instances\" : [{\"b64\": \"%s\"}]}' % jpeg_bytes\n\n    # make prediction\n    response = requests.post(app.server_url, data=predict_request)\n    response.raise_for_status()\n    label_id = response.json()[\"predictions\"][0][\"classes\"]\n    return {\"image_prediction\": app.labels[label_id]}\n"
  },
  {
    "path": "test/apis/realtime/multi-container/app/requirements.txt",
    "content": "uvicorn[standard]\nfastapi\nrequests\n"
  },
  {
    "path": "test/apis/realtime/multi-container/build-tfs-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-multi-container-tfs-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/multi-container/build-web-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-multi-container-web-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/multi-container/cortex_cpu.yaml",
    "content": "- name: multi-container\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    containers:\n    - name: web-server\n      image: quay.io/cortexlabs-test/realtime-multi-container-web-cpu:latest\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n    - name: tfs-server\n      image: quay.io/cortexlabs-test/realtime-multi-container-tfs-cpu:latest\n      readiness_probe:\n        exec:\n          command: [\"tfs_model_status_probe\", \"-addr\", \"localhost:8500\", \"-model-name\", \"resnet50\"]\n      compute:\n        cpu: 1\n        mem: 2G\n"
  },
  {
    "path": "test/apis/realtime/multi-container/sample.json",
    "content": "{\n    \"image_url\": \"https://tensorflow.org/images/blogs/serving/cat.jpg\"\n}\n"
  },
  {
    "path": "test/apis/realtime/multi-container/tfs-cpu.Dockerfile",
    "content": "FROM tensorflow/serving:2.3.0\n\nRUN apt-get update -qq && apt-get install -y --no-install-recommends -q \\\n        wget \\\n    && apt-get clean -qq && rm -rf /var/lib/apt/lists/*\n\nRUN TFS_PROBE_VERSION=1.0.1 \\\n    && wget -qO /bin/tfs_model_status_probe https://github.com/codycollier/tfs-model-status-probe/releases/download/v${TFS_PROBE_VERSION}/tfs_model_status_probe_${TFS_PROBE_VERSION}_linux_amd64 \\\n    && chmod +x /bin/tfs_model_status_probe\n\nRUN mkdir -p /model/resnet50/ \\\n    && wget -qO- http://download.tensorflow.org/models/official/20181001_resnet/savedmodels/resnet_v2_fp32_savedmodel_NHWC_jpg.tar.gz | \\\n    tar --strip-components=2 -C /model/resnet50 -xvz\n\nENTRYPOINT tensorflow_model_server --rest_api_port=8501 --rest_api_num_threads=8 --model_name=\"resnet50\" --model_base_path=\"/model/resnet50\"\n"
  },
  {
    "path": "test/apis/realtime/multi-container/web-cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/realtime/prime-generator/app/main.py",
    "content": "from typing import DefaultDict\nfrom fastapi import FastAPI\nfrom pydantic import BaseModel\n\napp = FastAPI()\n\n\ndef generate_primes(limit=None):\n    \"\"\"Sieve of Eratosthenes\"\"\"\n    not_prime = DefaultDict(list)\n    num = 2\n    while limit is None or num <= limit:\n        if num in not_prime:\n            for prime in not_prime[num]:\n                not_prime[prime + num].append(prime)\n            del not_prime[num]\n        else:\n            yield num\n            not_prime[num * num] = [num]\n        num += 1\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    return \"ok\"\n\n\nclass Body(BaseModel):\n    primes_to_generate: float\n\n\n@app.post(\"/\")\ndef prime_numbers(body: Body):\n    return {\"prime_numbers\": list(generate_primes(body.primes_to_generate))}\n"
  },
  {
    "path": "test/apis/realtime/prime-generator/app/requirements.txt",
    "content": "uvicorn[standard]\nfastapi\n"
  },
  {
    "path": "test/apis/realtime/prime-generator/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-prime-generator-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/prime-generator/cortex_cpu.yaml",
    "content": "- name: prime-generator\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-prime-generator-cpu:latest\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 128Mi\n"
  },
  {
    "path": "test/apis/realtime/prime-generator/cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/realtime/prime-generator/sample.json",
    "content": "{\n    \"primes_to_generate\": 100\n}\n"
  },
  {
    "path": "test/apis/realtime/sleep/app/main.py",
    "content": "import time\n\nfrom fastapi import FastAPI\nfrom fastapi.responses import PlainTextResponse\n\napp = FastAPI()\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    return PlainTextResponse(\"ok\")\n\n\n@app.post(\"/\")\ndef sleep(sleep: float = 0):\n    time.sleep(sleep)\n    return PlainTextResponse(\"ok\")\n"
  },
  {
    "path": "test/apis/realtime/sleep/app/requirements.txt",
    "content": "uvicorn[standard]\nfastapi\n"
  },
  {
    "path": "test/apis/realtime/sleep/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-sleep-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/sleep/cortex_cpu.yaml",
    "content": "- name: sleep\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    max_queue_length: 128\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-sleep-cpu:latest\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 128Mi\n  autoscaling:\n    target_in_flight: 1\n"
  },
  {
    "path": "test/apis/realtime/sleep/cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/realtime/text-generator/app/main.py",
    "content": "import os\n\nfrom fastapi import FastAPI, status\nfrom fastapi.responses import PlainTextResponse\nfrom pydantic import BaseModel\nfrom transformers import GPT2Tokenizer, GPT2LMHeadModel\n\napp = FastAPI()\napp.device = os.getenv(\"TARGET_DEVICE\", \"cpu\")\napp.ready = False\n\n\n@app.on_event(\"startup\")\ndef startup():\n    app.tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n    app.model = GPT2LMHeadModel.from_pretrained(\"gpt2\").to(app.device)\n    app.ready = True\n\n\n@app.get(\"/healthz\")\ndef healthz():\n    if app.ready:\n        return PlainTextResponse(\"ok\")\n    return PlainTextResponse(\"service unavailable\", status_code=status.HTTP_503_SERVICE_UNAVAILABLE)\n\n\nclass Body(BaseModel):\n    text: str\n\n\n@app.post(\"/\")\ndef text_generator(body: Body):\n    input_length = len(body.text.split())\n    tokens = app.tokenizer.encode(body.text, return_tensors=\"pt\").to(app.device)\n    prediction = app.model.generate(tokens, max_length=input_length + 20, do_sample=True)\n    return {\"text\": app.tokenizer.decode(prediction[0])}\n"
  },
  {
    "path": "test/apis/realtime/text-generator/app/requirements-cpu.txt",
    "content": "uvicorn[standard]\nfastapi\ntransformers==3.0.*\n-f https://download.pytorch.org/whl/torch_stable.html\ntorch==1.7.1+cpu\n"
  },
  {
    "path": "test/apis/realtime/text-generator/app/requirements-gpu.txt",
    "content": "uvicorn[standard]==0.16.0\nsentencepiece==0.1.94\nfastapi\ntransformers==3.0.*\ntorch==1.10.2\n"
  },
  {
    "path": "test/apis/realtime/text-generator/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-text-generator-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/text-generator/build-gpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-gpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"realtime-text-generator-gpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/realtime/text-generator/cortex_cpu.yaml",
    "content": "- name: text-generator\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-text-generator-cpu:latest\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 1\n        mem: 2.5Gi\n"
  },
  {
    "path": "test/apis/realtime/text-generator/cortex_gpu.yaml",
    "content": "- name: text-generator\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-text-generator-gpu:latest\n      env:\n        TARGET_DEVICE: \"cuda\"\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 1\n        gpu: 1\n        mem: 512Mi\n"
  },
  {
    "path": "test/apis/realtime/text-generator/cpu.Dockerfile",
    "content": "FROM python:3.8-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements-cpu.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/realtime/text-generator/gpu.Dockerfile",
    "content": "FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu18.04\n\nRUN apt-get update \\\n    && apt-get install -y \\\n        python3 \\\n        python3-pip \\\n        pkg-config \\\n        build-essential \\\n        git \\\n        cmake \\\n    && apt-get clean -qq && rm -rf /var/lib/apt/lists/*\n\nENV LC_ALL=C.UTF-8 LANG=C.UTF-8\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements-gpu.txt /app/requirements.txt\nRUN pip3 install \\\n    --no-cache-dir \\\n    --extra-index-url https://download.pytorch.org/whl/cu113 \\\n    -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\nENV PYTHONPATH=/app\n\nENV CORTEX_PORT=8080\nCMD uvicorn --workers 1 --host 0.0.0.0 --port $CORTEX_PORT main:app\n"
  },
  {
    "path": "test/apis/realtime/text-generator/sample.json",
    "content": "{\n    \"text\": \"machine learning is\"\n}\n"
  },
  {
    "path": "test/apis/task/iris-classifier-trainer/app/main.py",
    "content": "import json\nimport pickle\nimport re\nimport os\nimport boto3\n\nfrom sklearn.datasets import load_iris\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\n\n\ndef main():\n    with open(\"/cortex/spec/job.json\", \"r\") as f:\n        job_spec = json.load(f)\n    print(json.dumps(job_spec, indent=2))\n\n    # get metadata\n    config = job_spec[\"config\"]\n    job_id = job_spec[\"job_id\"]\n    s3_path = None\n    if config is not None and \"dest_s3_dir\" in config:\n        s3_path = config[\"dest_s3_dir\"]\n\n    # Train the model\n    iris = load_iris()\n    data, labels = iris.data, iris.target\n    training_data, test_data, training_labels, test_labels = train_test_split(data, labels)\n\n    model = LogisticRegression(solver=\"lbfgs\", multi_class=\"multinomial\", max_iter=1000)\n    model.fit(training_data, training_labels)\n    accuracy = model.score(test_data, test_labels)\n    print(\"accuracy: {:.2f}\".format(accuracy))\n\n    # Upload the model\n    if s3_path:\n        pickle.dump(model, open(\"model.pkl\", \"wb\"))\n        bucket, key = re.match(\"s3://(.+?)/(.+)\", s3_path).groups()\n        s3 = boto3.client(\"s3\")\n        s3.upload_file(\"model.pkl\", bucket, os.path.join(key, job_id, \"model.pkl\"))\n    else:\n        print(\"not uploading the model to the s3 bucket\")\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "test/apis/task/iris-classifier-trainer/app/requirements.txt",
    "content": "numpy==1.18.*\nscikit-learn==0.21.*\nboto3==1.17.*\n"
  },
  {
    "path": "test/apis/task/iris-classifier-trainer/build-cpu.sh",
    "content": "#!/usr/bin/env bash\n\n# usage: build-cpu.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nimage_name=\"task-iris-classifier-trainer-cpu\"\n\n\"$(dirname \"${BASH_SOURCE[0]}\")\"/../../../utils/build.sh $(realpath \"${BASH_SOURCE[0]}\") \"$image_name\" \"$@\"\n"
  },
  {
    "path": "test/apis/task/iris-classifier-trainer/cortex_cpu.yaml",
    "content": "- name: iris-classifier-trainer\n  kind: TaskAPI\n  pod:\n    containers:\n    - name: trainer\n      image: quay.io/cortexlabs-test/task-iris-classifier-trainer-cpu:latest\n      command:\n        - python\n        - main.py\n      compute:\n        cpu: 200m\n        mem: 256Mi\n"
  },
  {
    "path": "test/apis/task/iris-classifier-trainer/cpu.Dockerfile",
    "content": "FROM python:3.7-slim\n\nENV PYTHONUNBUFFERED TRUE\n\nCOPY app/requirements.txt /app/requirements.txt\nRUN pip install --no-cache-dir -r /app/requirements.txt\n\nCOPY app /app\nWORKDIR /app/\n\nCMD exec python app/main.py\n"
  },
  {
    "path": "test/apis/task/iris-classifier-trainer/submit.py",
    "content": "\"\"\"\nTypical usage example:\n\n    python submit.py <cortex-env> <dest-s3-dir>\n\"\"\"\n\nimport sys\nimport json\nimport requests\nimport cortex\n\n\ndef main():\n    # parse args\n    if len(sys.argv) != 3:\n        print(\"usage: python submit.py <cortex-env> <dest-s3-dir>\")\n        sys.exit(1)\n    env_name = sys.argv[1]\n    dest_s3_dir = sys.argv[2]\n\n    # get task endpoint\n    cx = cortex.client(env_name)\n    task_endpoint = cx.get_api(\"iris-classifier-trainer\")[\"endpoint\"]\n\n    # submit job\n    job_spec = {\"config\": {\"dest_s3_dir\": dest_s3_dir}}\n    response = requests.post(task_endpoint, json=job_spec)\n    print(json.dumps(response.json(), indent=2))\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "test/apis/trafficsplitter/hello-world/.dockerignore",
    "content": "*.dockerfile\nREADME.md\nsample.json\n*.pyc\n*.pyo\n*.pyd\n__pycache__\n.pytest_cache\n"
  },
  {
    "path": "test/apis/trafficsplitter/hello-world/cortex_cpu.yaml",
    "content": "- name: hello-world-a\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest\n      env:\n        RESPONSE: \"hello from API A\"\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 128Mi\n\n- name: hello-world-b\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest\n      env:\n        RESPONSE: \"hello from API B\"\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 128Mi\n\n- name: hello-world-shadow\n  kind: RealtimeAPI\n  pod:\n    port: 8080\n    max_concurrency: 1\n    containers:\n    - name: api\n      image: quay.io/cortexlabs-test/realtime-hello-world-cpu:latest\n      env:\n        RESPONSE: \"hello from shadow API\"\n      readiness_probe:\n        http_get:\n          path: \"/healthz\"\n          port: 8080\n      compute:\n        cpu: 200m\n        mem: 128Mi\n\n- name: hello-world\n  kind: TrafficSplitter\n  apis:\n    - name: hello-world-a\n      weight: 30\n    - name: hello-world-b\n      weight: 70\n    - name: hello-world-shadow\n      shadow: true\n      weight: 100\n"
  },
  {
    "path": "test/apis/trafficsplitter/hello-world/sample.json",
    "content": "{}\n"
  },
  {
    "path": "test/e2e/README.md",
    "content": "# End-to-end Tests\n\n## Dependencies\n\nInstall the `e2e` package, from the project directory:\n\n```shell\npip install -e test/e2e\n```\n\nThis only needs to be installed once (not on every code change).\n\n_note: you may need to run `pip3 uninstall cortex` and `pip3 install -e python/client/` before the command above_\n\n## Running the tests\n\nBefore running tests, instruct the Python client to use your development CLI binary:\n\n```shell\nexport CORTEX_CLI_PATH=<cortex_repo_path>/bin/cortex\n```\n\nFrom an existing cluster:\n\n```shell\npytest test/e2e/tests --env <env_name>\n```\n\nUsing a new cluster, created for testing only and deleted afterwards:\n\n```shell\npytest test/e2e/tests --config <cluster.yaml>\n```\n\n**Note:** For the BatchAPI tests, the `--s3-path` option should be provided with an S3 bucket for testing purposes.\nIt is more convenient however to define this bucket through an environment variable, see [configuration](#configuration)\n.\n\n### Skip GPU Tests\n\nIt is possible to skip GPU tests by passing the `--skip-gpus` flag to the pytest command.\n\n### Skip Inferentia Tests\n\nIt is possible to skip Inferentia tests by passing the `--skip-infs` flag to the pytest command.\n\n### Skip Autoscaling Test\n\nIt is possible to skip the autoscaling test by passing the `--skip-autoscaling` flag to the pytest command.\n\n### Skip Load Test\n\nIt is possible to skip the load tests by passing the `--skip-load` flag to the pytest command.\n\n### Skip Long Running Test\n\nIt is possible to skip the long running test by passing the `--skip-long-running` flag to the pytest command.\n\n## Configuration\n\nIt is possible to configure the behaviour of the tests by defining environment variables or a `.env` file at the project\ndirectory.\n\n```dotenv\n# .env file\nCORTEX_TEST_REALTIME_DEPLOY_TIMEOUT=120\nCORTEX_TEST_BATCH_DEPLOY_TIMEOUT=60\nCORTEX_TEST_BATCH_JOB_TIMEOUT=120\nCORTEX_TEST_BATCH_S3_PATH=s3://<s3_bucket>/test/jobs\n```\n"
  },
  {
    "path": "test/e2e/e2e/__init__.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom .cluster import create_cluster, delete_cluster\n\n__all__ = [\"create_cluster\", \"delete_cluster\"]\n"
  },
  {
    "path": "test/e2e/e2e/cluster.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport subprocess\nimport sys\n\nimport yaml\n\nfrom e2e.exceptions import ClusterCreationException, ClusterDeletionException\n\n\ndef create_cluster(cluster_config: str):\n    \"\"\"Create a cortex cluster from a cluster config\"\"\"\n    with open(cluster_config) as f:\n        config = yaml.safe_load(f)\n\n    p = subprocess.run(\n        [\n            \"cortex\",\n            \"cluster\",\n            \"up\",\n            cluster_config,\n            \"-y\",\n            \"--configure-env\",\n            config[\"cluster_name\"],\n        ],\n        stdout=sys.stdout,\n        stderr=sys.stderr,\n    )\n\n    if p.returncode != 0:\n        raise ClusterCreationException(f\"failed to create cluster with config: {cluster_config}\")\n\n\ndef delete_cluster(cluster_config: str):\n    \"\"\"Delete a cortex cluster from a cluster config\"\"\"\n    with open(cluster_config) as f:\n        config = yaml.safe_load(f)\n\n    p = subprocess.run(\n        [\"cortex\", \"cluster\", \"down\", \"-y\", \"--config\", cluster_config],\n        stdout=sys.stdout,\n        stderr=sys.stderr,\n    )\n\n    if p.returncode != 0:\n        raise ClusterDeletionException(f\"failed to delete cluster with config: {cluster_config}\")\n"
  },
  {
    "path": "test/e2e/e2e/exceptions.py",
    "content": "class ClusterCreationException(Exception):\n    pass\n\n\nclass ClusterDeletionException(Exception):\n    pass\n\n\nclass ExpectationsValidationException(Exception):\n    pass\n\n\nclass GeneratorValidationException(Exception):\n    pass\n"
  },
  {
    "path": "test/e2e/e2e/expectations.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport pathlib\nimport types\nfrom typing import Dict, Any\n\nimport jsonschema\nimport requests\nimport yaml\nfrom jsonschema import Draft7Validator\n\nfrom e2e.exceptions import ExpectationsValidationException\n\nCONTENT_TO_ATTR = {\"text\": \"text\", \"json\": \"json\", \"binary\": \"content\"}\n\n\ndef assert_response_expectations(response: requests.Response, expectations: Dict[str, Any]):\n    content_type = expectations[\"content_type\"]\n\n    expected = expectations.get(\"expected\")\n    if expected:\n        output = _get_response_content(response, content_type)\n        assert output == expected, f\"unexpected response: got {output}, expected {expected}\"\n\n    expected_json_schema = expectations.get(\"json_schema\")\n    if expected_json_schema:\n        output = _get_response_content(response, content_type)\n        jsonschema.validate(output, schema=expected_json_schema)\n\n\ndef assert_json_expectations(response_json: Dict[str, Any], expectations: Dict[str, Any]):\n    expected_json_schema = expectations.get(\"json_schema\")\n    if expected_json_schema:\n        jsonschema.validate(response_json, schema=expected_json_schema)\n\n\ndef parse_expectations(expectations_file: str) -> Dict[str, Any]:\n    with open(expectations_file) as f:\n        expectations = yaml.safe_load(f)\n\n    validate_expectations(expectations)\n\n    return expectations\n\n\ndef validate_expectations(expectations):\n    if \"response\" in expectations:\n        validate_response_expectations(expectations[\"response\"])\n\n\ndef validate_response_expectations(expectations: Dict[str, Any]):\n    if not expectations[\"content_type\"] in CONTENT_TO_ATTR.keys():\n        raise ExpectationsValidationException(\n            f\"response.content_type should be one of {CONTENT_TO_ATTR.keys()}\"\n        )\n\n    if \"expected\" in expectations and \"json_schema\" in expectations:\n        raise ExpectationsValidationException(\"expected and json_schema are mutually exclusive\")\n\n    if \"json_schema\" in expectations:\n        if expectations[\"content_type\"] != \"json\":\n            raise ExpectationsValidationException(\n                \"json_schema is only valid when content_type is set to json\"\n            )\n\n        try:\n            Draft7Validator.check_schema(schema=expectations[\"json_schema\"])\n        except Exception as e:\n            raise ExpectationsValidationException(\"json_schema is invalid\") from e\n\n    if \"grpc\" in expectations:\n        grpc = expectations[\"grpc\"]\n        required_fields = [\n            \"proto_module_pb2\",\n            \"proto_module_pb2_grpc\",\n            \"stub_service_name\",\n            \"input_spec\",\n            \"output_spec\",\n        ]\n        for required_field in required_fields:\n            if required_field not in grpc:\n                raise ExpectationsValidationException(f\"missing grpc.{required_field} field\")\n\n        p1 = str(pathlib.Path(grpc[\"proto_module_pb2\"]).parent)\n        p2 = str(pathlib.Path(grpc[\"proto_module_pb2_grpc\"]).parent)\n        if p1 != p2:\n            raise ExpectationsValidationException(\n                \"the parent directories of proto_module_pb2 and proto_module_pb2_grpc don't match\"\n            )\n\n        input_spec = grpc[\"input_spec\"]\n        if \"class_name\" not in input_spec:\n            raise ExpectationsValidationException(\"missing grpc.input_spec.class_name field\")\n        if \"input\" not in input_spec:\n            raise ExpectationsValidationException(\"missing grpc.input_spec.input field\")\n\n        output_spec = grpc[\"output_spec\"]\n        if \"class_name\" not in output_spec:\n            raise ExpectationsValidationException(\"missing grpc.output_spec.class_name field\")\n        if \"stream\" not in output_spec:\n            raise ExpectationsValidationException(\"missing grpc.output_spec.stream field\")\n\n\ndef _get_response_content(response: requests.Response, content_type: str) -> str:\n    attr = CONTENT_TO_ATTR.get(content_type, \"content\")\n    content = getattr(response, attr)\n    if isinstance(content, types.MethodType):\n        return content()\n\n    return content\n"
  },
  {
    "path": "test/e2e/e2e/generator.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport importlib\nimport pathlib\nfrom typing import Any, Callable, List\nimport sys\nimport inspect\n\nfrom e2e.exceptions import GeneratorValidationException\n\n\ndef load_generator(sample_generator: pathlib.Path) -> Callable[[], List[int]]:\n    api_dir = str(sample_generator.parent)\n\n    sys.path.append(api_dir)\n    sample_generator_module = importlib.import_module(str(pathlib.Path(sample_generator).stem))\n    sys.path.pop()\n\n    validate_module(sample_generator_module)\n\n    return sample_generator_module.generate_sample\n\n\ndef validate_module(sample_generator_module: Any):\n    if not hasattr(sample_generator_module, \"generate_sample\"):\n        raise GeneratorValidationException(\n            \"sample generator module doesn't have a function called 'generate_sample'\"\n        )\n\n    if not inspect.isfunction(getattr(sample_generator_module, \"generate_sample\")):\n        raise GeneratorValidationException(\"'generate_sample' is not a function\")\n\n    if inspect.getfullargspec(getattr(sample_generator_module, \"generate_sample\")).args != []:\n        raise GeneratorValidationException(\n            \"'generate_sample' function must not have any parameters\"\n        )\n"
  },
  {
    "path": "test/e2e/e2e/tests.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport json\nimport math\nimport os\nimport re\nimport threading as td\nimport time\nfrom http import HTTPStatus\nfrom pathlib import Path\nfrom typing import Callable, Dict, Any, List, Union\n\nimport boto3\nimport cortex as cx\nimport requests\nimport yaml\n\nfrom e2e.expectations import (\n    parse_expectations,\n    assert_response_expectations,\n    assert_json_expectations,\n)\nfrom e2e.generator import load_generator\nfrom e2e.utils import (\n    apis_ready,\n    api_updated,\n    wait_on_event,\n    wait_on_futures,\n    endpoint_ready,\n    post_request,\n    get_request,\n    job_done,\n    jobs_done,\n    request_batch_prediction,\n    request_task,\n    retrieve_async_result,\n    make_requests_concurrently,\n    check_futures_healthy,\n    retrieve_results_concurrently,\n    stream_api_logs,\n    stream_job_logs,\n    wait_for,\n)\n\nTEST_APIS_DIR = Path(__file__).parent.parent.parent / \"apis\"\n\n\ndef delete_apis(client: cx.Client, api_names: List[str]):\n    for name in api_names:\n        client.delete(name)\n\n\ndef test_realtime_api(\n    printer: Callable,\n    client: cx.Client,\n    api: str,\n    timeout: int = None,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n    extra_path: str = \"\",\n    method: str = \"POST\",\n):\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n    assert len(api_specs) == 1\n\n    if len(node_groups) > 0:\n        api_specs[0][\"node_groups\"] = node_groups\n\n    expectations = None\n    expectations_file = api_dir / \"expectations.yaml\"\n    if expectations_file.exists():\n        expectations = parse_expectations(str(expectations_file))\n\n    api_name = api_specs[0][\"name\"]\n    for api_spec in api_specs:\n        client.deploy(api_spec=api_spec)\n\n    try:\n        assert apis_ready(\n            client=client, api_names=[api_name], timeout=timeout\n        ), f\"apis {api_name} not ready\"\n\n        with open(str(api_dir / \"sample.json\")) as f:\n            payload = json.load(f)\n        if method == \"POST\":\n            response = post_request(client, api_name, payload, extra_path)\n        else:\n            response = get_request(client, api_name, payload, extra_path)\n\n        assert (\n            response.status_code == HTTPStatus.OK\n        ), f\"status code: got {response.status_code}, expected {HTTPStatus.OK}\"\n\n        if expectations and \"response\" in expectations:\n            assert_response_expectations(response, expectations[\"response\"])\n    except:\n        # best effort\n        try:\n            api_info = client.get_api(api_name)\n            printer(json.dumps(api_info, indent=2))\n            td.Thread(target=lambda: stream_api_logs(client, api_name), daemon=True).start()\n            time.sleep(5)\n        finally:\n            raise\n    finally:\n        delete_apis(client, [api_name])\n\n\ndef test_batch_api(\n    printer: Callable,\n    client: cx.Client,\n    api: str,\n    test_s3_path: str,\n    deploy_timeout: int = None,\n    job_timeout: int = None,\n    retry_attempts: int = 0,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n    local_operator: bool = False,\n):\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n    assert len(api_specs) == 1\n\n    if len(node_groups) > 0:\n        api_specs[0][\"node_groups\"] = node_groups\n\n    api_name = api_specs[0][\"name\"]\n    client.deploy(api_spec=api_specs[0])\n\n    try:\n        endpoint_override = f\"http://localhost:8888/batch/{api_name}\" if local_operator else None\n        assert endpoint_ready(\n            client=client,\n            api_name=api_name,\n            timeout=deploy_timeout,\n            endpoint_override=endpoint_override,\n        ), f\"api {api_name} not ready\"\n\n        with open(str(api_dir / \"sample.json\")) as f:\n            payload = json.load(f)\n\n        response = None\n        for _ in range(retry_attempts + 1):\n            response = request_batch_prediction(\n                client,\n                api_name,\n                item_list=payload,\n                batch_size=2,\n                config={\"dest_s3_dir\": test_s3_path},\n                local_operator=local_operator,\n            )\n            if response.status_code == HTTPStatus.OK:\n                break\n\n            time.sleep(1)\n\n        assert (\n            response.status_code == HTTPStatus.OK\n        ), f\"status code: got {response.status_code}, expected {HTTPStatus.OK} ({response.text})\"\n\n        job_spec = response.json()\n\n        # monitor job progress\n        job_id = job_spec[\"job_id\"]\n        endpoint_override = (\n            f\"http://localhost:8888/batch/{api_name}?jobID={job_id}\" if local_operator else None\n        )\n        assert job_done(\n            client=client,\n            api_name=api_name,\n            job_id=job_id,\n            timeout=job_timeout,\n            endpoint_override=endpoint_override,\n        ), f\"job did not succeed (api_name: {api_name}, job_id: {job_spec['job_id']})\"\n\n    except:\n        # best effort\n        try:\n            api_info = client.get_api(api_name)\n            printer(json.dumps(api_info, indent=2))\n\n            job_status = client.get_job(api_name, job_spec[\"job_id\"])\n            printer(json.dumps(job_status, indent=2))\n\n            td.Thread(\n                target=lambda: stream_job_logs(client, api_name, job_spec[\"job_id\"]),\n                daemon=True,\n            ).start()\n            time.sleep(5)\n        finally:\n            raise\n    finally:\n        delete_apis(client, [api_name])\n\n\ndef test_async_api(\n    printer: Callable,\n    client: cx.Client,\n    api: str,\n    deploy_timeout: int = None,\n    poll_retries: int = 5,\n    poll_sleep_seconds: int = 1,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n):\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n    assert len(api_specs) == 1\n\n    if len(node_groups) > 0:\n        api_specs[0][\"node_groups\"] = node_groups\n\n    expectations = None\n    expectations_file = api_dir / \"expectations.yaml\"\n    if expectations_file.exists():\n        expectations = parse_expectations(str(expectations_file))\n\n    api_name = api_specs[0][\"name\"]\n    client.deploy(api_spec=api_specs[0])\n\n    try:\n        assert apis_ready(\n            client=client, api_names=[api_name], timeout=deploy_timeout\n        ), f\"apis {api_name} not ready\"\n\n        with open(str(api_dir / \"sample.json\")) as f:\n            payload = json.load(f)\n\n        response = post_request(client, api_name, payload)\n\n        assert (\n            response.status_code == HTTPStatus.OK\n        ), f\"workload submission status code: got {response.status_code}, expected {HTTPStatus.OK}\"\n\n        response_json = response.json()\n        assert \"id\" in response_json\n\n        request_id = response_json[\"id\"]\n\n        result_response = None\n        for _ in range(poll_retries + 1):\n            result_response = retrieve_async_result(\n                client=client, api_name=api_name, request_id=request_id\n            )\n\n            if result_response.status_code != HTTPStatus.OK:\n                time.sleep(poll_sleep_seconds)\n                continue\n\n            result_response_json = result_response.json()\n            assert (\n                \"id\" in result_response_json\n            ), f\"id key was not present in result response (response: {result_response_json})\"\n            assert (\n                \"status\" in result_response_json\n            ), f\"status key was not present in result response (response: {result_response_json})\"\n\n            if result_response_json[\"status\"] != \"completed\":\n                time.sleep(poll_sleep_seconds)\n                continue\n            break\n\n        assert (\n            result_response.status_code == HTTPStatus.OK\n        ), f\"result retrieval status code: got {result_response.status_code}, expected {HTTPStatus.OK}\"\n\n        # validate keys are in the result json response\n        assert (\n            \"result\" in result_response_json\n        ), f\"result key was not present in result response (response: {result_response_json})\"\n\n        assert (\n            \"timestamp\" in result_response_json\n        ), f\"timestamp key was not present in result response (response: {result_response_json})\"\n\n        # validate result json response has valid values\n        assert (\n            result_response_json[\"id\"] == request_id\n        ), f\"result 'id' and request 'id' mismatch ({result_response_json['id']} != {request_id})\"\n        assert (\n            result_response_json[\"status\"] == \"completed\"\n        ), f\"async workload did not complete (response: {result_response_json})\"\n        assert result_response_json[\"timestamp\"] != \"\", \"result 'timestamp' value was empty\"\n        assert result_response_json[\"result\"] != \"\", \"result 'result' value was empty\"\n\n        # assert result expectations\n        if expectations:\n            assert_json_expectations(result_response_json[\"result\"], expectations[\"response\"])\n\n    except:\n        # best effort\n        try:\n            api_info = client.get_api(api_name)\n            printer(json.dumps(api_info, indent=2))\n            printer(json.dumps(result_response_json, indent=2))\n            td.Thread(\n                target=lambda: stream_api_logs(client, api_name),\n                daemon=True,\n            ).start()\n            time.sleep(5)\n        except:\n            pass\n        raise\n\n    finally:\n        delete_apis(client, [api_name])\n\n\ndef test_task_api(\n    printer: Callable,\n    client: cx.Client,\n    api: str,\n    deploy_timeout: int = None,\n    job_timeout: int = None,\n    retry_attempts: int = 0,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n    local_operator: bool = False,\n):\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n\n    assert len(api_specs) == 1\n\n    if len(node_groups) > 0:\n        api_specs[0][\"node_groups\"] = node_groups\n\n    api_name = api_specs[0][\"name\"]\n    client.deploy(api_spec=api_specs[0])\n\n    try:\n        endpoint_override = f\"http://localhost:8888/tasks/{api_name}\" if local_operator else None\n        assert endpoint_ready(\n            client=client,\n            api_name=api_name,\n            timeout=deploy_timeout,\n            endpoint_override=endpoint_override,\n        ), f\"api {api_name} not ready\"\n\n        response = None\n        for _ in range(retry_attempts + 1):\n            response = request_task(client, api_name, local_operator=local_operator)\n            if response.status_code == HTTPStatus.OK:\n                break\n\n            time.sleep(1)\n\n        job_spec = response.json()\n\n        job_id = job_spec[\"job_id\"]\n        endpoint_override = (\n            f\"http://localhost:8888/tasks/{api_name}?jobID={job_id}\" if local_operator else None\n        )\n        assert job_done(\n            client=client,\n            api_name=api_name,\n            job_id=job_spec[\"job_id\"],\n            timeout=job_timeout,\n            endpoint_override=endpoint_override,\n        ), f\"task job did not succeed (api_name: {api_name}, job_id: {job_spec['job_id']})\"\n\n    except:\n        # best effort\n        try:\n            api_info = client.get_api(api_name)\n            printer(json.dumps(api_info, indent=2))\n\n            job_status = client.get_job(api_name, job_spec[\"job_id\"])\n            printer(json.dumps(job_status, indent=2))\n            td.Thread(\n                target=lambda: stream_job_logs(client, api_name, job_spec[\"job_id\"]), daemon=True\n            ).start()\n            time.sleep(5)\n        except:\n            pass\n        raise\n\n    finally:\n        delete_apis(client, [api_name])\n\n\ndef test_autoscaling(\n    printer: Callable,\n    client: cx.Client,\n    apis: Dict[str, Any],\n    autoscaling_config: Dict[str, Union[int, float]],\n    deploy_timeout: int = None,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n):\n    max_replicas = autoscaling_config[\"max_replicas\"]\n    query_params = apis[\"query_params\"]\n\n    # increase the concurrency by 1 to ensure we get max_replicas replicas\n    concurrency = max_replicas + 1\n\n    all_apis = [apis[\"primary\"]] + apis[\"dummy\"]\n    all_api_names = []\n    for api in all_apis:\n        api_dir = TEST_APIS_DIR / api\n        with open(str(api_dir / api_config_name)) as f:\n            api_specs = yaml.safe_load(f)\n        assert len(api_specs) == 1\n\n        if len(node_groups) > 0:\n            api_specs[0][\"node_groups\"] = node_groups\n        api_specs[0][\"autoscaling\"] = {\n            \"max_replicas\": max_replicas,\n            \"downscale_stabilization_period\": \"1m\",\n        }\n\n        all_api_names.append(api_specs[0][\"name\"])\n        client.deploy(api_spec=api_specs[0])\n\n    primary_api_name = all_api_names[0]\n    autoscaling = client.get_api(primary_api_name)[\"spec\"][\"autoscaling\"]\n\n    # controls the flow of requests\n    request_stopper = td.Event()\n\n    # determine upscale/downscale replica requests\n    current_replicas = 1  # starting number of replicas\n    test_timeout = 0  # measured in seconds\n    while current_replicas < max_replicas:\n        upscale_ceil = math.ceil(current_replicas * autoscaling[\"max_upscale_factor\"])\n        if upscale_ceil > current_replicas + 1:\n            current_replicas = upscale_ceil\n        else:\n            current_replicas += 1\n        if current_replicas > max_replicas:\n            current_replicas = max_replicas\n        test_timeout += int(autoscaling[\"upscale_stabilization_period\"] / (1000 ** 3))\n    while current_replicas > 1:\n        downscale_ceil = math.ceil(current_replicas * autoscaling[\"max_downscale_factor\"])\n        if downscale_ceil < current_replicas - 1:\n            current_replicas = downscale_ceil\n        else:\n            current_replicas -= 1\n        test_timeout += int(autoscaling[\"downscale_stabilization_period\"] / (1000 ** 3))\n\n    # add overhead to the test timeout to account for the process of downloading images or adding nodes to the cluster\n    test_timeout *= 2\n\n    try:\n        assert apis_ready(\n            client=client, api_names=all_api_names, timeout=deploy_timeout\n        ), f\"apis {all_api_names} not ready\"\n\n        threads_futures = make_requests_concurrently(\n            client, primary_api_name, concurrency, request_stopper, query_params=query_params\n        )\n\n        test_start_time = time.time()\n\n        # upscale/downscale the api\n        printer(f\"scaling up to {max_replicas} replicas\")\n        while True:\n            assert api_updated(\n                client, primary_api_name, timeout=deploy_timeout\n            ), \"api didn't scale up to the desired number of replicas in time\"\n            current_replicas = client.get_api(primary_api_name)[\"status\"][\"requested\"]\n\n            # stop the requests from being made\n            if current_replicas == max_replicas and not request_stopper.is_set():\n                printer(f\"scaling back down to 1 replica\")\n                request_stopper.set()\n\n            # check if the requesting threads are still healthy\n            # if not, they'll raise an exception\n            check_futures_healthy(threads_futures)\n\n            # check if the test is taking too much time\n            assert (\n                time.time() - test_start_time < test_timeout\n            ), f\"autoscaling test for api {primary_api_name} did not finish in {test_timeout}s; current number of replicas is {current_replicas}/{concurrency}\"\n\n            # stop the test if it has finished\n            if current_replicas == 1 and request_stopper.is_set():\n                break\n\n            # add some delay to reduce the number of gets\n            time.sleep(1)\n\n    except:\n        # best effort\n        try:\n            api_info = client.get_api(primary_api_name)\n            printer(json.dumps(api_info, indent=2))\n        finally:\n            raise\n    finally:\n        request_stopper.set()\n        delete_apis(client, all_api_names)\n\n\ndef test_load_realtime(\n    printer: Callable,\n    client: cx.Client,\n    api: str,\n    load_config: Dict[str, Union[int, float]],\n    deploy_timeout: int = None,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n):\n\n    total_requests = load_config[\"total_requests\"]\n    desired_replicas = load_config[\"desired_replicas\"]\n    concurrency = load_config[\"concurrency\"]\n    status_code_timeout = load_config[\"status_code_timeout\"]\n\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n    assert len(api_specs) == 1\n\n    api_specs[0][\"autoscaling\"] = {\n        \"min_replicas\": desired_replicas,\n        \"max_replicas\": desired_replicas,\n    }\n    if len(node_groups) > 0:\n        api_specs[0][\"node_groups\"] = node_groups\n\n    api_name = api_specs[0][\"name\"]\n    client.deploy(api_spec=api_specs[0])\n\n    # controls the flow of requests\n    request_stopper = td.Event()\n    failed = False\n    try:\n        printer(f\"getting {desired_replicas} replicas ready\")\n        assert apis_ready(\n            client=client, api_names=[api_name], timeout=deploy_timeout\n        ), f\"api {api_name} not ready\"\n\n        # give the APIs some time to prevent getting high latency spikes in the beginning\n        time.sleep(5)\n\n        with open(str(api_dir / \"sample.json\")) as f:\n            payload = json.load(f)\n\n        printer(\"start making requests concurrently\")\n        threads_futures = make_requests_concurrently(\n            client,\n            api_name,\n            concurrency,\n            request_stopper,\n            max_total_requests=total_requests,\n            payload=payload,\n        )\n\n        while not request_stopper.is_set():\n            # check if the requesting threads are still healthy\n            # if not, they'll raise an exception\n            check_futures_healthy(threads_futures)\n\n            # don't stress the CPU too hard\n            time.sleep(1)\n    except:\n        # best effort\n        failed = True\n        try:\n            api_info = client.get_api(api_name)\n            printer(json.dumps(api_info, indent=2))\n        finally:\n            raise\n\n    finally:\n        request_stopper.set()\n        delete_apis(client, [api_name])\n        if failed:\n            time.sleep(30)\n\n\ndef test_load_async(\n    printer: Callable,\n    client: cx.Client,\n    api: str,\n    load_config: Dict[str, Union[int, float]],\n    deploy_timeout: int = None,\n    poll_sleep_seconds: int = 1,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n):\n\n    total_requests = load_config[\"total_requests\"]\n    desired_replicas = load_config[\"desired_replicas\"]\n    concurrency = load_config[\"concurrency\"]\n    submit_timeout = load_config[\"submit_timeout\"]\n    workload_timeout = load_config[\"workload_timeout\"]\n\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n    assert len(api_specs) == 1\n\n    api_specs[0][\"autoscaling\"] = {\n        \"min_replicas\": desired_replicas,\n        \"max_replicas\": desired_replicas,\n    }\n    if len(node_groups) > 0:\n        api_specs[0][\"node_groups\"] = node_groups\n\n    api_name = api_specs[0][\"name\"]\n    client.deploy(api_spec=api_specs[0])\n\n    request_stopper = td.Event()\n    map_stopper = td.Event()\n    responses: List[Dict[str, Any]] = []\n    failed = False\n    try:\n        printer(f\"getting {desired_replicas} replicas ready\")\n        assert apis_ready(\n            client=client, api_names=[api_name], timeout=deploy_timeout\n        ), f\"api {api_name} not ready\"\n\n        with open(str(api_dir / \"sample.json\")) as f:\n            payload = json.load(f)\n\n        printer(\"start making prediction requests concurrently\")\n        threads_futures = make_requests_concurrently(\n            client,\n            api_name,\n            concurrency,\n            request_stopper,\n            responses=responses,\n            max_total_requests=total_requests,\n            payload=payload,\n        )\n        assert wait_on_event(\n            request_stopper, submit_timeout\n        ), f\"{total_requests} couldn't be submitted in {submit_timeout}s\"\n        check_futures_healthy(threads_futures)\n        wait_on_futures(threads_futures)\n\n        printer(\"finished making prediction requests\")\n        assert (\n            len(responses) == total_requests\n        ), f\"the submitted number of requests doesn't match the returned number of responses\"\n\n        job_ids = []\n        for response in responses:\n            response_json = response.json()\n            assert \"id\" in response_json\n            job_ids.append(response_json[\"id\"])\n\n        # assert the results\n        printer(\"start retrieving the async results concurrently\")\n        results = []\n        retrieve_results_concurrently(\n            client,\n            api_name,\n            concurrency,\n            map_stopper,\n            job_ids,\n            results,\n            poll_sleep_seconds,\n            workload_timeout,\n        )\n        for request_id, json_result in results:\n            # validate keys are in the result json response\n            assert (\n                \"id\" in json_result\n            ), f\"id key was not present in result response (response: {json_result})\"\n            assert (\n                \"result\" in json_result\n            ), f\"result key was not present in result response (response: {json_result})\"\n            assert (\n                \"timestamp\" in json_result\n            ), f\"timestamp key was not present in result response (response: {json_result})\"\n\n            # validate result json response has valid values\n            assert (\n                json_result[\"id\"] == request_id\n            ), f\"result 'id' and request 'id' mismatch ({json_result['id']} != {request_id})\"\n            assert (\n                json_result[\"status\"] == \"completed\"\n            ), f\"async workload did not complete (response: {json_result})\"\n            assert json_result[\"timestamp\"] != \"\", \"result 'timestamp' value was empty\"\n            assert json_result[\"result\"] != \"\", \"result 'result' value was empty\"\n\n    except:\n        # best effort\n        failed = True\n        try:\n            api_info = client.get_api(api_name)\n            printer(json.dumps(api_info, indent=2))\n        finally:\n            raise\n\n    finally:\n        if \"results\" in vars() and len(results) < total_requests:\n            printer(f\"{len(results)}/{total_requests} have been successfully retrieved\")\n        map_stopper.set()\n        delete_apis(client, [api_name])\n        if failed:\n            time.sleep(30)\n\n\ndef test_load_batch(\n    printer: Callable,\n    client: cx.Client,\n    api: str,\n    test_s3_path: str,\n    load_config: Dict[str, Union[int, float]],\n    deploy_timeout: int = None,\n    retry_attempts: int = 0,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n):\n\n    jobs = load_config[\"jobs\"]\n    workers_per_job = load_config[\"workers_per_job\"]\n    items_per_job = load_config[\"items_per_job\"]\n    batch_size = load_config[\"batch_size\"]\n    workload_timeout = load_config[\"workload_timeout\"]\n\n    bucket, key = re.match(\"s3://(.+?)/(.+)\", test_s3_path).groups()\n    s3 = boto3.client(\"s3\")\n    paginator = s3.get_paginator(\"list_objects_v2\")\n\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n    assert len(api_specs) == 1\n\n    if len(node_groups) > 0:\n        api_specs[0][\"node_groups\"] = node_groups\n\n    sample_generator_path = api_dir / \"sample_generator.py\"\n    assert (\n        sample_generator_path.exists()\n    ), \"sample_generator.py must be present for the batch load test\"\n    sample_generator = load_generator(sample_generator_path)\n\n    api_name = api_specs[0][\"name\"]\n    client.deploy(api_spec=api_specs[0])\n    api_endpoint = client.get_api(api_name)[\"endpoint\"]\n    failed = False\n    try:\n        assert endpoint_ready(\n            client=client, api_name=api_name, timeout=deploy_timeout\n        ), f\"api {api_name} not ready\"\n\n        # submit jobs\n        printer(f\"submitting {jobs} jobs\")\n        job_specs = []\n        for _ in range(jobs):\n            for _ in range(retry_attempts + 1):\n                response = request_batch_prediction(\n                    client,\n                    api_name,\n                    item_list=[sample_generator() for _ in range(items_per_job)],\n                    batch_size=batch_size,\n                    workers=workers_per_job,\n                    config={\"dest_s3_dir\": test_s3_path},\n                )\n                if response.status_code == HTTPStatus.OK:\n                    break\n                time.sleep(1)\n            # retries are only required once\n            retry_attempts = 0\n\n            assert (\n                response.status_code == HTTPStatus.OK\n            ), f\"status code: got {response.status_code}, expected {HTTPStatus.OK} ({response.text})\"\n            job_specs.append(response.json())\n\n        # wait the jobs to finish\n        printer(\"waiting on the jobs\")\n        assert jobs_done(\n            client, api_name, [job_spec[\"job_id\"] for job_spec in job_specs], workload_timeout\n        ), f\"not all jobs succeed in {workload_timeout}s\"\n\n        # assert jobs\n        printer(\"checking the jobs' responses\")\n        for job_spec in job_specs:\n            job_id: str = job_spec[\"job_id\"]\n            job = requests.get(f\"{api_endpoint}?jobID={job_id}\").json()\n            job_status = job[\"job_status\"]\n            job_metrics = job[\"metrics\"]\n\n            assert (\n                job_status[\"batches_in_queue\"] == 0\n            ), f\"there are still batches in queue ({job_status['batches_in_queue']}) for job ID {job_id}\"\n            assert job_metrics[\"succeeded\"] == math.ceil(items_per_job / batch_size)\n\n            num_objects = 0\n            for page in paginator.paginate(Bucket=bucket, Prefix=os.path.join(key, job_id)):\n                num_objects += len(page[\"Contents\"])\n            assert num_objects == 1\n\n    except:\n        # best effort\n        failed = True\n        try:\n            api_info = client.get_api(api_name)\n\n            # only get the last 10 job statuses\n            if \"batch_job_statuses\" in api_info and len(api_info[\"batch_job_statuses\"]) > 10:\n                api_info[\"batch_job_statuses\"] = api_info[\"batch_job_statuses\"][-10:]\n\n            printer(json.dumps(api_info, indent=2))\n        finally:\n            raise\n\n    finally:\n        delete_apis(client, [api_name])\n        if failed:\n            time.sleep(30)\n\n\ndef test_long_running_realtime(\n    printer: Callable,\n    client: cx.Client,\n    api: str,\n    long_running_config: Dict[str, Union[int, float]],\n    deploy_timeout: int = None,\n    api_config_name: str = \"cortex_cpu.yaml\",\n    node_groups: List[str] = [],\n):\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n    assert len(api_specs) == 1\n\n    time_to_run = long_running_config[\"time_to_run\"]\n\n    if len(node_groups) > 0:\n        api_specs[0][\"node_groups\"] = node_groups\n\n    expectations = None\n    expectations_file = api_dir / \"expectations.yaml\"\n    if expectations_file.exists():\n        expectations = parse_expectations(str(expectations_file))\n\n    api_name = api_specs[0][\"name\"]\n    for api_spec in api_specs:\n        client.deploy(api_spec=api_spec)\n\n    try:\n        assert apis_ready(\n            client=client, api_names=[api_name], timeout=deploy_timeout\n        ), f\"apis {api_name} not ready\"\n\n        with open(str(api_dir / \"sample.json\")) as f:\n            payload = json.load(f)\n\n        counter = 0\n        start_time = time.time()\n        while time.time() - start_time <= time_to_run:\n            response = post_request(client, api_name, payload)\n\n            assert (\n                response.status_code == HTTPStatus.OK\n            ), f\"status code: got {response.status_code}, expected {HTTPStatus.OK}\"\n\n            if expectations and \"response\" in expectations:\n                assert_response_expectations(response, expectations[\"response\"])\n\n            counter += 1\n\n    except:\n        # best effort\n        try:\n            api_info = client.get_api(api_name)\n            printer(json.dumps(api_info, indent=2))\n            td.Thread(target=lambda: stream_api_logs(client, api_name), daemon=True).start()\n            time.sleep(5)\n        finally:\n            raise\n    finally:\n        delete_apis(client, [api_name])\n\n\ndef test_realtime_scale_to_zero(\n    client: cx.Client,\n    api: str,\n    timeout: int = None,\n    api_config_name: str = \"cortex_cpu.yaml\",\n):\n    api_dir = TEST_APIS_DIR / api\n    with open(str(api_dir / api_config_name)) as f:\n        api_specs = yaml.safe_load(f)\n\n    api_name = api_specs[0][\"name\"]\n    for api_spec in api_specs:\n        client.deploy(api_spec=api_spec)\n\n    try:\n        assert apis_ready(\n            client=client, api_names=[api_name], timeout=timeout, greater_or_equal_to=0\n        ), f\"apis {api_name} not ready\"\n\n        api_info = client.get_api(api_name)\n        endpoint = api_info[\"endpoint\"]\n\n        response = requests.post(endpoint, json={}, timeout=30)\n\n        # make first request, which should go to activator\n        assert (\n            response.status_code == HTTPStatus.OK\n        ), f\"status code: got {response.status_code}, expected {HTTPStatus.OK}\"\n\n        assert response.headers.get(\"x-cortex-origin\") == \"activator\"\n\n        def _make_request() -> bool:\n            res = requests.post(endpoint, json={}, timeout=30)\n\n            # make second request, which should go directly to the api\n            assert (\n                res.status_code == HTTPStatus.OK\n            ), f\"status code: got {res.status_code}, expected {HTTPStatus.OK}\"\n\n            return res.headers.get(\"x-cortex-origin\") == \"api\"\n\n        assert wait_for(_make_request, timeout=60)\n    finally:\n        delete_apis(client, [api_name])\n"
  },
  {
    "path": "test/e2e/e2e/utils.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport threading as td\nimport time\nfrom concurrent import futures\nfrom http import HTTPStatus\nfrom typing import Any, List, Optional, Tuple, Union, Dict, Callable\n\nimport cortex as cx\nimport requests\nimport yaml\n\n\ndef wait_for(fn: Callable[[], bool], timeout=None) -> bool:\n    deadline = time.time() + timeout if timeout else None\n    while True:\n        if deadline is not None and time.time() > deadline:\n            return False\n\n        done = fn()\n        if done:\n            return True\n\n        time.sleep(1)\n\n\ndef apis_ready(\n    client: cx.Client,\n    api_names: List[str],\n    timeout: Optional[int] = None,\n    greater_or_equal_to: int = 1,\n) -> bool:\n    def _check_liveness(status):\n        return (\n            status[\"requested\"] >= greater_or_equal_to\n            and status[\"requested\"] == status[\"ready\"] == status[\"up_to_date\"]\n        )\n\n    def _is_ready():\n        return all([_check_liveness(client.get_api(name)[\"status\"]) for name in api_names])\n\n    return wait_for(_is_ready, timeout=timeout)\n\n\ndef api_updated(client: cx.Client, api_name: str, timeout: Optional[int] = None) -> bool:\n    def _is_ready():\n        status = client.get_api(api_name)[\"status\"]\n        return status[\"requested\"] == status[\"ready\"]\n\n    return wait_for(_is_ready, timeout=timeout)\n\n\ndef wait_on_event(event: td.Event, timeout: Optional[int] = None) -> bool:\n    def _is_ready():\n        return event.is_set()\n\n    return wait_for(_is_ready, timeout=timeout)\n\n\ndef wait_on_futures(futures_list: List[futures.Future], timeout: Optional[int] = None):\n    def _is_ready():\n        return all([future.done() for future in futures_list])\n\n    return wait_for(_is_ready, timeout=timeout)\n\n\ndef endpoint_ready(\n    client: cx.Client, api_name: str, timeout: int = None, endpoint_override: str = None\n) -> bool:\n    def _is_ready():\n        if endpoint_override:\n            endpoint = endpoint_override\n        else:\n            endpoint = client.get_api(api_name)[\"endpoint\"]\n\n        response = requests.post(endpoint)\n        return response.status_code == HTTPStatus.BAD_REQUEST\n\n    return wait_for(_is_ready, timeout=timeout)\n\n\ndef job_done(\n    client: cx.Client,\n    api_name: str,\n    job_id: str,\n    timeout: int = None,\n    endpoint_override: str = None,\n) -> bool:\n    def _is_ready():\n        if endpoint_override:\n            job_info = requests.get(endpoint_override)\n            job_info = job_info.json()\n            return job_info[\"job_status\"][\"status\"] == \"succeeded\"\n\n        job_info = client.get_job(api_name, job_id)\n        return job_info[\"job_status\"][\"status\"] == \"succeeded\"\n\n    return wait_for(_is_ready, timeout=timeout)\n\n\ndef jobs_done(client: cx.Client, api_name: str, job_ids: List[str], timeout: int = None) -> bool:\n    exec = futures.ThreadPoolExecutor(10)\n\n    def _runnable(job_id):\n        return job_done(client, api_name, job_id)\n\n    try:\n        for _ in exec.map(_runnable, job_ids, timeout=timeout):\n            pass\n    except:\n        return False\n\n    return True\n\n\ndef post_request(\n    client: cx.Client,\n    api_name: str,\n    payload: Union[List, Dict],\n    extra_path: Optional[str] = None,\n) -> requests.Response:\n    api_info = client.get_api(api_name)\n    endpoint = api_info[\"endpoint\"]\n    if extra_path and extra_path != \"\":\n        endpoint = os.path.join(endpoint, extra_path)\n    response = requests.post(endpoint, json=payload)\n\n    return response\n\n\ndef get_request(\n    client: cx.Client,\n    api_name: str,\n    payload: Union[List, Dict],\n    extra_path: Optional[str] = None,\n) -> requests.Response:\n    api_info = client.get_api(api_name)\n    endpoint = api_info[\"endpoint\"]\n    if extra_path and extra_path != \"\":\n        endpoint = os.path.join(endpoint, extra_path)\n    response = requests.get(endpoint, json=payload)\n\n    return response\n\n\ndef retrieve_async_result(client: cx.Client, api_name: str, request_id: str) -> requests.Response:\n    api_info = client.get_api(api_name)\n    response = requests.get(f\"{api_info['endpoint']}/{request_id}\")\n\n    return response\n\n\ndef request_batch_prediction(\n    client: cx.Client,\n    api_name: str,\n    item_list: List,\n    batch_size: int,\n    workers: int = 1,\n    config: Dict = None,\n    local_operator: bool = False,\n) -> requests.Response:\n\n    if local_operator:\n        endpoint = f\"http://localhost:8888/batch/{api_name}\"\n    else:\n        api_info = client.get_api(api_name)\n        endpoint = api_info[\"endpoint\"]\n\n    batch_payload = {\n        \"workers\": workers,\n        \"item_list\": {\"items\": item_list, \"batch_size\": batch_size},\n        \"config\": config,\n    }\n    response = requests.post(endpoint, json=batch_payload)\n\n    return response\n\n\ndef request_task(\n    client: cx.Client,\n    api_name: str,\n    config: Dict = None,\n    timeout: int = None,\n    local_operator: bool = False,\n):\n    if local_operator:\n        endpoint = f\"http://localhost:8888/tasks/{api_name}\"\n    else:\n        api_info = client.get_api(api_name)\n        endpoint = api_info[\"endpoint\"]\n\n    payload = {}\n    if config is not None:\n        payload[\"config\"] = config\n    if timeout is not None:\n        payload[\"timeout\"] = timeout\n\n    response = requests.post(endpoint, json=payload)\n    return response\n\n\n_make_requests_concurrently_max_total_requests: int = 0\n\n\ndef make_requests_concurrently(\n    client: cx.Client,\n    api_name: str,\n    concurrency: int,\n    event_stopper: td.Event,\n    latencies: Optional[List[float]] = None,\n    responses: Optional[List[Dict[str, Any]]] = None,\n    max_total_requests: Optional[int] = None,\n    payload: Optional[Union[List, Dict]] = None,\n    query_params: Dict[str, str] = {},\n) -> List[futures.Future]:\n\n    lock = td.RLock()\n    thread_local = td.local()\n    start_sync = td.Barrier(concurrency)\n    end_sync = td.Barrier(concurrency)\n    executor = futures.ThreadPoolExecutor(concurrency)\n    api_info = client.get_api(api_name)\n    endpoint = api_info[\"endpoint\"]\n\n    global _make_requests_concurrently_max_total_requests\n    _make_requests_concurrently_max_total_requests = max_total_requests\n\n    def get_session() -> requests.Session:\n        if not hasattr(thread_local, \"session\"):\n            thread_local.session = requests.Session()\n        return thread_local.session\n\n    def runnable():\n        session = get_session()\n        global _make_requests_concurrently_max_total_requests\n        start_sync.wait()\n        while not event_stopper.is_set():\n            if _make_requests_concurrently_max_total_requests is not None:\n                with lock:\n                    if _make_requests_concurrently_max_total_requests == 0:\n                        break\n                    _make_requests_concurrently_max_total_requests -= 1\n            start = time.time()\n            response = session.post(endpoint, json=payload, params=query_params)\n            assert (\n                response.status_code == HTTPStatus.OK\n            ), f\"status code: got {response.status_code}, expected {HTTPStatus.OK}\"\n            if latencies is not None:\n                latencies.append(time.time() - start)\n            if responses is not None:\n                responses.append(response)\n\n        if _make_requests_concurrently_max_total_requests is not None:\n            end_sync.wait()\n            event_stopper.set()\n\n    futures_list = []\n    for _ in range(concurrency):\n        future = executor.submit(runnable)\n        futures_list.append(future)\n\n    return futures_list\n\n\ndef retrieve_results_concurrently(\n    client: cx.Client,\n    api_name: str,\n    concurrency: int,\n    event_stopper: td.Event,\n    job_ids: List[str],\n    responses: List[Tuple[str, Dict[str, Any]]] = [],\n    poll_sleep_seconds: int = 1,\n    timeout: Optional[int] = None,\n):\n    api_info = client.get_api(api_name)\n    task_kind = api_info[\"spec\"][\"kind\"] == \"TaskAPI\"\n    async_kind = api_info[\"spec\"][\"kind\"] == \"AsyncAPI\"\n    if not task_kind and not async_kind:\n        raise ValueError(\"function can only be called for TaskAPI/AsyncAPI kinds\")\n\n    exec = futures.ThreadPoolExecutor(concurrency)\n    thread_local = td.local()\n\n    def _get_session() -> requests.Session:\n        if not hasattr(thread_local, \"session\"):\n            thread_local.session = requests.Session()\n        return thread_local.session\n\n    def _retriever(request_id: str):\n        session = _get_session()\n\n        while not event_stopper.is_set():\n            if task_kind:\n                result_response = session.get(f\"{api_info['endpoint']}?jobID={request_id}\")\n            if async_kind:\n                result_response = session.get(f\"{api_info['endpoint']}/{request_id}\")\n\n            if result_response.status_code != HTTPStatus.OK:\n                content = result_response.content.decode(\"utf-8\")\n                if \"error\" in content:\n                    event_stopper.set()\n                    raise RuntimeError(\n                        f\"received {result_response.status_code} status code with the following message: {content}\"\n                    )\n                time.sleep(poll_sleep_seconds)\n                continue\n\n            result_response_json = result_response.json()\n            if async_kind and \"status\" in result_response_json:\n                if result_response_json[\"status\"] == \"completed\":\n                    break\n                if result_response_json[\"status\"] not in [\"in_progress\", \"in_queue\"]:\n                    raise RuntimeError(\n                        f\"status for request ID {request_id} got set to {result_response_json['status']}\"\n                    )\n\n            if (\n                task_kind\n                and \"job_status\" in result_response_json\n                and \"status\" in result_response_json[\"job_status\"]\n            ):\n                if result_response_json[\"job_status\"][\"status\"] == \"succeeded\":\n                    break\n                if result_response_json[\"job_status\"][\"status\"] not in [\n                    \"pending\",\n                    \"enqueuing\",\n                    \"running\",\n                ]:\n                    raise RuntimeError(\n                        f\"status for job ID {request_id} got set to {result_response_json['job_status']['status']}\"\n                    )\n\n        if event_stopper.is_set():\n            return\n\n        responses.append((request_id, result_response_json))\n\n    # will throw an exception if something failed in any thread\n    for _ in exec.map(_retriever, job_ids, timeout=timeout):\n        pass\n\n\ndef check_futures_healthy(futures_list: List[futures.Future]):\n    for future in futures_list:\n        has_exception = None\n        try:\n            has_exception = future.exception(timeout=0.0)\n        except futures.TimeoutError:\n            pass\n        if has_exception:\n            future.result()\n\n\ndef client_from_config(config_path: str) -> cx.Client:\n    with open(config_path) as f:\n        config = yaml.safe_load(f)\n\n    cluster_name = config[\"cluster_name\"]\n\n    return cx.client(f\"{cluster_name}\")\n\n\ndef stream_api_logs(client: cx.Client, api_name: str):\n    cx.run_cli([\"logs\", api_name, \"--random-pod\", \"-e\", client.env_name])\n\n\ndef stream_job_logs(client: cx.Client, api_name: str, job_id: str):\n    cx.run_cli([\"logs\", api_name, job_id, \"--random-pod\", \"-e\", client.env_name])\n"
  },
  {
    "path": "test/e2e/pytest.ini",
    "content": "# pytest.ini\n[pytest]\nminversion = 6.0\naddopts = -s -v -r sxf\n"
  },
  {
    "path": "test/e2e/setup.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom pathlib import Path\n\nfrom setuptools import setup, find_packages\n\nroot = Path(__file__).parent.absolute()\ncortex_client_dir = root.parent.parent / \"python\" / \"client\"\n\nif not cortex_client_dir.exists():\n    raise ModuleNotFoundError(f\"cortex client not found in {cortex_client_dir}\")\n\nsetup(\n    name=\"e2e\",\n    version=\"master\",  # CORTEX_VERSION\n    packages=find_packages(exclude=[\"tests\"]),\n    url=\"https://github.com/cortexlabs/cortex\",\n    license=\"Apache License 2.0\",\n    python_requires=\">=3.6\",\n    install_requires=[\n        \"requests==2.24.0\",\n        \"jsonschema==3.2.0\",\n        \"pytest==6.1.*\",\n        \"pytest-print==0.2.1\",\n        \"python-dotenv==0.15.0\",\n        \"pyyaml>=5.3.1\",\n        \"boto3>=1.14.53\",\n        \"cortex\",\n    ],\n    dependency_links=[f\"file://{cortex_client_dir}#egg=cortex\"],\n    author=\"Cortex Labs\",\n    author_email=\"dev@cortexlabs.com\",\n    description=\"Cortex E2E tests package\",\n)\n"
  },
  {
    "path": "test/e2e/tests/__init__.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n"
  },
  {
    "path": "test/e2e/tests/aws/__init__.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n"
  },
  {
    "path": "test/e2e/tests/aws/conftest.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport cortex as cx\nimport pytest\n\nimport e2e\nfrom e2e.utils import client_from_config\n\n\n@pytest.fixture\ndef client(config):\n    env_name = config[\"aws\"][\"env\"]\n    if env_name:\n        return cx.client(env_name)\n\n    config_path = config[\"aws\"][\"config\"]\n    if config_path is not None:\n        return client_from_config(config_path)\n\n    pytest.skip(\"--env or --config must be passed to run tests\")\n\n\ndef pytest_configure(config):\n    cluster_config = config.getoption(\"--config\")\n    if cluster_config:\n        e2e.create_cluster(cluster_config)\n\n\ndef pytest_unconfigure(config):\n    cluster_config = config.getoption(\"--config\")\n    if cluster_config:\n        e2e.delete_cluster(cluster_config)\n"
  },
  {
    "path": "test/e2e/tests/aws/test_async.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Callable, Dict\n\nimport cortex as cx\nimport pytest\n\nimport e2e.tests\n\nTEST_APIS = [\"async/text-generator\"]\nTEST_APIS_GPU = [\"async/text-generator\"]\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS)\ndef test_async_api(printer: Callable, config: Dict, client: cx.Client, api: str):\n    e2e.tests.test_async_api(\n        printer=printer,\n        client=client,\n        api=api,\n        deploy_timeout=config[\"global\"][\"async_deploy_timeout\"],\n        poll_retries=config[\"global\"][\"async_workload_timeout\"],\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n    )\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS_GPU)\ndef test_async_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: str):\n    skip_gpus = config[\"global\"].get(\"skip_gpus\", False)\n    if skip_gpus:\n        pytest.skip(\"--skip-gpus flag detected, skipping GPU tests\")\n\n    e2e.tests.test_async_api(\n        printer=printer,\n        client=client,\n        api=api,\n        deploy_timeout=config[\"global\"][\"async_deploy_timeout\"],\n        poll_retries=config[\"global\"][\"async_workload_timeout\"],\n        api_config_name=\"cortex_gpu.yaml\",\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n    )\n"
  },
  {
    "path": "test/e2e/tests/aws/test_autoscaling.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Any, Callable, Dict\n\nimport cortex as cx\nimport pytest\n\nimport e2e.tests\n\nTEST_APIS = [\n    {\n        \"primary\": \"realtime/sleep\",\n        \"dummy\": [\"realtime/prime-generator\"],\n        \"query_params\": {\n            \"sleep\": \"1.0\",\n        },\n    }\n]\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"apis\", TEST_APIS, ids=[api[\"primary\"] for api in TEST_APIS])\ndef test_autoscaling(printer: Callable, config: Dict, client: cx.Client, apis: Dict[str, Any]):\n    skip_autoscaling_test = config[\"global\"].get(\"skip_autoscaling\", False)\n    if skip_autoscaling_test:\n        pytest.skip(\"--skip-autoscaling flag detected, skipping autoscaling tests\")\n\n    e2e.tests.test_autoscaling(\n        printer,\n        client,\n        apis,\n        autoscaling_config=config[\"global\"][\"autoscaling_test_config\"],\n        deploy_timeout=config[\"global\"][\"realtime_deploy_timeout\"],\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n    )\n"
  },
  {
    "path": "test/e2e/tests/aws/test_batch.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Callable, Dict\n\nimport cortex as cx\nimport pytest\n\nimport e2e.tests\n\nTEST_APIS = [\"batch/image-classifier-alexnet\"]\nTEST_APIS_GPU = [\"batch/image-classifier-alexnet\"]\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS)\ndef test_batch_api(printer: Callable, config: Dict, client: cx.Client, api: str):\n    s3_path = config[\"aws\"].get(\"s3_path\")\n    if not s3_path:\n        pytest.skip(\n            \"--s3-path option is required to run batch tests (alternatively set the \"\n            \"CORTEX_TEST_BATCH_S3_PATH env var) )\"\n        )\n\n    e2e.tests.test_batch_api(\n        printer,\n        client,\n        api,\n        test_s3_path=s3_path,\n        deploy_timeout=config[\"global\"][\"batch_deploy_timeout\"],\n        job_timeout=config[\"global\"][\"batch_job_timeout\"],\n        retry_attempts=5,\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n        local_operator=config[\"global\"][\"local_operator\"],\n    )\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS_GPU)\ndef test_batch_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: str):\n    skip_gpus = config[\"global\"].get(\"skip_gpus\", False)\n    if skip_gpus:\n        pytest.skip(\"--skip-gpus flag detected, skipping GPU tests\")\n\n    s3_path = config[\"aws\"].get(\"s3_path\")\n    if not s3_path:\n        pytest.skip(\n            \"--s3-path option is required to run batch tests (alternatively set the \"\n            \"CORTEX_TEST_BATCH_S3_PATH env var) )\"\n        )\n\n    e2e.tests.test_batch_api(\n        printer=printer,\n        client=client,\n        api=api,\n        test_s3_path=s3_path,\n        deploy_timeout=config[\"global\"][\"batch_deploy_timeout\"],\n        job_timeout=config[\"global\"][\"batch_job_timeout\"],\n        retry_attempts=5,\n        api_config_name=\"cortex_gpu.yaml\",\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n        local_operator=config[\"global\"][\"local_operator\"],\n    )\n"
  },
  {
    "path": "test/e2e/tests/aws/test_load.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Callable, Dict\n\nimport cortex as cx\nimport pytest\n\nimport e2e.tests\n\nTEST_APIS_REALTIME = [\"realtime/prime-generator\"]\nTEST_APIS_ASYNC = [\"async/text-generator\"]\nTEST_APIS_BATCH = [\"batch/sum\"]\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS_REALTIME)\ndef test_load_realtime(printer: Callable, config: Dict, client: cx.Client, api: str):\n    skip_load_test = config[\"global\"].get(\"skip_load\", False)\n    if skip_load_test:\n        pytest.skip(\"--skip-load flag detected, skipping load tests\")\n\n    e2e.tests.test_load_realtime(\n        printer,\n        client,\n        api,\n        load_config=config[\"global\"][\"load_test_config\"][\"realtime\"],\n        deploy_timeout=config[\"global\"][\"realtime_deploy_timeout\"],\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n    )\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS_ASYNC)\ndef test_load_async(printer: Callable, config: Dict, client: cx.Client, api: str):\n    skip_load_test = config[\"global\"].get(\"skip_load\", False)\n    if skip_load_test:\n        pytest.skip(\"--skip-load flag detected, skipping load tests\")\n\n    e2e.tests.test_load_async(\n        printer,\n        client,\n        api,\n        load_config=config[\"global\"][\"load_test_config\"][\"async\"],\n        deploy_timeout=config[\"global\"][\"async_deploy_timeout\"],\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n    )\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS_BATCH)\ndef test_load_batch(printer: Callable, config: Dict, client: cx.Client, api: str):\n    skip_load_test = config[\"global\"].get(\"skip_load\", False)\n    if skip_load_test:\n        pytest.skip(\"--skip-load flag detected, skipping load tests\")\n\n    s3_path = config[\"aws\"].get(\"s3_path\")\n    if not s3_path:\n        pytest.skip(\n            \"--s3-path option is required to run batch tests (alternatively set the \"\n            \"CORTEX_TEST_BATCH_S3_PATH env var) )\"\n        )\n\n    e2e.tests.test_load_batch(\n        printer,\n        client,\n        api,\n        test_s3_path=s3_path,\n        load_config=config[\"global\"][\"load_test_config\"][\"batch\"],\n        deploy_timeout=config[\"global\"][\"batch_deploy_timeout\"],\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n    )\n"
  },
  {
    "path": "test/e2e/tests/aws/test_long_running.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Callable, Dict\n\nimport cortex as cx\nimport pytest\n\nimport e2e.tests\n\nTEST_APIS = [\"realtime/text-generator\"]\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS)\ndef test_long_running_realtime(printer: Callable, config: Dict, client: cx.Client, api: str):\n    skip_load_test = config[\"global\"].get(\"skip_long_running\", False)\n    if skip_load_test:\n        pytest.skip(\"--skip-long-running flag detected, skipping long-running test\")\n\n    e2e.tests.test_long_running_realtime(\n        printer,\n        client,\n        api,\n        long_running_config=config[\"global\"][\"long_running_test_config\"],\n        deploy_timeout=config[\"global\"][\"realtime_deploy_timeout\"],\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n    )\n"
  },
  {
    "path": "test/e2e/tests/aws/test_realtime.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nfrom typing import Callable, Dict\n\nimport cortex as cx\nimport pytest\n\nimport e2e.tests\n\nTEST_APIS = [\n    {\n        \"name\": \"realtime/image-classifier-resnet50\",\n        \"extra_path\": \"v1/models/resnet50:predict\",\n    },\n    {\n        \"name\": \"realtime/prime-generator\",\n        \"extra_path\": \"\",\n    },\n    {\n        \"name\": \"realtime/text-generator\",\n        \"extra_path\": \"\",\n    },\n]\nTEST_APIS_ARM = [\n    {\n        \"name\": \"realtime/hello-world\",\n        \"extra_path\": \"\",\n    },\n]\nTEST_APIS_GPU = [\n    {\n        \"name\": \"realtime/image-classifier-resnet50\",\n        \"extra_path\": \"v1/models/resnet50:predict\",\n    },\n    {\n        \"name\": \"realtime/text-generator\",\n        \"extra_path\": \"\",\n    },\n]\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS, ids=[api[\"name\"] for api in TEST_APIS])\ndef test_realtime_api(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]):\n    e2e.tests.test_realtime_api(\n        printer=printer,\n        client=client,\n        api=api[\"name\"],\n        timeout=config[\"global\"][\"realtime_deploy_timeout\"],\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n        extra_path=api[\"extra_path\"],\n    )\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS_ARM, ids=[api[\"name\"] for api in TEST_APIS_ARM])\ndef test_realtime_api_arm(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]):\n    e2e.tests.test_realtime_api(\n        printer=printer,\n        client=client,\n        api=api[\"name\"],\n        timeout=config[\"global\"][\"realtime_deploy_timeout\"],\n        api_config_name=\"cortex_cpu_arm64.yaml\",\n        node_groups=config[\"aws\"][\"arm_nodegroups\"],\n        extra_path=api[\"extra_path\"],\n        method=\"GET\",\n    )\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS_GPU, ids=[api[\"name\"] for api in TEST_APIS_GPU])\ndef test_realtime_api_gpu(printer: Callable, config: Dict, client: cx.Client, api: Dict[str, str]):\n    skip_gpus = config[\"global\"].get(\"skip_gpus\", False)\n    if skip_gpus:\n        pytest.skip(\"--skip-gpus flag detected, skipping GPU tests\")\n\n    e2e.tests.test_realtime_api(\n        printer=printer,\n        client=client,\n        api=api[\"name\"],\n        timeout=config[\"global\"][\"realtime_deploy_timeout\"],\n        api_config_name=\"cortex_gpu.yaml\",\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n        extra_path=api[\"extra_path\"],\n    )\n"
  },
  {
    "path": "test/e2e/tests/aws/test_scale_to_zero.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Dict\n\nimport pytest\n\nimport cortex as cx\nimport e2e.tests\n\nTEST_APIS = [{\"api\": \"realtime/hello-world\", \"config\": \"cortex_scale_to_zero.yaml\"}]\n\n\n@pytest.mark.parametrize(\"api\", TEST_APIS, ids=[api[\"api\"] for api in TEST_APIS])\n@pytest.mark.usefixtures(\"client\")\ndef test_scale_to_zero_realtime(config: Dict, client: cx.Client, api: Dict[str, str]):\n    e2e.tests.test_realtime_scale_to_zero(\n        client=client,\n        api=api[\"api\"],\n        timeout=config[\"global\"][\"realtime_deploy_timeout\"],\n        api_config_name=api[\"config\"],\n    )\n"
  },
  {
    "path": "test/e2e/tests/aws/test_task.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom typing import Callable, Dict\n\nimport cortex as cx\nimport pytest\n\nimport e2e.tests\n\nTEST_APIS = [\"task/iris-classifier-trainer\"]\n\n\n@pytest.mark.usefixtures(\"client\")\n@pytest.mark.parametrize(\"api\", TEST_APIS)\ndef test_task_api(printer: Callable, config: Dict, client: cx.Client, api: str):\n    e2e.tests.test_task_api(\n        printer,\n        client,\n        api,\n        retry_attempts=5,\n        deploy_timeout=config[\"global\"][\"task_deploy_timeout\"],\n        job_timeout=config[\"global\"][\"task_job_timeout\"],\n        node_groups=config[\"aws\"][\"x86_nodegroups\"],\n        local_operator=config[\"global\"][\"local_operator\"],\n    )\n"
  },
  {
    "path": "test/e2e/tests/conftest.py",
    "content": "# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport os\n\nimport pytest\nimport yaml\nfrom dotenv import load_dotenv\n\n\ndef pytest_addoption(parser):\n    parser.addoption(\n        \"--env\",\n        action=\"store\",\n        default=None,\n        help=\"set cortex environment, to test on an existing cluster\",\n    )\n    parser.addoption(\n        \"--config\",\n        action=\"store\",\n        default=None,\n        help=\"set cortex cluster config, to test on a new cluster\",\n    )\n    parser.addoption(\n        \"--s3-path\",\n        action=\"store\",\n        default=None,\n        help=\"set s3 path where batch jobs results will be stored\",\n    )\n    parser.addoption(\n        \"--skip-gpus\",\n        action=\"store_true\",\n        help=\"skip GPU tests\",\n    )\n    parser.addoption(\n        \"--skip-infs\",\n        action=\"store_true\",\n        help=\"skip Inferentia tests\",\n    )\n    parser.addoption(\n        \"--skip-autoscaling\",\n        action=\"store_true\",\n        help=\"skip autoscaling tests\",\n    )\n    parser.addoption(\n        \"--skip-load\",\n        action=\"store_true\",\n        help=\"skip load tests\",\n    )\n    parser.addoption(\n        \"--skip-long-running\",\n        action=\"store_true\",\n        help=\"skip long-running test\",\n    )\n    parser.addoption(\n        \"--local-operator\",\n        action=\"store_true\",\n        help=\"enable for using testing against BatchAPI with a local operator\",\n    )\n    parser.addoption(\n        \"--arm-nodegroups\",\n        action=\"store\",\n        default=None,\n        help=\"arm nodegroups to run the arm tests on\",\n    )\n    parser.addoption(\n        \"--x86-nodegroups\",\n        action=\"store\",\n        default=None,\n        help=\"x86 nodegroups to run the x86 tests on\",\n    )\n\n\ndef pytest_configure(config):\n    load_dotenv(\".env\")\n\n    s3_path = os.environ.get(\"CORTEX_TEST_BATCH_S3_PATH\")\n    s3_path = config.getoption(\"--s3-path\") if not s3_path else s3_path\n\n    arm_nodegroups = []\n    if config.getoption(\"--arm-nodegroups\"):\n        arm_nodegroups = config.getoption(\"--arm-nodegroups\").split(\",\")\n\n    x86_nodegroups = []\n    if config.getoption(\"--x86-nodegroups\"):\n        x86_nodegroups = config.getoption(\"--x86-nodegroups\").split(\",\")\n\n    configuration = {\n        \"aws\": {\n            \"env\": config.getoption(\"--env\"),\n            \"config\": config.getoption(\"--config\"),\n            \"s3_path\": s3_path,\n            \"arm_nodegroups\": arm_nodegroups,\n            \"x86_nodegroups\": x86_nodegroups,\n        },\n        \"global\": {\n            \"local_operator\": config.getoption(\"--local-operator\"),\n            \"realtime_deploy_timeout\": int(\n                os.environ.get(\"CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT\", 320)\n            ),\n            \"batch_deploy_timeout\": int(os.environ.get(\"CORTEX_TEST_BATCH_DEPLOY_TIMEOUT\", 150)),\n            \"batch_job_timeout\": int(os.environ.get(\"CORTEX_TEST_BATCH_JOB_TIMEOUT\", 200)),\n            \"async_deploy_timeout\": int(os.environ.get(\"CORTEX_TEST_ASYNC_DEPLOY_TIMEOUT\", 320)),\n            \"async_workload_timeout\": int(\n                os.environ.get(\"CORTEX_TEST_ASYNC_WORKLOAD_TIMEOUT\", 200)\n            ),\n            \"task_deploy_timeout\": int(os.environ.get(\"CORTEX_TEST_TASK_DEPLOY_TIMEOUT\", 75)),\n            \"task_job_timeout\": int(os.environ.get(\"CORTEX_TEST_TASK_JOB_TIMEOUT\", 200)),\n            \"skip_gpus\": config.getoption(\"--skip-gpus\"),\n            \"skip_infs\": config.getoption(\"--skip-infs\"),\n            \"skip_autoscaling\": config.getoption(\"--skip-autoscaling\"),\n            \"skip_long_running\": config.getoption(\"--skip-long-running\"),\n            \"skip_load\": config.getoption(\"--skip-load\"),\n            \"autoscaling_test_config\": {\n                \"max_replicas\": 20,\n            },\n            \"load_test_config\": {\n                \"realtime\": {\n                    \"total_requests\": 10 ** 5,\n                    \"desired_replicas\": 50,\n                    \"concurrency\": 50,\n                    \"status_code_timeout\": 60,  # measured in seconds\n                },\n                \"async\": {\n                    \"total_requests\": 10 ** 3,\n                    \"desired_replicas\": 20,\n                    \"concurrency\": 10,\n                    \"submit_timeout\": 120,  # measured in seconds\n                    \"workload_timeout\": 120,  # measured in seconds\n                },\n                \"batch\": {\n                    \"jobs\": 10,\n                    \"workers_per_job\": 10,\n                    \"items_per_job\": 10 ** 5,\n                    \"batch_size\": 10 * 2,\n                    \"workload_timeout\": 300,  # measured in seconds\n                },\n            },\n            \"long_running_test_config\": {\n                \"time_to_run\": 5 * 24 * 3600,  # measured in seconds\n                \"status_code_timeout\": 60,  # measured in seconds\n            },\n        },\n    }\n\n    class Config:\n        @pytest.fixture(autouse=True)\n        def config(self):\n            return configuration\n\n    config.pluginmanager.register(Config())\n\n    print(\"\\n----- Test Configuration -----\\n\")\n    print(yaml.dump(configuration, indent=2))\n\n    if configuration[\"aws\"][\"env\"] and configuration[\"aws\"][\"config\"]:\n        raise ValueError(\"--env and --config are mutually exclusive\")\n"
  },
  {
    "path": "test/utils/README.md",
    "content": "## Throughput tester\n\n[throughput_test.py](throughput_test.py) is a Python CLI that can be used to test the throughput of your deployed API. The throughput will vary depending on your API's configuration (specified in your `cortex.yaml` file), your local machine's resources (mostly CPU, since it has to spawn many concurrent requests), and the internet connection on your local machine.\n\n```bash\nUsage: throughput_test.py [OPTIONS] ENDPOINT PAYLOAD\n\n  Program for testing the throughput of Cortex-deployed APIs.\n\nOptions:\n  -w, --processes INTEGER   Number of processes for prediction requests.  [default: 1]\n  -t, --threads INTEGER     Number of threads per process for prediction requests.  [default: 1]\n  -s, --samples INTEGER     Number of samples to run per thread.  [default: 10]\n  -i, --time-based FLOAT    How long the thread making predictions will run for in seconds.\n                            If set, -s option will be ignored.\n  --help                    Show this message and exit.\n```\n\n`ENDPOINT` is the API's endpoint, which you can get by running `cortex get <API-name>`. This argument can also be exported as an environment variable instead of being passed to the CLI.\n\n`PAYLOAD` can either be a local file or an URL resource that points to a file. The allowed extension types for the file are `json` and `jpg`. This argument can also be exported as an environment variable instead of being passed to the CLI.\n\n* `json` files are generally `sample.json`s as they are found in most Cortex examples. Each of these is attached to the request as payload. The content type of the request is `\"application/json\"`.\n* `jpg` images are read as numpy arrays and then are converted to a bytes object using `cv2.imencode` function. The content type of the request is `\"application/octet-stream\"`.\n\nThe same payload `PAYLOAD` is attached to all requests the script makes.\n\n### Dependencies\n\nThe [throughput_test.py](throughput_test.py) CLI has been tested with Python 3.6.9. To install the CLI's dependencies, run the following:\n\n```bash\npip install requests click opencv-contrib-python numpy validator-collection imageio\n```\n"
  },
  {
    "path": "test/utils/build-all.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# usage: ./build-all.sh [REGISTRY] [--skip-push]\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nset -eo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/../.. >/dev/null && pwd)\"\n\nfor f in $(find $ROOT/test/apis -type f -name 'build-*.sh'); do\n  \"$f\" \"$@\"\ndone\n"
  },
  {
    "path": "test/utils/build.sh",
    "content": "#!/bin/bash\n\n# Copyright 2022 Cortex Labs, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# note: this only meant to be called from the build*.sh files in each test api directory\n# usage: ./build.sh BUILDER_PATH IMAGE_NAME [REGISTRY] [--skip-push]\n#   PATH is e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/sleep/build-cpu.sh\n#   REGISTRY defaults to $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY; e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs or quay.io/cortexlabs-test\n\nset -eo pipefail\n\nROOT=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\"/../.. >/dev/null && pwd)\"\nsource $ROOT/dev/util.sh\n\nfunction registry_login() {\n  login_url=$1  # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu\n  region=$2\n\n  blue_echo \"\\nLogging in to ECR\"\n  aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $login_url\n  green_echo \"\\nSuccess\"\n}\n\nfunction create_ecr_repo() {\n  repo_name=$1  # e.g. cortexlabs/realtime-sleep-cpu\n  region=$2\n\n  blue_echo \"\\nCreating ECR repo $repo_name\"\n  aws ecr create-repository --repository-name=$repo_name --region=$region\n  green_echo \"\\nSuccess\"\n}\n\nshould_skip_push=\"false\"\npositional_args=()\nwhile [[ $# -gt 0 ]]; do\n  key=\"$1\"\n  case $key in\n    -s|--skip-push)\n    should_skip_push=\"true\"\n    shift\n    ;;\n    *)\n    positional_args+=(\"$1\")\n    shift\n    ;;\n  esac\ndone\nset -- \"${positional_args[@]}\"\n\nbuilder_path=\"$1\"  # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world/build-cpu.sh\nimage_name=\"$2\"  # e.g. realtime-hello-world-cpu\nregistry=${3:-$CORTEX_DEV_DEFAULT_IMAGE_REGISTRY}  # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs\n\nif [ -z \"$registry\" ]; then\n  error_echo \"registry must be provided as a positional arg, or $CORTEX_DEV_DEFAULT_IMAGE_REGISTRY must be set\"\nfi\n\nimage_url=\"${registry}/${image_name}\"  # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/realtime-sleep-cpu\ndockerfile_name=\"$(echo \"$builder_path\" | sed 's/.*build-//' | sed 's/\\..*//')\"  # e.g. cpu\napi_dir=\"$(dirname $builder_path)\"  # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world\ndockerfile_path=\"${api_dir}/${dockerfile_name}.Dockerfile\"  # e.g. /home/ubuntu/src/github.com/cortexlabs/cortex/test/apis/realtime/hello-world/cpu.Dockerfile\nif [[ \"$registry\" == *\".ecr.\"* ]]; then\n  login_url=\"$(echo \"$registry\" | sed 's/\\/.*//')\"  # e.g. 764403040460.dkr.ecr.us-west-2.amazonaws.com\n  repo_name=\"$(echo $image_url | sed 's/[^\\/]*\\///')\"  # e.g. cortexlabs/realtime-sleep-cpu\n  region=\"$(echo \"$registry\" | sed 's/.*\\.ecr\\.//' | sed 's/\\..*//')\"  # e.g. us-west-2\nfi\n\nif [ ! -f \"$dockerfile_path\" ]; then\n  error_echo \"$dockerfile_path does not exist\"\n  exit 1\nfi\n\nblue_echo \"Building $image_url:latest\\n\"\ndocker build \"$api_dir\" -f \"$dockerfile_path\" -t \"$image_url\"\ngreen_echo \"\\nBuilt $image_url:latest\"\n\nif [ \"$should_skip_push\" = \"true\" ]; then\n  exit 0\nfi\n\nwhile true; do\n  blue_echo \"\\nPushing $image_url:latest\"\n  exec 5>&1\n  set +e\n  out=$(docker push $image_url 2>&1 | tee /dev/fd/5; exit ${PIPESTATUS[0]})\n  exit_code=$?\n  set -e\n  if [ $exit_code -ne 0 ]; then\n    if [[ \"$image_url\" != *\".ecr.\"* ]]; then\n      exit $exit_code\n    else\n      if [[ \"$out\" == *\"authorization token has expired\"* ]] || [[ \"$out\" == *\"no basic auth credentials\"* ]]; then\n        registry_login $login_url $region\n        continue\n      elif [[ \"$out\" == *\"repository with name\"*\"does not exist\"* ]]; then\n        create_ecr_repo $repo_name $region\n        continue\n      else\n        exit $exit_code\n      fi\n    fi\n  fi\n  green_echo \"\\nPushed $image_url:latest\"\n  break\ndone\n\n# update api config\nfind $api_dir -type f -name 'cortex_*.yaml' \\\n  -exec sed -i \"s|quay.io/cortexlabs-test/${image_name}|${image_url}|g\" {} \\;\n"
  },
  {
    "path": "test/utils/throughput_test.py",
    "content": "import os\nimport sys\nimport click\nimport concurrent.futures\nimport requests\nimport imageio\nimport json\nimport time\nimport itertools\nimport cv2\nimport numpy as np\n\nfrom validator_collection import checkers\n\n\n@click.command(help=\"Program for testing the throughput of Cortex-deployed APIs.\")\n@click.argument(\"endpoint\", type=str, envvar=\"ENDPOINT\")\n@click.argument(\"payload\", type=str, envvar=\"PAYLOAD\")\n@click.option(\n    \"--processes\",\n    \"-p\",\n    type=int,\n    default=1,\n    show_default=True,\n    help=\"Number of processes for requests.\",\n)\n@click.option(\n    \"--threads\",\n    \"-t\",\n    type=int,\n    default=1,\n    show_default=True,\n    help=\"Number of threads per process for requests.\",\n)\n@click.option(\n    \"--samples\",\n    \"-s\",\n    type=int,\n    default=10,\n    show_default=True,\n    help=\"Number of samples to run per thread.\",\n)\n@click.option(\n    \"--time-based\",\n    \"-i\",\n    type=float,\n    default=0.0,\n    help=\"How long the thread making requests will run for in seconds. If set, -s option will be ignored.\",\n)\ndef main(payload, endpoint, processes, threads, samples, time_based):\n    file_type = None\n    if checkers.is_url(payload):\n        if payload.lower().endswith(\".json\"):\n            file_type = \"json\"\n            payload_data = requests.get(payload).json()\n        elif payload.lower().endswith(\".jpg\"):\n            file_type = \"jpg\"\n            payload_data = imageio.imread(payload)\n    elif checkers.is_file(payload):\n        if payload.lower().endswith(\".json\"):\n            file_type = \"json\"\n            with open(payload, \"r\") as f:\n                payload_data = json.load(f)\n        elif payload.lower().endswith(\".jpg\"):\n            file_type = \"jpg\"\n            payload_data = cv2.imread(payload, cv2.IMREAD_COLOR)\n    else:\n        print(f\"'{payload}' isn't an URL resource, nor is it a local file\")\n        sys.exit(1)\n\n    if file_type is None:\n        print(f\"'{payload}' doesn't point to a jpg image or to a json file\")\n        sys.exit(1)\n    if file_type == \"jpg\":\n        data = image_to_jpeg_bytes(payload_data)\n    if file_type == \"json\":\n        data = json.dumps(payload_data)\n\n    print(\"Starting the inference throughput test...\")\n    results = []\n    start = time.time()\n    with concurrent.futures.ProcessPoolExecutor(max_workers=processes) as executor:\n        results = executor_submitter(\n            executor, processes, process_worker, threads, data, endpoint, samples, time_based\n        )\n    end = time.time()\n    elapsed = end - start\n\n    total_requests = sum(results)\n\n    print(f\"A total of {total_requests} requests have been served in {elapsed} seconds\")\n    print(f\"Avg number of inferences/sec is {total_requests / elapsed}\")\n    print(f\"Avg time spent on an inference is {elapsed / total_requests} seconds\")\n\n\ndef process_worker(threads, data, endpoint, samples, time_based):\n    results = []\n    with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:\n        results = executor_submitter(executor, threads, task, data, endpoint, samples, time_based)\n\n    return results\n\n\ndef executor_submitter(executor, workers, *args, **kwargs):\n    futures = []\n    for worker in range(workers):\n        future = executor.submit(*args, **kwargs)\n        futures.append(future)\n\n    results = [future.result() for future in futures]\n    results = list(itertools.chain.from_iterable(results))\n\n    return results\n\n\ndef task(data, endpoint, samples, time_based):\n    timeout = 60\n\n    if isinstance(data, str):\n        headers = {\"content-type\": \"application/json\"}\n    elif isinstance(data, bytes):\n        headers = {\"content-type\": \"application/octet-stream\"}\n    else:\n        return\n\n    if time_based == 0.0:\n        for i in range(samples):\n            try:\n                resp = requests.post(\n                    endpoint,\n                    data=data,\n                    headers=headers,\n                    timeout=timeout,\n                )\n            except Exception as e:\n                print(e)\n                break\n            time.sleep(0.1)\n        return [samples]\n    else:\n        start = time.time()\n        counter = 0\n        while start + time_based >= time.time():\n            try:\n                resp = requests.post(\n                    endpoint,\n                    data=data,\n                    headers=headers,\n                    timeout=timeout,\n                )\n            except Exception as e:\n                print(e)\n                break\n            time.sleep(0.1)\n            counter += 1\n        return [counter]\n\n\ndef image_to_jpeg_nparray(image, quality=[int(cv2.IMWRITE_JPEG_QUALITY), 95]):\n    \"\"\"\n    Convert numpy image to jpeg numpy vector.\n    \"\"\"\n    is_success, im_buf_arr = cv2.imencode(\".jpg\", image, quality)\n    return im_buf_arr\n\n\ndef image_to_jpeg_bytes(image, quality=[int(cv2.IMWRITE_JPEG_QUALITY), 95]):\n    \"\"\"\n    Convert numpy image to bytes-encoded jpeg image.\n    \"\"\"\n    buf = image_to_jpeg_nparray(image, quality)\n    byte_im = buf.tobytes()\n    return byte_im\n\n\nif __name__ == \"__main__\":\n    main()\n"
  }
]