[ { "path": ".dockerignore", "content": ".git/\n.github/\nbin/\nCHANGELOG/\nDockerfile\nMakefile\naws-k8s-tester\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "*Issue #, if available:*\n\n*Description of changes:*\n\n\nBy submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.\n" }, { "path": ".github/workflows/build-neuron-ci.yaml", "content": "name: \"Neuron Images CI\"\non:\n pull_request:\n types:\n - opened\n - reopened\n - synchronize\n paths:\n - 'test/images/neuron**'\njobs:\n build-image-neuronx:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: ./hack/free-disk-space.sh\n - run: docker build --file test/images/neuron/Dockerfile .\n build-image-neuron-training:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: ./hack/free-disk-space.sh\n - run: docker build --file test/images/neuron-training/Dockerfile test/images/neuron-training\n build-image-neuron-inference:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: ./hack/free-disk-space.sh\n - run: docker build --file test/images/neuron-inference/Dockerfile test/images/neuron-inference\n" }, { "path": ".github/workflows/build-nvidia-ci.yaml", "content": "name: \"Nvidia Images CI\"\non:\n pull_request:\n types:\n - opened\n - reopened\n - synchronize\n paths:\n - 'test/images/nvidia**'\njobs:\n build-image-nvidia:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: ./hack/free-disk-space.sh\n - run: docker build --file test/images/nvidia/Dockerfile .\n build-image-nvidia-training:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: ./hack/free-disk-space.sh\n - run: |\n docker build --file test/images/nvidia-training/Dockerfile test/images/nvidia-training \\\n --build-arg PYTORCH_BUILD_ENV=\"MAX_JOBS=$(($(nproc) - 2)) USE_MKLDNN=0 USE_DISTRIBUTED=0 USE_CUDA=0 USE_ROCM=0 USE_CAFFE2=0 USE_QNNPACK=0 USE_NNPACK=0 USE_XNNPACK=0 USE_MPS=0 BUILD_SHARED_LIBS=OFF USE_FLASH_ATTENTION=0 USE_MEM_EFF_ATTENTION=0 BUILD_TEST=0\"\n build-image-nvidia-inference:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: ./hack/free-disk-space.sh\n - run: |\n docker build --file test/images/nvidia-inference/Dockerfile test/images/nvidia-inference \\\n --build-arg PYTORCH_BUILD_ENV=\"MAX_JOBS=$(($(nproc) - 2)) USE_MKLDNN=0 USE_DISTRIBUTED=0 USE_CUDA=0 USE_ROCM=0 USE_CAFFE2=0 USE_QNNPACK=0 USE_NNPACK=0 USE_XNNPACK=0 USE_MPS=0 BUILD_SHARED_LIBS=OFF USE_FLASH_ATTENTION=0 USE_MEM_EFF_ATTENTION=0 BUILD_TEST=0\"\n" }, { "path": ".github/workflows/ci.yaml", "content": "name: \"CI\"\non:\n pull_request:\n types:\n - opened\n - reopened\n - synchronize\n paths-ignore:\n - 'test/images/nvidia**'\n - 'test/images/neuron**'\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: go build ./...\n - run: go test ./...\n build-test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: go test -c -tags=e2e ./test/...\n build-image:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: ./hack/free-disk-space.sh\n - run: docker build --build-arg=KUBERNETES_MINOR_VERSION=latest --file Dockerfile .\n build-image-efa:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - run: ./hack/free-disk-space.sh\n - run: docker build --file test/images/efa/Dockerfile .\n" }, { "path": ".github/workflows/update-go-dependencies.yaml", "content": "name: \"[CI] update-go-dependencies\"\non:\n workflow_dispatch:\n schedule:\n # once a week\n - cron: \"0 0 * * 0\"\npermissions:\n id-token: write\n contents: write\n pull-requests: write\njobs:\n update-dependencies:\n runs-on: ubuntu-latest\n if: github.repository == 'aws/aws-k8s-tester'\n steps:\n - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2\n - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # 5.5.0\n - run: |\n ./hack/update-go-dependencies.sh\n - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # 7.0.8\n with:\n branch: update-go-dependencies\n base: main\n add-paths: |\n .\n commit-message: \"chore: update go dependencies\"\n committer: \"GitHub \"\n author: \"GitHub \"\n title: \"chore: update go dependencies\"\n body: |\n Generated by:\n ```\n ./hack/update-go-dependencies.sh\n ```\n" }, { "path": ".github/workflows/update-image-tags.yaml", "content": "name: \"[CI] update-image-tags\"\non:\n workflow_dispatch:\n schedule:\n # once a week\n - cron: \"0 0 * * 0\"\npermissions:\n id-token: write\n contents: write\n pull-requests: write\njobs:\n update-dependencies:\n runs-on: ubuntu-latest\n if: github.repository == 'aws/aws-k8s-tester'\n steps:\n - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2\n - run: ./hack/update-image-tags.sh\n - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # 7.0.8\n with:\n branch: update-image-tags\n base: main\n add-paths: |\n test/images/\n commit-message: \"chore: update image tags\"\n committer: \"GitHub \"\n author: \"GitHub \"\n title: \"chore: update image tags\"\n body: |\n Generated by:\n ```\n ./hack/update-image-tags.sh\n ```\n" }, { "path": ".github/workflows/update-neuron-dependencies.yaml", "content": "name: \"[CI] update-neuron-dependencies\"\non:\n workflow_dispatch:\n schedule:\n # once a week\n - cron: \"0 0 * * 0\"\npermissions:\n id-token: write\n contents: write\n pull-requests: write\njobs:\n update-dependencies:\n runs-on: ubuntu-latest\n if: github.repository == 'aws/aws-k8s-tester'\n steps:\n - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2\n - run: |\n ./hack/update-neuron-dependencies.sh\n - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # 7.0.8\n with:\n branch: update-neuron-dependencies\n base: main\n add-paths: |\n test/images/\n commit-message: \"chore: update neuron dependencies\"\n committer: \"GitHub \"\n author: \"GitHub \"\n title: \"chore: update neuron dependencies\"\n body: |\n Generated by:\n ```\n ./hack/update-neuron-dependencies.sh\n ```\n\n See the following URL for artifactes in the latest Neuron SDK release: https://awsdocs-neuron.readthedocs-hosted.com/en/latest/release-notes/releasecontent.html#latest-neuron-release-artifacts\n" }, { "path": ".github/workflows/update-nvidia-dependencies.yaml", "content": "name: \"[CI] update-nvidia-dependencies\"\non:\n workflow_dispatch:\n schedule:\n # once a week\n - cron: \"0 0 * * 0\"\npermissions:\n id-token: write\n contents: write\n pull-requests: write\njobs:\n update-dependencies:\n runs-on: ubuntu-latest\n if: github.repository == 'aws/aws-k8s-tester'\n steps:\n - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2\n - run: ./hack/update-nvidia-dependencies.sh\n - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # 7.0.8\n with:\n branch: update-nvidia-dependencies\n base: main\n add-paths: |\n test/images/\n commit-message: \"chore: update nvidia test dependencies\"\n committer: \"GitHub \"\n author: \"GitHub \"\n title: \"chore: update nvidia test dependencies\"\n body: |\n Generated by:\n ```\n ./hack/update-nvidia-dependencies.sh\n ```\n" }, { "path": ".gitignore", "content": "/.DS_Store\n/bin\n/_tmp\n.idea\n*.swp\n/aws-k8s-tester\n*/*/.DS_Store\n*/.DS_Store\n/_artifacts\n/_rundir\n" }, { "path": ".vscode/settings.json", "content": "{\n \"git.ignoreLimitWarning\": true\n}" }, { "path": "CODE_OF_CONDUCT.md", "content": "## Code of Conduct\nThis project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). \nFor more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact \nopensource-codeofconduct@amazon.com with any additional questions or comments.\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing Guidelines\n\nThank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional\ndocumentation, we greatly value feedback and contributions from our community.\n\nPlease read through this document before submitting any issues or pull requests to ensure we have all the necessary\ninformation to effectively respond to your bug report or contribution.\n\n\n## Reporting Bugs/Feature Requests\n\nWe welcome you to use the GitHub issue tracker to report bugs or suggest features.\n\nWhen filing an issue, please check [existing open](https://github.com/aws/aws-k8s-tester/issues), or [recently closed](https://github.com/aws/aws-k8s-tester/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already\nreported the issue. Please try to include as much information as you can. Details like these are incredibly useful:\n\n* A reproducible test case or series of steps\n* The version of our code being used\n* Any modifications you've made relevant to the bug\n* Anything unusual about your environment or deployment\n\n\n## Contributing via Pull Requests\nContributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:\n\n1. You are working against the latest source on the *master* branch.\n2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.\n3. You open an issue to discuss any significant work - we would hate for your time to be wasted.\n\nTo send us a pull request, please:\n\n1. Fork the repository.\n2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.\n3. Ensure local tests pass.\n4. Commit to your fork using clear commit messages.\n5. Send us a pull request, answering any default questions in the pull request interface.\n6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.\n\nGitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and\n[creating a pull request](https://help.github.com/articles/creating-a-pull-request/).\n\n\n## Finding contributions to work on\nLooking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/aws/aws-k8s-tester/labels/help%20wanted) issues is a great place to start.\n\n\n## Code of Conduct\nThis project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).\nFor more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact\nopensource-codeofconduct@amazon.com with any additional questions or comments.\n\n\n## Security issue notifications\nIf you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue.\n\n\n## Licensing\n\nSee the [LICENSE](https://github.com/aws/aws-k8s-tester/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.\n\nWe may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.\n" }, { "path": "Config", "content": "# This file is for Amazon internal build processes\n\n# Copyright 2025 Amazon.com, Inc. or its affiliates.\n# SPDX-License-Identifier: Apache-2.0\n\npackage.Aws-k8s-tester-mirror = {\n interfaces = (1.0);\n\n build-system = bgo-wrap-make;\n build-tools = {\n 1.0 = {\n BrazilMakeGo = 3.0;\n GoLang = 1.x;\n };\n };\n};\n" }, { "path": "Dockerfile", "content": "FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS builder\nARG TARGETOS\nARG TARGETARCH\nRUN dnf install -y git tar gzip make unzip gcc rsync wget jq\nARG GO_MINOR_VERSION=1.25\nRUN curl https://go.dev/dl/?mode=json | jq -r .[].version | grep \"^go${GO_MINOR_VERSION}\" | head -n1 > go-version.txt\nRUN wget -O go.tar.gz https://go.dev/dl/$(cat go-version.txt).${TARGETOS}-${TARGETARCH}.tar.gz && \\\n rm -rf /usr/local/go && \\\n tar -C /usr/local -xzf go.tar.gz\nENV GOPATH=/usr/local/go\nENV PATH=$PATH:$GOPATH/bin\nENV GOPROXY=direct\n\nWORKDIR $GOPATH/src/github.com/aws/aws-k8s-tester\nCOPY . .\nRUN go install ./...\nRUN go test -c -tags=e2e ./test/... -o $GOPATH/bin/\n\nRUN go install sigs.k8s.io/kubetest2 && \\\n go install sigs.k8s.io/kubetest2/kubetest2-tester-exec && \\\n go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo && \\\n go install sigs.k8s.io/hydrophone@latest\n\nFROM public.ecr.aws/amazonlinux/amazonlinux:2023\nARG TARGETOS\nARG TARGETARCH\nWORKDIR /workdir\nRUN dnf install -y tar gzip unzip wget openssh diffutils\nRUN wget -O awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \\\n unzip awscli.zip && \\\n ./aws/install\n# we need gsutil from the gcloud CLI for kubetest-tester-ginkgo\nRUN dnf install -y python3.13\nARG GCLOUD_SDK_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz\nRUN wget -O google-cloud-sdk.tar.gz -q $GCLOUD_SDK_URL && \\\n tar xzf google-cloud-sdk.tar.gz -C / && \\\n rm google-cloud-sdk.tar.gz && \\\n /google-cloud-sdk/install.sh \\\n --disable-installation-options \\\n --bash-completion=false \\\n --path-update=false \\\n --usage-reporting=false\nENV PATH=$PATH:/google-cloud-sdk/bin\nARG EKSCTL_VERSION=latest\nRUN wget -O eksctl.tar.gz \"https://github.com/eksctl-io/eksctl/releases/${EKSCTL_VERSION}/download/eksctl_Linux_${TARGETARCH}.tar.gz\" && \\\n tar xzf eksctl.tar.gz -C /bin/ && \\\n rm eksctl.tar.gz\nARG HELM_VERSION=v4.1.4\nRUN wget -O helm.tar.gz \"https://get.helm.sh/helm-${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz\" && \\\n tar xzf helm.tar.gz --strip-components=1 -C /bin/ \"${TARGETOS}-${TARGETARCH}/helm\" && \\\n rm helm.tar.gz\nARG KUBERNETES_MINOR_VERSION\nCOPY hack/download-kubernetes-binaries.sh .\nRUN ./download-kubernetes-binaries.sh \"${KUBERNETES_MINOR_VERSION}\" \"${TARGETOS}\" \"${TARGETARCH}\"\nRUN mkdir /info\nENV PATH=$PATH:/info\nRUN cp kubernetes-version.txt /info/\nRUN mv kubernetes/*/bin/* /bin/\nRUN rm -rf /workdir\nCOPY --from=builder /usr/local/go/bin/* /bin/\n" }, { "path": "LICENSE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "Makefile", "content": "include ${BGO_MAKEFILE}\n\npre-release::\n\tgo test -c -tags=e2e ./test/... -o $(GOBIN)\n\tgo install sigs.k8s.io/kubetest2/...@latest\n\nupdate-deps:\n\tfor SCRIPT in ./hack/update-*.sh; do \\\n \t\"$$SCRIPT\" ; \\\n\tdone\n\n.PHONY: test-integration\ntest-integration: ## Run unit and integration tests\n\tgo test -v -tags=integration ./...\n" }, { "path": "NOTICE", "content": "Awstester\nCopyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. \n" }, { "path": "README.md", "content": "# Tools for testing Kubernetes on AWS\n\n## Installation\n\nThis project will use rolling releases going forward; we recommend fetching the latest commit:\n```\ngo install github.com/aws/aws-k8s-tester/...@HEAD\n```\n\nYou'll need the standard `kubetest` tools as well:\n```\ngo install sigs.k8s.io/kubetest2/...@latest\n```\n\n## `kubetest2` deployers and testers for EKS\n\n\n### Usage\n\n**Auto-detect cluster version**\n\nThe deployers will search for a file called `kubernetes-version.txt` on your `PATH`.\nThis file should contain a valid tag for a Kubernetes release.\nThe `--kubernetes-version` flag can be omitted if this file exists.\n\n---\n\n### `eksctl` deployer\n\nThis deployer is a thin wrapper around `eksctl`.\n\nThe simplest usage is:\n```\nkubetest2 \\\n eksctl \\\n --kubernetes-version=X.XX \\\n --up \\\n --down \\\n --test=exec \\\n -- echo \"Hello world\"\n```\n\n**Additional flags**\n\n- `--instance-types` - comma-separated list of instance types to use for nodes\n- `--ami` - AMI ID for nodes\n- `--nodes` - number of nodes\n- `--region` - AWS region\n- `--config-file` - Path to eksctl config file (**if provided, other flags are ignored**)\n- `--availability-zones` - Node availability zones\n- `--ami-family` - AMI family to use: `AmazonLinux2023` | `Bottlerocket`\n- `--efa-enabled` - Enable Elastic Fabric Adapter for the nodegroup\n- `--volume-size` - Size of the node root volume in GB\n- `--private-networking` - Use private networking for nodes\n- `--with-oidc` - Enable OIDC provider for IAM roles for service accounts\n- `--deploy-target` - The target to deploy: `cluster` | `nodegroup` (defaults to `cluster`)\n- `--cluster-name` - Name of the EKS cluster (defaults to RunID if not specified)\n- `--unmanaged-nodegroup` - Use unmanaged nodegroup instead of managed nodegroup\n- `--nodegroup-name` - Name of the nodegroup (defaults to `ng-1`)\n\n---\n\n### `eksapi` deployer\n\nThis deployer calls the EKS API directly, instead of using CloudFormation for EKS resources.\n\nThe simplest usage is:\n```\nkubetest2 \\\n eksapi \\\n --kubernetes-version=X.XX \\\n --up \\\n --down \\\n --test=exec \\\n -- echo \"Hello world\"\n```\n\n**Additional flags**\n\n- `--instance-types` - comma-separated list of instance types to use for nodes\n- `--ami` - AMI ID for nodes\n- `--nodes` - number of nodes\n- `--region` - AWS region\n- `--endpoint-url` - Override the EKS endpoint URL\n- `--cluster-role-service-principal` - Additional service principal that can assume the cluster IAM role.\n\n---\n\n### `multi` tester\n\nThis tester wraps multiple executions of other testers.\n\nTester argument groups are separated by `--`, with the first group being passed to the `multi` tester itself.\n\nThe first positional argument of each subsequent group should be the name of a tester.\n\n```\nkubetest2 \\\n noop \\\n --test=multi \\\n -- \\\n --fail-fast=true \\\n -- \\\n ginkgo \\\n --focus-regex='\\[Conformance\\]' \\\n --parallel=4 \\\n -- \\\n exec \\\n go test ./my/test/package\n```\n" }, { "path": "bmg.json", "content": "{\n \"binary_artifacts_only\": true\n}\n" }, { "path": "cmd/kubetest2-eksapi/main.go", "content": "package main\n\nimport (\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksapi\"\n\t\"sigs.k8s.io/kubetest2/pkg/app\"\n)\n\nfunc main() {\n\tapp.Main(eksapi.DeployerName, eksapi.NewDeployer)\n}\n" }, { "path": "cmd/kubetest2-eksapi-janitor/main.go", "content": "package main\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"log/slog\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksapi\"\n)\n\nfunc main() {\n\tvar maxResourceAge time.Duration\n\tflag.DurationVar(&maxResourceAge, \"max-resource-age\", time.Hour*3, \"Maximum resource age\")\n\tvar workers int\n\tflag.IntVar(&workers, \"workers\", 1, \"number of workers to processes resources in parallel\")\n\tvar stackStatus string\n\tflag.StringVar(&stackStatus, \"stack-status\", \"\", \"only process stacks with a specific status\")\n\tvar emitMetrics bool\n\tflag.BoolVar(&emitMetrics, \"emit-metrics\", false, \"Send metrics to CloudWatch\")\n\tflag.Parse()\n\tj := eksapi.NewJanitor(maxResourceAge, emitMetrics, workers, stackStatus)\n\tif err := j.Sweep(context.Background()); err != nil {\n\t\tslog.Error(\"failed to sweep resources\", \"error\", err)\n\t\tos.Exit(1)\n\t}\n}\n" }, { "path": "cmd/kubetest2-eksctl/main.go", "content": "package main\n\nimport (\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksctl\"\n\t\"sigs.k8s.io/kubetest2/pkg/app\"\n)\n\nfunc main() {\n\tapp.Main(eksctl.DeployerName, eksctl.NewDeployer)\n}\n" }, { "path": "cmd/kubetest2-tester-ginkgo-v1/main.go", "content": "package main\n\nimport (\n\t\"github.com/aws/aws-k8s-tester/internal/testers/ginkgov1\"\n)\n\nfunc main() {\n\tginkgov1.Main()\n}\n" }, { "path": "cmd/kubetest2-tester-multi/main.go", "content": "package main\n\nimport \"github.com/aws/aws-k8s-tester/internal/testers/multi\"\n\nfunc main() {\n\tmulti.Main()\n}\n" }, { "path": "external/tools.go", "content": "//go:build tools\n// +build tools\n\npackage external\n\n// this file allows us to declare direct dependencies on our required external tools.\n// this file will not compile! that's expected.\n\nimport (\n\t_ \"sigs.k8s.io/kubetest2\"\n\t_ \"sigs.k8s.io/kubetest2/kubetest2-tester-exec\"\n\t_ \"sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo\"\n)\n" }, { "path": "go.mod", "content": "module github.com/aws/aws-k8s-tester\n\ngo 1.25.5\n\nrequire (\n\tgithub.com/aws/aws-sdk-go v1.55.8\n\tgithub.com/aws/aws-sdk-go-v2 v1.41.1\n\tgithub.com/aws/aws-sdk-go-v2/config v1.32.7\n\tgithub.com/aws/aws-sdk-go-v2/service/autoscaling v1.62.5\n\tgithub.com/aws/aws-sdk-go-v2/service/cloudformation v1.71.5\n\tgithub.com/aws/aws-sdk-go-v2/service/cloudwatch v1.53.1\n\tgithub.com/aws/aws-sdk-go-v2/service/ec2 v1.279.1\n\tgithub.com/aws/aws-sdk-go-v2/service/eks v1.76.4\n\tgithub.com/aws/aws-sdk-go-v2/service/iam v1.53.2\n\tgithub.com/aws/aws-sdk-go-v2/service/s3 v1.95.1\n\tgithub.com/aws/smithy-go v1.24.0\n\tgithub.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51\n\tgithub.com/spf13/pflag v1.0.10\n\tgithub.com/stretchr/testify v1.11.1\n\tgolang.org/x/exp v0.0.0-20251219203646-944ab1f22d93\n\tk8s.io/api v0.35.0\n\tk8s.io/apimachinery v0.35.0\n\tk8s.io/client-go v0.35.0\n\tk8s.io/klog v1.0.0\n\tk8s.io/klog/v2 v2.130.1\n\tsigs.k8s.io/controller-runtime v0.22.4\n\tsigs.k8s.io/karpenter v1.8.0\n\tsigs.k8s.io/kubetest2 v0.0.0-20260108084739-2f9a9397f033\n)\n\nrequire (\n\tgithub.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect\n\tgithub.com/awslabs/operatorpkg v0.0.0-20250909182303-e8e550b6f339 // indirect\n\tgithub.com/evanphx/json-patch/v5 v5.9.11 // indirect\n\tgithub.com/fxamacker/cbor/v2 v2.9.0 // indirect\n\tgithub.com/go-logr/zapr v1.3.0 // indirect\n\tgithub.com/mitchellh/hashstructure/v2 v2.0.2 // indirect\n\tgithub.com/pkg/errors v0.9.1\n\tgithub.com/robfig/cron/v3 v3.0.1 // indirect\n\tgithub.com/samber/lo v1.51.0 // indirect\n\tgithub.com/x448/float16 v0.8.4 // indirect\n\tgolang.org/x/crypto v0.46.0 // indirect\n\tgomodules.xyz/jsonpatch/v2 v2.4.0 // indirect\n\tk8s.io/apiextensions-apiserver v0.34.1 // indirect\n)\n\nrequire (\n\tcloud.google.com/go v0.121.2 // indirect\n\tcloud.google.com/go/compute/metadata v0.9.0 // indirect\n\tcloud.google.com/go/iam v1.5.2 // indirect\n\tcloud.google.com/go/storage v1.53.0 // indirect\n\tcuelabs.dev/go/oci/ociregistry v0.0.0-20240404174027-a39bec0462d2 // indirect\n\tcuelang.org/go v0.9.2 // indirect\n\tdario.cat/mergo v1.0.2 // indirect\n\tfilippo.io/edwards25519 v1.1.0 // indirect\n\tgithub.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect\n\tgithub.com/Azure/go-autorest v14.2.0+incompatible // indirect\n\tgithub.com/Azure/go-autorest/autorest v0.11.29 // indirect\n\tgithub.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect\n\tgithub.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect\n\tgithub.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect\n\tgithub.com/Azure/go-autorest/autorest/date v0.3.0 // indirect\n\tgithub.com/Azure/go-autorest/logger v0.2.1 // indirect\n\tgithub.com/Azure/go-autorest/tracing v0.6.0 // indirect\n\tgithub.com/MakeNowJust/heredoc/v2 v2.0.1 // indirect\n\tgithub.com/Microsoft/go-winio v0.6.2 // indirect\n\tgithub.com/ProtonMail/go-crypto v1.1.6 // indirect\n\tgithub.com/ThalesIgnite/crypto11 v1.2.5 // indirect\n\tgithub.com/agnivade/levenshtein v1.2.1 // indirect\n\tgithub.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect\n\tgithub.com/alibabacloud-go/cr-20160607 v1.0.1 // indirect\n\tgithub.com/alibabacloud-go/cr-20181201 v1.0.10 // indirect\n\tgithub.com/alibabacloud-go/darabonba-openapi v0.2.1 // indirect\n\tgithub.com/alibabacloud-go/debug v1.0.0 // indirect\n\tgithub.com/alibabacloud-go/endpoint-util v1.1.1 // indirect\n\tgithub.com/alibabacloud-go/openapi-util v0.1.0 // indirect\n\tgithub.com/alibabacloud-go/tea v1.2.2 // indirect\n\tgithub.com/alibabacloud-go/tea-utils v1.4.5 // indirect\n\tgithub.com/alibabacloud-go/tea-xml v1.1.3 // indirect\n\tgithub.com/aliyun/credentials-go v1.3.2 // indirect\n\tgithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/credentials v1.19.7 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/ssm v1.67.8\n\tgithub.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect\n\tgithub.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240318154307-a1a918375412 // indirect\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/blang/semver v3.5.1+incompatible // indirect\n\tgithub.com/blang/semver/v4 v4.0.0 // indirect\n\tgithub.com/buildkite/agent/v3 v3.81.0 // indirect\n\tgithub.com/buildkite/go-pipeline v0.13.1 // indirect\n\tgithub.com/buildkite/interpolate v0.1.3 // indirect\n\tgithub.com/buildkite/roko v1.2.0 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.3.0 // indirect\n\tgithub.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect\n\tgithub.com/clbanning/mxj/v2 v2.7.0 // indirect\n\tgithub.com/cloudflare/circl v1.6.3 // indirect\n\tgithub.com/cockroachdb/apd/v3 v3.2.1 // indirect\n\tgithub.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect\n\tgithub.com/containerd/stargz-snapshotter/estargz v0.18.1 // indirect\n\tgithub.com/coreos/go-oidc/v3 v3.17.0 // indirect\n\tgithub.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f // indirect\n\tgithub.com/cyphar/filepath-securejoin v0.4.1 // indirect\n\tgithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect\n\tgithub.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect\n\tgithub.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 // indirect\n\tgithub.com/dimchansky/utfbom v1.1.1 // indirect\n\tgithub.com/docker/cli v29.0.3+incompatible // indirect\n\tgithub.com/docker/distribution v2.8.3+incompatible // indirect\n\tgithub.com/docker/docker-credential-helpers v0.9.3 // indirect\n\tgithub.com/dustin/go-humanize v1.0.1 // indirect\n\tgithub.com/emicklei/go-restful/v3 v3.12.2 // indirect\n\tgithub.com/emicklei/proto v1.13.2 // indirect\n\tgithub.com/emirpasic/gods v1.18.1 // indirect\n\tgithub.com/felixge/httpsnoop v1.0.4 // indirect\n\tgithub.com/fsnotify/fsnotify v1.9.0 // indirect\n\tgithub.com/glebarez/go-sqlite v1.22.0 // indirect\n\tgithub.com/go-chi/chi v4.1.2+incompatible // indirect\n\tgithub.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect\n\tgithub.com/go-git/go-billy/v5 v5.8.0 // indirect\n\tgithub.com/go-git/go-git/v5 v5.17.1 // indirect\n\tgithub.com/go-ini/ini v1.67.0 // indirect\n\tgithub.com/go-jose/go-jose/v3 v3.0.4 // indirect\n\tgithub.com/go-logr/logr v1.4.3 // indirect\n\tgithub.com/go-logr/stdr v1.2.2 // indirect\n\tgithub.com/go-ole/go-ole v1.3.0 // indirect\n\tgithub.com/go-openapi/analysis v0.23.0 // indirect\n\tgithub.com/go-openapi/errors v0.22.1 // indirect\n\tgithub.com/go-openapi/jsonpointer v0.21.0 // indirect\n\tgithub.com/go-openapi/jsonreference v0.21.0 // indirect\n\tgithub.com/go-openapi/loads v0.22.0 // indirect\n\tgithub.com/go-openapi/runtime v0.28.0 // indirect\n\tgithub.com/go-openapi/spec v0.21.0 // indirect\n\tgithub.com/go-openapi/strfmt v0.23.0 // indirect\n\tgithub.com/go-openapi/swag v0.23.0 // indirect\n\tgithub.com/go-openapi/validate v0.24.0 // indirect\n\tgithub.com/go-piv/piv-go v1.11.0 // indirect\n\tgithub.com/gobwas/glob v0.2.3 // indirect\n\tgithub.com/gogo/protobuf v1.3.2 // indirect\n\tgithub.com/golang-jwt/jwt/v4 v4.5.2 // indirect\n\tgithub.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect\n\tgithub.com/golang/snappy v0.0.4 // indirect\n\tgithub.com/google/certificate-transparency-go v1.3.2-0.20250507091337-0eddb39e94f8 // indirect\n\tgithub.com/google/gnostic-models v0.7.0 // indirect\n\tgithub.com/google/go-cmp v0.7.0 // indirect\n\tgithub.com/google/go-containerregistry v0.20.7 // indirect\n\tgithub.com/google/go-github/v55 v55.0.0 // indirect\n\tgithub.com/google/go-querystring v1.1.0 // indirect\n\tgithub.com/google/licenseclassifier/v2 v2.0.0 // indirect\n\tgithub.com/google/s2a-go v0.1.9 // indirect\n\tgithub.com/google/uuid v1.6.0 // indirect\n\tgithub.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect\n\tgithub.com/googleapis/gax-go/v2 v2.15.0 // indirect\n\tgithub.com/gorilla/mux v1.8.1 // indirect\n\tgithub.com/hashicorp/go-cleanhttp v0.5.2 // indirect\n\tgithub.com/hashicorp/go-retryablehttp v0.7.7 // indirect\n\tgithub.com/in-toto/in-toto-golang v0.9.0 // indirect\n\tgithub.com/inconshreveable/mousetrap v1.1.0 // indirect\n\tgithub.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect\n\tgithub.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 // indirect\n\tgithub.com/jellydator/ttlcache/v3 v3.3.0 // indirect\n\tgithub.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect\n\tgithub.com/josharian/intern v1.0.0 // indirect\n\tgithub.com/json-iterator/go v1.1.12 // indirect\n\tgithub.com/kevinburke/ssh_config v1.2.0 // indirect\n\tgithub.com/klauspost/compress v1.18.1 // indirect\n\tgithub.com/knqyf263/go-rpmdb v0.1.0 // indirect\n\tgithub.com/mailru/easyjson v0.9.0 // indirect\n\tgithub.com/mattn/go-isatty v0.0.20 // indirect\n\tgithub.com/miekg/pkcs11 v1.1.1 // indirect\n\tgithub.com/mitchellh/go-homedir v1.1.0 // indirect\n\tgithub.com/mitchellh/go-wordwrap v1.0.1 // indirect\n\tgithub.com/mitchellh/mapstructure v1.5.0 // indirect\n\tgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect\n\tgithub.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect\n\tgithub.com/mozillazg/docker-credential-acr-helper v0.4.0 // indirect\n\tgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect\n\tgithub.com/ncruces/go-strftime v0.1.9 // indirect\n\tgithub.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 // indirect\n\tgithub.com/oklog/ulid v1.3.1 // indirect\n\tgithub.com/oleiade/reflections v1.1.0 // indirect\n\tgithub.com/open-policy-agent/opa v1.4.0 // indirect\n\tgithub.com/opencontainers/go-digest v1.0.0 // indirect\n\tgithub.com/opencontainers/image-spec v1.1.1 // indirect\n\tgithub.com/opentracing/opentracing-go v1.2.0 // indirect\n\tgithub.com/package-url/packageurl-go v0.1.2 // indirect\n\tgithub.com/pborman/uuid v1.2.1 // indirect\n\tgithub.com/pelletier/go-toml/v2 v2.2.4 // indirect\n\tgithub.com/pjbgf/sha1cd v0.3.2 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect\n\tgithub.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect\n\tgithub.com/prometheus/client_golang v1.23.2 // indirect\n\tgithub.com/prometheus/client_model v0.6.2 // indirect\n\tgithub.com/prometheus/common v0.66.1 // indirect\n\tgithub.com/prometheus/procfs v0.17.0 // indirect\n\tgithub.com/protocolbuffers/txtpbfmt v0.0.0-20240116145035-ef3ab179eed6 // indirect\n\tgithub.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect\n\tgithub.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect\n\tgithub.com/rogpeppe/go-internal v1.14.1 // indirect\n\tgithub.com/sagikazarmark/locafero v0.9.0 // indirect\n\tgithub.com/sassoftware/relic v7.2.1+incompatible // indirect\n\tgithub.com/secure-systems-lab/go-securesystemslib v0.10.0 // indirect\n\tgithub.com/sergi/go-diff v1.4.0 // indirect\n\tgithub.com/shibumi/go-pathspec v1.3.0 // indirect\n\tgithub.com/shirou/gopsutil/v3 v3.24.5 // indirect\n\tgithub.com/sigstore/cosign/v2 v2.4.1 // indirect\n\tgithub.com/sigstore/fulcio v1.6.3 // indirect\n\tgithub.com/sigstore/rekor v1.3.9 // indirect\n\tgithub.com/sigstore/sigstore v1.10.3 // indirect\n\tgithub.com/sigstore/timestamp-authority v1.2.2 // indirect\n\tgithub.com/sirupsen/logrus v1.9.3 // indirect\n\tgithub.com/skeema/knownhosts v1.3.1 // indirect\n\tgithub.com/sourcegraph/conc v0.3.0 // indirect\n\tgithub.com/spf13/afero v1.15.0 // indirect\n\tgithub.com/spf13/cast v1.9.2 // indirect\n\tgithub.com/spf13/cobra v1.10.2 // indirect\n\tgithub.com/spf13/viper v1.20.1 // indirect\n\tgithub.com/spiffe/go-spiffe/v2 v2.6.0 // indirect\n\tgithub.com/subosito/gotenv v1.6.0 // indirect\n\tgithub.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect\n\tgithub.com/tchap/go-patricia/v2 v2.3.2 // indirect\n\tgithub.com/thales-e-security/pool v0.0.2 // indirect\n\tgithub.com/theupdateframework/go-tuf v0.7.0 // indirect\n\tgithub.com/tjfoc/gmsm v1.4.1 // indirect\n\tgithub.com/transparency-dev/merkle v0.0.2 // indirect\n\tgithub.com/vbatts/tar-split v0.12.2 // indirect\n\tgithub.com/xanzy/go-gitlab v0.109.0 // indirect\n\tgithub.com/xanzy/ssh-agent v0.3.3 // indirect\n\tgithub.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect\n\tgithub.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect\n\tgithub.com/yashtewari/glob-intersection v0.2.0 // indirect\n\tgithub.com/yusufpapurcu/wmi v1.2.4 // indirect\n\tgitlab.alpinelinux.org/alpine/go v0.10.0 // indirect\n\tgo.mongodb.org/mongo-driver v1.17.2 // indirect\n\tgo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect\n\tgo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect\n\tgo.opentelemetry.io/otel v1.39.0 // indirect\n\tgo.opentelemetry.io/otel/metric v1.39.0 // indirect\n\tgo.opentelemetry.io/otel/sdk v1.39.0 // indirect\n\tgo.opentelemetry.io/otel/trace v1.39.0 // indirect\n\tgo.step.sm/crypto v0.57.0 // indirect\n\tgo.uber.org/multierr v1.11.0 // indirect\n\tgo.uber.org/zap v1.27.0 // indirect\n\tgolang.org/x/mod v0.31.0 // indirect\n\tgolang.org/x/net v0.48.0 // indirect\n\tgolang.org/x/oauth2 v0.34.0 // indirect\n\tgolang.org/x/sync v0.19.0 // indirect\n\tgolang.org/x/sys v0.39.0 // indirect\n\tgolang.org/x/term v0.38.0 // indirect\n\tgolang.org/x/text v0.32.0 // indirect\n\tgolang.org/x/time v0.13.0 // indirect\n\tgolang.org/x/tools/go/vcs v0.1.0-deprecated // indirect\n\tgolang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect\n\tgoogle.golang.org/api v0.242.0 // indirect\n\tgoogle.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect\n\tgoogle.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect\n\tgoogle.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect\n\tgoogle.golang.org/grpc v1.79.3 // indirect\n\tgoogle.golang.org/protobuf v1.36.10 // indirect\n\tgopkg.in/inf.v0 v0.9.1 // indirect\n\tgopkg.in/ini.v1 v1.67.0 // indirect\n\tgopkg.in/warnings.v0 v0.1.2 // indirect\n\tgopkg.in/yaml.v2 v2.4.0 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n\tk8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect\n\tk8s.io/release v0.18.0 // indirect\n\tk8s.io/utils v0.0.0-20260108192941-914a6e750570\n\tmodernc.org/libc v1.45.2 // indirect\n\tmodernc.org/mathutil v1.6.0 // indirect\n\tmodernc.org/memory v1.7.2 // indirect\n\tmodernc.org/sqlite v1.29.5 // indirect\n\tsigs.k8s.io/bom v0.6.0 // indirect\n\tsigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect\n\tsigs.k8s.io/promo-tools/v3 v3.6.0 // indirect\n\tsigs.k8s.io/release-sdk v0.12.2 // indirect\n\tsigs.k8s.io/release-utils v0.12.0 // indirect\n\tsigs.k8s.io/yaml v1.6.0\n)\n\nrequire (\n\tgithub.com/urfave/sflags v0.4.1\n\tgithub.com/weaveworks/eksctl v0.221.0\n\tk8s.io/cli-runtime v0.35.0\n\tk8s.io/cloud-provider-aws v1.35.0\n\tsigs.k8s.io/e2e-framework v0.6.1-0.20250909060333-8677714ff9a6 // bump version once https://github.com/kubernetes-sigs/e2e-framework/pull/517 gets released\n)\n\nrequire (\n\tcel.dev/expr v0.25.1 // indirect\n\tcloud.google.com/go/auth v0.16.5 // indirect\n\tcloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect\n\tcloud.google.com/go/monitoring v1.24.2 // indirect\n\tgithub.com/AliyunContainerService/ack-ram-tool/pkg/credentials/provider v0.14.0 // indirect\n\tgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect\n\tgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect\n\tgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect\n\tgithub.com/avast/retry-go/v4 v4.6.1 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/cloudtrail v1.55.1 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.61.1 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.15 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.2 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/kms v1.47.1 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/outposts v1.57.8 // indirect\n\tgithub.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect\n\tgithub.com/cenkalti/backoff/v5 v5.0.3 // indirect\n\tgithub.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect\n\tgithub.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect\n\tgithub.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect\n\tgithub.com/fatih/color v1.18.0 // indirect\n\tgithub.com/go-errors/errors v1.5.1 // indirect\n\tgithub.com/go-jose/go-jose/v4 v4.1.3 // indirect\n\tgithub.com/go-resty/resty/v2 v2.16.5 // indirect\n\tgithub.com/go-viper/mapstructure/v2 v2.4.0 // indirect\n\tgithub.com/google/btree v1.1.3 // indirect\n\tgithub.com/google/go-github/v60 v60.0.0 // indirect\n\tgithub.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect\n\tgithub.com/hashicorp/go-version v1.7.0 // indirect\n\tgithub.com/in-toto/attestation v1.1.0 // indirect\n\tgithub.com/kris-nova/logger v0.2.2 // indirect\n\tgithub.com/mattn/go-colorable v0.1.14 // indirect\n\tgithub.com/mattn/go-runewidth v0.0.16 // indirect\n\tgithub.com/miekg/dns v1.1.61 // indirect\n\tgithub.com/moby/spdystream v0.5.0 // indirect\n\tgithub.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect\n\tgithub.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect\n\tgithub.com/octago/sflags v0.3.1 // indirect\n\tgithub.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 // indirect\n\tgithub.com/olekukonko/ll v0.0.8 // indirect\n\tgithub.com/olekukonko/tablewriter v1.0.8 // indirect\n\tgithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect\n\tgithub.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect\n\tgithub.com/rivo/uniseg v0.4.7 // indirect\n\tgithub.com/sigstore/protobuf-specs v0.5.0 // indirect\n\tgithub.com/sigstore/sigstore-go v0.6.1 // indirect\n\tgithub.com/stretchr/objx v0.5.2 // indirect\n\tgithub.com/theupdateframework/go-tuf/v2 v2.3.1 // indirect\n\tgithub.com/vladimirvivien/gexe v0.5.0 // indirect\n\tgithub.com/xlab/treeprint v1.2.0 // indirect\n\tgo.opentelemetry.io/auto/sdk v1.2.1 // indirect\n\tgo.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect\n\tgo.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect\n\tgo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect\n\tgo.opentelemetry.io/otel/sdk/metric v1.39.0 // indirect\n\tgo.yaml.in/yaml/v2 v2.4.3 // indirect\n\tgo.yaml.in/yaml/v3 v3.0.4 // indirect\n\tgopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect\n\tgopkg.in/gcfg.v1 v1.2.3 // indirect\n\tk8s.io/cloud-provider v0.35.0 // indirect\n\tk8s.io/component-base v0.35.0 // indirect\n\tk8s.io/kubelet v0.35.0 // indirect\n\tsigs.k8s.io/kustomize/api v0.20.1 // indirect\n\tsigs.k8s.io/kustomize/kyaml v0.20.1 // indirect\n\tsigs.k8s.io/randfill v1.0.0 // indirect\n\tsigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect\n)\n" }, { "path": "go.sum", "content": "cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=\ncel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=\ncloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.121.2 h1:v2qQpN6Dx9x2NmwrqlesOt3Ys4ol5/lFZ6Mg1B7OJCg=\ncloud.google.com/go v0.121.2/go.mod h1:nRFlrHq39MNVWu+zESP2PosMWA0ryJw8KUBZ2iZpxbw=\ncloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI=\ncloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ=\ncloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=\ncloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=\ncloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=\ncloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=\ncloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8=\ncloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE=\ncloud.google.com/go/kms v1.22.0 h1:dBRIj7+GDeeEvatJeTB19oYZNV0aj6wEqSIT/7gLqtk=\ncloud.google.com/go/kms v1.22.0/go.mod h1:U7mf8Sva5jpOb4bxYZdtw/9zsbIjrklYwPcvMk34AL8=\ncloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc=\ncloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA=\ncloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE=\ncloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY=\ncloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=\ncloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=\ncloud.google.com/go/storage v1.53.0 h1:gg0ERZwL17pJ+Cz3cD2qS60w1WMDnwcm5YPAIQBHUAw=\ncloud.google.com/go/storage v1.53.0/go.mod h1:7/eO2a/srr9ImZW9k5uufcNahT2+fPb8w5it1i5boaA=\ncloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4=\ncloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI=\ncuelabs.dev/go/oci/ociregistry v0.0.0-20240404174027-a39bec0462d2 h1:BnG6pr9TTr6CYlrJznYUDj6V7xldD1W+1iXPum0wT/w=\ncuelabs.dev/go/oci/ociregistry v0.0.0-20240404174027-a39bec0462d2/go.mod h1:pK23AUVXuNzzTpfMCA06sxZGeVQ/75FdVtW249de9Uo=\ncuelang.org/go v0.9.2 h1:pfNiry2PdRBr02G/aKm5k2vhzmqbAOoaB4WurmEbWvs=\ncuelang.org/go v0.9.2/go.mod h1:qpAYsLOf7gTM1YdEg6cxh553uZ4q9ZDWlPbtZr9q1Wk=\ndario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=\ndario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=\nfilippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=\nfilippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=\ngithub.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d h1:zjqpY4C7H15HjRPEenkS4SAn3Jy2eRRjkjZbGR30TOg=\ngithub.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d/go.mod h1:XNqJ7hv2kY++g8XEHREpi+JqZo3+0l+CH2egBVN4yqM=\ngithub.com/AliyunContainerService/ack-ram-tool/pkg/credentials/provider v0.14.0 h1:kcnfY4vljxXliXDBrA9K9lwF8IoEZ4Up6Eg9kWTIm28=\ngithub.com/AliyunContainerService/ack-ram-tool/pkg/credentials/provider v0.14.0/go.mod h1:tlqp9mUGbsP+0z3Q+c0Q5MgSdq/OMwQhm5bffR3Q3ss=\ngithub.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=\ngithub.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=\ngithub.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=\ngithub.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4=\ngithub.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=\ngithub.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=\ngithub.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.0 h1:7rKG7UmnrxX4N53TFhkYqjc+kVUZuw0fL8I3Fh+Ld9E=\ngithub.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.0/go.mod h1:Wjo+24QJVhhl/L7jy6w9yzFF2yDOf3cKECAa8ecf9vE=\ngithub.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.0 h1:eXnN9kaS8TiDwXjoie3hMRLuwdUBUMW9KRgOqB3mCaw=\ngithub.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.0/go.mod h1:XIpam8wumeZ5rVMuhdDQLMfIPDf1WO3IzrCRO3e3e3o=\ngithub.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=\ngithub.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=\ngithub.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=\ngithub.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=\ngithub.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc=\ngithub.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw=\ngithub.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs=\ngithub.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=\ngithub.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk=\ngithub.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8=\ngithub.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c=\ngithub.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk=\ngithub.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=\ngithub.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg=\ngithub.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc=\ngithub.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0=\ngithub.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=\ngithub.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=\ngithub.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=\ngithub.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=\ngithub.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=\ngithub.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=\ngithub.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=\ngithub.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=\ngithub.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=\ngithub.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=\ngithub.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI=\ngithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0=\ngithub.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=\ngithub.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=\ngithub.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=\ngithub.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=\ngithub.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=\ngithub.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=\ngithub.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=\ngithub.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=\ngithub.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=\ngithub.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=\ngithub.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=\ngithub.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY+9ef8E=\ngithub.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE=\ngithub.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=\ngithub.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU=\ngithub.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=\ngithub.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=\ngithub.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.2/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc=\ngithub.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 h1:iC9YFYKDGEy3n/FtqJnOkZsene9olVspKmkX5A2YBEo=\ngithub.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc=\ngithub.com/alibabacloud-go/cr-20160607 v1.0.1 h1:WEnP1iPFKJU74ryUKh/YDPHoxMZawqlPajOymyNAkts=\ngithub.com/alibabacloud-go/cr-20160607 v1.0.1/go.mod h1:QHeKZtZ3F3FOE+/uIXCBAp8POwnUYekpLwr1dtQa5r0=\ngithub.com/alibabacloud-go/cr-20181201 v1.0.10 h1:B60f6S1imsgn2fgC6X6FrVNrONDrbCT0NwYhsJ0C9/c=\ngithub.com/alibabacloud-go/cr-20181201 v1.0.10/go.mod h1:VN9orB/w5G20FjytoSpZROqu9ZqxwycASmGqYUJSoDc=\ngithub.com/alibabacloud-go/darabonba-openapi v0.1.12/go.mod h1:sTAjsFJmVsmcVeklL9d9uDBlFsgl43wZ6jhI6BHqHqU=\ngithub.com/alibabacloud-go/darabonba-openapi v0.1.14/go.mod h1:w4CosR7O/kapCtEEMBm3JsQqWBU/CnZ2o0pHorsTWDI=\ngithub.com/alibabacloud-go/darabonba-openapi v0.2.1 h1:WyzxxKvhdVDlwpAMOHgAiCJ+NXa6g5ZWPFEzaK/ewwY=\ngithub.com/alibabacloud-go/darabonba-openapi v0.2.1/go.mod h1:zXOqLbpIqq543oioL9IuuZYOQgHQ5B8/n5OPrnko8aY=\ngithub.com/alibabacloud-go/darabonba-string v1.0.0/go.mod h1:93cTfV3vuPhhEwGGpKKqhVW4jLe7tDpo3LUM0i0g6mA=\ngithub.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6pb/Qy8c+lqua8cFpEy7g39NRRqOWc3rOwAy8m5Y2BY=\ngithub.com/alibabacloud-go/debug v1.0.0 h1:3eIEQWfay1fB24PQIEzXAswlVJtdQok8f3EVN5VrBnA=\ngithub.com/alibabacloud-go/debug v1.0.0/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc=\ngithub.com/alibabacloud-go/endpoint-util v1.1.0/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE=\ngithub.com/alibabacloud-go/endpoint-util v1.1.1 h1:ZkBv2/jnghxtU0p+upSU0GGzW1VL9GQdZO3mcSUTUy8=\ngithub.com/alibabacloud-go/endpoint-util v1.1.1/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE=\ngithub.com/alibabacloud-go/openapi-util v0.0.9/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws=\ngithub.com/alibabacloud-go/openapi-util v0.0.10/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws=\ngithub.com/alibabacloud-go/openapi-util v0.0.11/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws=\ngithub.com/alibabacloud-go/openapi-util v0.1.0 h1:0z75cIULkDrdEhkLWgi9tnLe+KhAFE/r5Pb3312/eAY=\ngithub.com/alibabacloud-go/openapi-util v0.1.0/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws=\ngithub.com/alibabacloud-go/tea v1.1.0/go.mod h1:IkGyUSX4Ba1V+k4pCtJUc6jDpZLFph9QMy2VUPTwukg=\ngithub.com/alibabacloud-go/tea v1.1.7/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4=\ngithub.com/alibabacloud-go/tea v1.1.8/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4=\ngithub.com/alibabacloud-go/tea v1.1.11/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4=\ngithub.com/alibabacloud-go/tea v1.1.17/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A=\ngithub.com/alibabacloud-go/tea v1.1.19/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A=\ngithub.com/alibabacloud-go/tea v1.2.2 h1:aTsR6Rl3ANWPfqeQugPglfurloyBJY85eFy7Gc1+8oU=\ngithub.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk=\ngithub.com/alibabacloud-go/tea-utils v1.3.1/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE=\ngithub.com/alibabacloud-go/tea-utils v1.3.9/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE=\ngithub.com/alibabacloud-go/tea-utils v1.4.3/go.mod h1:KNcT0oXlZZxOXINnZBs6YvgOd5aYp9U67G+E3R8fcQw=\ngithub.com/alibabacloud-go/tea-utils v1.4.5 h1:h0/6Xd2f3bPE4XHTvkpjwxowIwRCJAJOqY6Eq8f3zfA=\ngithub.com/alibabacloud-go/tea-utils v1.4.5/go.mod h1:KNcT0oXlZZxOXINnZBs6YvgOd5aYp9U67G+E3R8fcQw=\ngithub.com/alibabacloud-go/tea-xml v1.1.2/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8=\ngithub.com/alibabacloud-go/tea-xml v1.1.3 h1:7LYnm+JbOq2B+T/B0fHC4Ies4/FofC4zHzYtqw7dgt0=\ngithub.com/alibabacloud-go/tea-xml v1.1.3/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8=\ngithub.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw=\ngithub.com/aliyun/credentials-go v1.3.2 h1:L4WppI9rctC8PdlMgyTkF8bBsy9pyKQEzBD1bHMRl+g=\ngithub.com/aliyun/credentials-go v1.3.2/go.mod h1:tlpz4uys4Rn7Ik4/piGRrTbXy2uLKvePgQJJduE+Y5c=\ngithub.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=\ngithub.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=\ngithub.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=\ngithub.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=\ngithub.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=\ngithub.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=\ngithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=\ngithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=\ngithub.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=\ngithub.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=\ngithub.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=\ngithub.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=\ngithub.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk=\ngithub.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA=\ngithub.com/aws/amazon-ec2-instance-selector/v3 v3.1.2 h1:F8GBspJo+RmR4rYyw75XywEEQHQxBbF7QYKaMMnYREc=\ngithub.com/aws/amazon-ec2-instance-selector/v3 v3.1.2/go.mod h1:wdlMRtz9G4IO6H1yZPsqfGBxR8E6B/bdxHlGkls4kGQ=\ngithub.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=\ngithub.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=\ngithub.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=\ngithub.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=\ngithub.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=\ngithub.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=\ngithub.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=\ngithub.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=\ngithub.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8=\ngithub.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=\ngithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=\ngithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=\ngithub.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=\ngithub.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=\ngithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=\ngithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=\ngithub.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=\ngithub.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=\ngithub.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=\ngithub.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=\ngithub.com/aws/aws-sdk-go-v2/service/autoscaling v1.62.5 h1:3maqUQlVW7C6zAdSknv6V/LInH/RJaDW0kTFcy7dkOw=\ngithub.com/aws/aws-sdk-go-v2/service/autoscaling v1.62.5/go.mod h1:8O5Pj92iNpfw/Fa7WdHbn6YiEjDoVdutz+9PGRNoP3Y=\ngithub.com/aws/aws-sdk-go-v2/service/cloudformation v1.71.5 h1:UNllAzfiRvz9il9s0yHJkySMJbxWqEVDfyLdDblnuT4=\ngithub.com/aws/aws-sdk-go-v2/service/cloudformation v1.71.5/go.mod h1:d6XSvIZM3pSKyXNbezwYT3nAcJeUzsJIXtZMNuQ9K2k=\ngithub.com/aws/aws-sdk-go-v2/service/cloudtrail v1.55.1 h1:fRFvc/mgSPujB9JrKuPt+HGnJE9I+nDwXMhEAwHI/GM=\ngithub.com/aws/aws-sdk-go-v2/service/cloudtrail v1.55.1/go.mod h1:XSNDmicqamWtX6yg5lisFAiFaf56PErQo/cMQvUQWX0=\ngithub.com/aws/aws-sdk-go-v2/service/cloudwatch v1.53.1 h1:ElB5x0nrBHgQs+XcpQ1XJpSJzMFCq6fDTpT6WQCWOtQ=\ngithub.com/aws/aws-sdk-go-v2/service/cloudwatch v1.53.1/go.mod h1:Cj+LUEvAU073qB2jInKV6Y0nvHX0k7bL7KAga9zZ3jw=\ngithub.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.61.1 h1:1Ci283hJE+S3XC4n5b2peV/wlcAo5rTVDb6j6JJ1aTo=\ngithub.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.61.1/go.mod h1:WXcA3mYRgWVIzjD+kxzap0axltmt4zBVDZaRX0S86gk=\ngithub.com/aws/aws-sdk-go-v2/service/ec2 v1.279.1 h1:hnNVFVOYrzJjkqI+mxc1M4ztgcVw986n0t0TCPlnDPY=\ngithub.com/aws/aws-sdk-go-v2/service/ec2 v1.279.1/go.mod h1:Uy+C+Sc58jozdoL1McQr8bDsEvNFx+/nBY+vpO1HVUY=\ngithub.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 h1:VDQaVwGOokbd3VUbHF+wupiffdrbAZPdQnr5XZMJqrs=\ngithub.com/aws/aws-sdk-go-v2/service/ecr v1.36.2/go.mod h1:lvUlMghKYmSxSfv0vU7pdU/8jSY+s0zpG8xXhaGKCw0=\ngithub.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2 h1:Zru9Iy2JPM5+uRnFnoqeOZzi8JIVIHJ0ua6JdeDHcyg=\ngithub.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2/go.mod h1:PtQC3XjutCYFCn1+i8+wtpDaXvEK+vXF2gyLIKAmh4A=\ngithub.com/aws/aws-sdk-go-v2/service/eks v1.76.4 h1:5f9jIMcEd0wvRpEoo925Ltfw/2Yalcf+amFm3e1tRd8=\ngithub.com/aws/aws-sdk-go-v2/service/eks v1.76.4/go.mod h1:Qg678m+87sCuJhcsZojenz8mblYG+Tq86V4m3hjVz0s=\ngithub.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.15 h1:dJtNm4/eMx8nczyN3P4iAARXMj2rAvOJnj608zCqCmw=\ngithub.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.15/go.mod h1:QEbuU4eh8HGdv4uvld0Jth+KW8L0lOSYlyPcW6+JJo8=\ngithub.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.2 h1:xJkfrBzq4b4JxnxwNNzjUKmbQj1hPa4uUikSeXQFBYk=\ngithub.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.2/go.mod h1:DpGMmFhQwV/HH9zugLT5Ovf9HMKdQ+6ejfJybqEC9i4=\ngithub.com/aws/aws-sdk-go-v2/service/iam v1.53.2 h1:62G6btFUwAa5uR5iPlnlNVAM0zJSLbWgDfKOfUC7oW4=\ngithub.com/aws/aws-sdk-go-v2/service/iam v1.53.2/go.mod h1:av9clChrbZbJ5E21msSsiT2oghl2BJHfQGhCkXmhyu8=\ngithub.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=\ngithub.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=\ngithub.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=\ngithub.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=\ngithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=\ngithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=\ngithub.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=\ngithub.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=\ngithub.com/aws/aws-sdk-go-v2/service/kms v1.47.1 h1:6+C0RoGF4HJQALrsecOXN7cm/l5rgNHCw2xbcvFgpH4=\ngithub.com/aws/aws-sdk-go-v2/service/kms v1.47.1/go.mod h1:VJcNH6BLr+3VJwinRKdotLOMglHO8mIKlD3ea5c7hbw=\ngithub.com/aws/aws-sdk-go-v2/service/outposts v1.57.8 h1:zB9Q/dG0NkURC5E1g4qL/lsUp7aOqilfb7Ru9EOigDU=\ngithub.com/aws/aws-sdk-go-v2/service/outposts v1.57.8/go.mod h1:3osURGv9q/2wxP1qYnB15GWYgr6w2AbQkSxYtE6vTaY=\ngithub.com/aws/aws-sdk-go-v2/service/pricing v1.34.3 h1:vAv0hi3SWcc8cotkWRP4mPkmRbp/XqWKFyPW4Nwpzv0=\ngithub.com/aws/aws-sdk-go-v2/service/pricing v1.34.3/go.mod h1:giTP9ufzBQJRB6bc7P30PO8s35hCp6au5uM70zkohU4=\ngithub.com/aws/aws-sdk-go-v2/service/s3 v1.95.1 h1:C2dUPSnEpy4voWFIq3JNd8gN0Y5vYGDo44eUE58a/p8=\ngithub.com/aws/aws-sdk-go-v2/service/s3 v1.95.1/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=\ngithub.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=\ngithub.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=\ngithub.com/aws/aws-sdk-go-v2/service/ssm v1.67.8 h1:31Llf5VfrZ78YvYs7sWcS7L2m3waikzRc6q1nYenVS4=\ngithub.com/aws/aws-sdk-go-v2/service/ssm v1.67.8/go.mod h1:/jgaDlU1UImoxTxhRNxXHvBAPqPZQ8oCjcPbbkR6kac=\ngithub.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=\ngithub.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=\ngithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=\ngithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=\ngithub.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=\ngithub.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=\ngithub.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=\ngithub.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=\ngithub.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240318154307-a1a918375412 h1:tfbmGNeOidVXzO1I7zo/WsT5QX7Aa0BGTbnEAE4FG3E=\ngithub.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20240318154307-a1a918375412/go.mod h1:kcUkjB9HwuV7PSck2b60kJtgDy+eTHWuAP0kb93FXsk=\ngithub.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20251001043626-89ce6578d960 h1:F/q1AN14KuY3I6HyEJxEUuQmEo5cDRpbXptP7UlB8GQ=\ngithub.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20251001043626-89ce6578d960/go.mod h1:cOBzmLe5lF+1C3h0SNnbl2LvMi+Gm8EXGlPxdXoucio=\ngithub.com/awslabs/operatorpkg v0.0.0-20250909182303-e8e550b6f339 h1:p4oSlQ9IaT7/DHfgcrs9zdNhdIp37VIMujZLuxSgECk=\ngithub.com/awslabs/operatorpkg v0.0.0-20250909182303-e8e550b6f339/go.mod h1:tNmCf0qIjaGbODGbm3DM8GIKBUvvxM7iW3KHbpSnVgw=\ngithub.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=\ngithub.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=\ngithub.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=\ngithub.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=\ngithub.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=\ngithub.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=\ngithub.com/buildkite/agent/v3 v3.81.0 h1:JVfkng2XnsXesFXwiFwLJFkuzVu4zvoJCvedfoIXD6E=\ngithub.com/buildkite/agent/v3 v3.81.0/go.mod h1:edJeyycODRxaFvpT22rDGwaQ5oa4eB8GjtbjgX5VpFw=\ngithub.com/buildkite/go-pipeline v0.13.1 h1:Y9p8pQIwPtauVwNrcmTDH6+XK7jE1nLuvWVaK8oymA8=\ngithub.com/buildkite/go-pipeline v0.13.1/go.mod h1:2HHqlSFTYgHFhzedJu0LhLs9n5c9XkYnHiQFVN5HE4U=\ngithub.com/buildkite/interpolate v0.1.3 h1:OFEhqji1rNTRg0u9DsSodg63sjJQEb1uWbENq9fUOBM=\ngithub.com/buildkite/interpolate v0.1.3/go.mod h1:UNVe6A+UfiBNKbhAySrBbZFZFxQ+DXr9nWen6WVt/A8=\ngithub.com/buildkite/roko v1.2.0 h1:hbNURz//dQqNl6Eo9awjQOVOZwSDJ8VEbBDxSfT9rGQ=\ngithub.com/buildkite/roko v1.2.0/go.mod h1:23R9e6nHxgedznkwwfmqZ6+0VJZJZ2Sg/uVcp2cP46I=\ngithub.com/bytecodealliance/wasmtime-go/v3 v3.0.2 h1:3uZCA/BLTIu+DqCfguByNMJa2HVHpXvjfy0Dy7g6fuA=\ngithub.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q=\ngithub.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=\ngithub.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=\ngithub.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=\ngithub.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=\ngithub.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE=\ngithub.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU=\ngithub.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=\ngithub.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=\ngithub.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs=\ngithub.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=\ngithub.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=\ngithub.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=\ngithub.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ=\ngithub.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE=\ngithub.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8=\ngithub.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=\ngithub.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=\ngithub.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=\ngithub.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4=\ngithub.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM=\ngithub.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=\ngithub.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=\ngithub.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=\ngithub.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=\ngithub.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=\ngithub.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/cloudflare/cfssl v1.6.5 h1:46zpNkm6dlNkMZH/wMW22ejih6gIaJbzL2du6vD7ZeI=\ngithub.com/cloudflare/cfssl v1.6.5/go.mod h1:Bk1si7sq8h2+yVEDrFJiz3d7Aw+pfjjJSZVaD+Taky4=\ngithub.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=\ngithub.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=\ngithub.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=\ngithub.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w=\ngithub.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=\ngithub.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=\ngithub.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=\ngithub.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=\ngithub.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=\ngithub.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=\ngithub.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=\ngithub.com/containerd/stargz-snapshotter/estargz v0.18.1 h1:cy2/lpgBXDA3cDKSyEfNOFMA/c10O1axL69EU7iirO8=\ngithub.com/containerd/stargz-snapshotter/estargz v0.18.1/go.mod h1:ALIEqa7B6oVDsrF37GkGN20SuvG/pIMm7FwP7ZmRb0Q=\ngithub.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc=\ngithub.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=\ngithub.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f h1:eHnXnuK47UlSTOQexbzxAZfekVz6i+LKRdj1CU5DPaM=\ngithub.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=\ngithub.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=\ngithub.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=\ngithub.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0=\ngithub.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY=\ngithub.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936/go.mod h1:ttKPnOepYt4LLzD+loXQ1rT6EmpyIYHro7TAJuIIlHo=\ngithub.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y=\ngithub.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA=\ngithub.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM=\ngithub.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=\ngithub.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo=\ngithub.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=\ngithub.com/digitorus/pkcs7 v0.0.0-20230713084857-e76b763bdc49/go.mod h1:SKVExuS+vpu2l9IoOc0RwqE7NYnb0JlcFHFnEJkVDzc=\ngithub.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 h1:ge14PCmCvPjpMQMIAH7uKg0lrtNSOdpYsRXlwk3QbaE=\ngithub.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352/go.mod h1:SKVExuS+vpu2l9IoOc0RwqE7NYnb0JlcFHFnEJkVDzc=\ngithub.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 h1:lxmTCgmHE1GUYL7P0MlNa00M67axePTq+9nBSGddR8I=\ngithub.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7/go.mod h1:GvWntX9qiTlOud0WkQ6ewFm0LPy5JUR1Xo0Ngbd1w6Y=\ngithub.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=\ngithub.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=\ngithub.com/docker/cli v29.0.3+incompatible h1:8J+PZIcF2xLd6h5sHPsp5pvvJA+Sr2wGQxHkRl53a1E=\ngithub.com/docker/cli v29.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=\ngithub.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=\ngithub.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=\ngithub.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8=\ngithub.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo=\ngithub.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=\ngithub.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=\ngithub.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=\ngithub.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=\ngithub.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=\ngithub.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=\ngithub.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY=\ngithub.com/emicklei/proto v1.13.2/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=\ngithub.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=\ngithub.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=\ngithub.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=\ngithub.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=\ngithub.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU=\ngithub.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g=\ngithub.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=\ngithub.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=\ngithub.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4=\ngithub.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA=\ngithub.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=\ngithub.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=\ngithub.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8=\ngithub.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=\ngithub.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=\ngithub.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=\ngithub.com/evertras/bubble-table v0.17.1 h1:HJwq3iQrZulXDE93ZcqJNiUVQCBbN4IJ2CkB/IxO3kk=\ngithub.com/evertras/bubble-table v0.17.1/go.mod h1:ifHujS1YxwnYSOgcR2+m3GnJ84f7CVU/4kUOxUCjEbQ=\ngithub.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=\ngithub.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=\ngithub.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=\ngithub.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=\ngithub.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=\ngithub.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=\ngithub.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=\ngithub.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI=\ngithub.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=\ngithub.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=\ngithub.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=\ngithub.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=\ngithub.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=\ngithub.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=\ngithub.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=\ngithub.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=\ngithub.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=\ngithub.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=\ngithub.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=\ngithub.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc=\ngithub.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=\ngithub.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=\ngithub.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=\ngithub.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=\ngithub.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=\ngithub.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=\ngithub.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=\ngithub.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=\ngithub.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0=\ngithub.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY=\ngithub.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=\ngithub.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=\ngithub.com/go-git/go-git/v5 v5.17.1 h1:WnljyxIzSj9BRRUlnmAU35ohDsjRK0EKmL0evDqi5Jk=\ngithub.com/go-git/go-git/v5 v5.17.1/go.mod h1:pW/VmeqkanRFqR6AljLcs7EA7FbZaN5MQqO7oZADXpo=\ngithub.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=\ngithub.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=\ngithub.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=\ngithub.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=\ngithub.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=\ngithub.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=\ngithub.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=\ngithub.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=\ngithub.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=\ngithub.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=\ngithub.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=\ngithub.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=\ngithub.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=\ngithub.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=\ngithub.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=\ngithub.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=\ngithub.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=\ngithub.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU=\ngithub.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo=\ngithub.com/go-openapi/errors v0.22.1 h1:kslMRRnK7NCb/CvR1q1VWuEQCEIsBGn5GgKD9e+HYhU=\ngithub.com/go-openapi/errors v0.22.1/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0=\ngithub.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=\ngithub.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=\ngithub.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=\ngithub.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=\ngithub.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco=\ngithub.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs=\ngithub.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ=\ngithub.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc=\ngithub.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=\ngithub.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=\ngithub.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c=\ngithub.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4=\ngithub.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=\ngithub.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=\ngithub.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58=\ngithub.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ=\ngithub.com/go-piv/piv-go v1.11.0 h1:5vAaCdRTFSIW4PeqMbnsDlUZ7odMYWnHBDGdmtU/Zhg=\ngithub.com/go-piv/piv-go v1.11.0/go.mod h1:NZ2zmjVkfFaL/CF8cVQ/pXdXtuj110zEKGdJM6fJZZM=\ngithub.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=\ngithub.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=\ngithub.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=\ngithub.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=\ngithub.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=\ngithub.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=\ngithub.com/go-sql-driver/mysql v1.9.1 h1:FrjNGn/BsJQjVRuSa8CBrM5BWA9BWoXXat3KrtSb/iI=\ngithub.com/go-sql-driver/mysql v1.9.1/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=\ngithub.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=\ngithub.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=\ngithub.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=\ngithub.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=\ngithub.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=\ngithub.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=\ngithub.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=\ngithub.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=\ngithub.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=\ngithub.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=\ngithub.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=\ngithub.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=\ngithub.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=\ngithub.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=\ngithub.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=\ngithub.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=\ngithub.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=\ngithub.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=\ngithub.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=\ngithub.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=\ngithub.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=\ngithub.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=\ngithub.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=\ngithub.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=\ngithub.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=\ngithub.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=\ngithub.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=\ngithub.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=\ngithub.com/google/certificate-transparency-go v1.3.2-0.20250507091337-0eddb39e94f8 h1:1RSWsOSxq2gk4pD/63bhsPwoOXgz2yXVadxXPbwZ0ec=\ngithub.com/google/certificate-transparency-go v1.3.2-0.20250507091337-0eddb39e94f8/go.mod h1:6Rm5w0Mlv87LyBNOCgfKYjdIBBpF42XpXGsbQvQGomQ=\ngithub.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=\ngithub.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=\ngithub.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=\ngithub.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=\ngithub.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=\ngithub.com/google/go-containerregistry v0.20.7 h1:24VGNpS0IwrOZ2ms2P1QE3Xa5X9p4phx0aUgzYzHW6I=\ngithub.com/google/go-containerregistry v0.20.7/go.mod h1:Lx5LCZQjLH1QBaMPeGwsME9biPeo1lPx6lbGj/UmzgM=\ngithub.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=\ngithub.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=\ngithub.com/google/go-github/v60 v60.0.0 h1:oLG98PsLauFvvu4D/YPxq374jhSxFYdzQGNCyONLfn8=\ngithub.com/google/go-github/v60 v60.0.0/go.mod h1:ByhX2dP9XT9o/ll2yXAu2VD8l5eNVg8hD4Cr0S/LmQk=\ngithub.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=\ngithub.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea h1:VcIYpAGBae3Z6BVncE0OnTE/ZjlDXqtYhOZky88neLM=\ngithub.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA=\ngithub.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM=\ngithub.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=\ngithub.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=\ngithub.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=\ngithub.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY=\ngithub.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=\ngithub.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=\ngithub.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=\ngithub.com/google/tink/go v1.7.0 h1:6Eox8zONGebBFcCBqkVmt60LaWZa6xg1cl/DwAh/J1w=\ngithub.com/google/tink/go v1.7.0/go.mod h1:GAUOd+QE3pgj9q8VKIGTCP33c/B7eb4NhxLcgTJZStM=\ngithub.com/google/trillian v1.7.1 h1:+zX8jLM3524bAMPS+VxaDIDgsMv3/ty6DuLWerHXcek=\ngithub.com/google/trillian v1.7.1/go.mod h1:E1UMAHqpZCA8AQdrKdWmHmtUfSeiD0sDWD1cv00Xa+c=\ngithub.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=\ngithub.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=\ngithub.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=\ngithub.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=\ngithub.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=\ngithub.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=\ngithub.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=\ngithub.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=\ngithub.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=\ngithub.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=\ngithub.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=\ngithub.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=\ngithub.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=\ngithub.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=\ngithub.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=\ngithub.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=\ngithub.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=\ngithub.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=\ngithub.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=\ngithub.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=\ngithub.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=\ngithub.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=\ngithub.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=\ngithub.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=\ngithub.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs=\ngithub.com/hashicorp/go-secure-stdlib/parseutil v0.1.7/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8=\ngithub.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=\ngithub.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=\ngithub.com/hashicorp/go-sockaddr v1.0.5 h1:dvk7TIXCZpmfOlM+9mlcrWmWjw/wlKT+VDq2wMvfPJU=\ngithub.com/hashicorp/go-sockaddr v1.0.5/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3lyjsj6+CCykpaxI=\ngithub.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=\ngithub.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=\ngithub.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=\ngithub.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=\ngithub.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=\ngithub.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM=\ngithub.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=\ngithub.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/pj5DA=\ngithub.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8=\ngithub.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM=\ngithub.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs=\ngithub.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=\ngithub.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=\ngithub.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=\ngithub.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=\ngithub.com/in-toto/attestation v1.1.0 h1:oRWzfmZPDSctChD0VaQV7MJrywKOzyNrtpENQFq//2Q=\ngithub.com/in-toto/attestation v1.1.0/go.mod h1:DB59ytd3z7cIHgXxwpSX2SABrU6WJUKg/grpdgHVgVs=\ngithub.com/in-toto/in-toto-golang v0.9.0 h1:tHny7ac4KgtsfrG6ybU8gVOZux2H8jN05AXJ9EBM1XU=\ngithub.com/in-toto/in-toto-golang v0.9.0/go.mod h1:xsBVrVsHNsB61++S6Dy2vWosKhuA3lUTQd+eF9HdeMo=\ngithub.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=\ngithub.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=\ngithub.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=\ngithub.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=\ngithub.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=\ngithub.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=\ngithub.com/jackc/pgx/v5 v5.7.4 h1:9wKznZrhWa2QiHL+NjTSPP6yjl3451BX3imWDnokYlg=\ngithub.com/jackc/pgx/v5 v5.7.4/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=\ngithub.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=\ngithub.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=\ngithub.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=\ngithub.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=\ngithub.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY=\ngithub.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E=\ngithub.com/jellydator/ttlcache/v3 v3.3.0 h1:BdoC9cE81qXfrxeb9eoJi9dWrdhSuwXMAnHTbnBm4Wc=\ngithub.com/jellydator/ttlcache/v3 v3.3.0/go.mod h1:bj2/e0l4jRnQdrnSTaGTsh4GSXvMjQcy41i7th0GVGw=\ngithub.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY=\ngithub.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=\ngithub.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=\ngithub.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=\ngithub.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=\ngithub.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=\ngithub.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=\ngithub.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=\ngithub.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=\ngithub.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=\ngithub.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=\ngithub.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=\ngithub.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=\ngithub.com/knqyf263/go-rpmdb v0.1.0 h1:pOgjtOGtW0B+ibY905hP3ETrYFmLZsHiReKsplcs+to=\ngithub.com/knqyf263/go-rpmdb v0.1.0/go.mod h1:9LQcoMCMQ9vrF7HcDtXfvqGO4+ddxFQ8+YF/0CVGDww=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=\ngithub.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/kris-nova/logger v0.2.2 h1:qdWg2fNr4Bni4obkgehwOSbCoxaX+wDGGrzQ1T2mA20=\ngithub.com/kris-nova/logger v0.2.2/go.mod h1:uOTzfb9ssx0XYb3UpeAjKsys8KByjD12OMN4szmym4w=\ngithub.com/kris-nova/lolgopher v0.0.0-20210112022122-73f0047e8b65/go.mod h1:V0HF/ZBlN86HqewcDC/cVxMmYDiRukWjSrgKLUAn9Js=\ngithub.com/kubicorn/kubicorn v0.0.0-20191114212505-a2c64ce430b9 h1:HgzA4yC4kPQfNIya55o4yA1WiKCXXA5wXvwoBKgIwXI=\ngithub.com/kubicorn/kubicorn v0.0.0-20191114212505-a2c64ce430b9/go.mod h1:Z/PU7XQicaZV6QFTAvm8EaWyfNbAb4a76kmR4Am4KA8=\ngithub.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=\ngithub.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=\ngithub.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=\ngithub.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=\ngithub.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=\ngithub.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=\ngithub.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=\ngithub.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=\ngithub.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=\ngithub.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=\ngithub.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=\ngithub.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=\ngithub.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=\ngithub.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=\ngithub.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=\ngithub.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=\ngithub.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=\ngithub.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=\ngithub.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=\ngithub.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=\ngithub.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=\ngithub.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=\ngithub.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=\ngithub.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=\ngithub.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=\ngithub.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=\ngithub.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=\ngithub.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=\ngithub.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=\ngithub.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=\ngithub.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=\ngithub.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=\ngithub.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=\ngithub.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU=\ngithub.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=\ngithub.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=\ngithub.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=\ngithub.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=\ngithub.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=\ngithub.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=\ngithub.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=\ngithub.com/mozillazg/docker-credential-acr-helper v0.4.0 h1:Uoh3Z9CcpEDnLiozDx+D7oDgRq7X+R296vAqAumnOcw=\ngithub.com/mozillazg/docker-credential-acr-helper v0.4.0/go.mod h1:2kiicb3OlPytmlNC9XGkLvVC+f0qTiJw3f/mhmeeQBg=\ngithub.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=\ngithub.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=\ngithub.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=\ngithub.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=\ngithub.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=\ngithub.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=\ngithub.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=\ngithub.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=\ngithub.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=\ngithub.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=\ngithub.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=\ngithub.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=\ngithub.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 h1:Up6+btDp321ZG5/zdSLo48H9Iaq0UQGthrhWC6pCxzE=\ngithub.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481/go.mod h1:yKZQO8QE2bHlgozqWDiRVqTFlLQSj30K/6SAK8EeYFw=\ngithub.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=\ngithub.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=\ngithub.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=\ngithub.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=\ngithub.com/octago/sflags v0.3.1 h1:LW65z20iAQKteEyjsnnc+/lyoCUnIoRuAocggr6RB6A=\ngithub.com/octago/sflags v0.3.1/go.mod h1:hVUkbnYwMU9kZiZJyOAIVN56YiVMMPxgJ46kRZ19jh0=\ngithub.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=\ngithub.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=\ngithub.com/oleiade/reflections v1.1.0 h1:D+I/UsXQB4esMathlt0kkZRJZdUDmhv5zGi/HOwYTWo=\ngithub.com/oleiade/reflections v1.1.0/go.mod h1:mCxx0QseeVCHs5Um5HhJeCKVC7AwS8kO67tky4rdisA=\ngithub.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6 h1:r3FaAI0NZK3hSmtTDrBVREhKULp8oUeqLT5Eyl2mSPo=\ngithub.com/olekukonko/errors v0.0.0-20250405072817-4e6d85265da6/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=\ngithub.com/olekukonko/ll v0.0.8 h1:sbGZ1Fx4QxJXEqL/6IG8GEFnYojUSQ45dJVwN2FH2fc=\ngithub.com/olekukonko/ll v0.0.8/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g=\ngithub.com/olekukonko/tablewriter v1.0.8 h1:f6wJzHg4QUtJdvrVPKco4QTrAylgaU0+b9br/lJxEiQ=\ngithub.com/olekukonko/tablewriter v1.0.8/go.mod h1:H428M+HzoUXC6JU2Abj9IT9ooRmdq9CxuDmKMtrOCMs=\ngithub.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 h1:Yl0tPBa8QPjGmesFh1D0rDy+q1Twx6FyU7VWHi8wZbI=\ngithub.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852/go.mod h1:eqOVx5Vwu4gd2mmMZvVZsgIqNSaW3xxRThUJ0k/TPk4=\ngithub.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=\ngithub.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=\ngithub.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=\ngithub.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=\ngithub.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=\ngithub.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=\ngithub.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=\ngithub.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=\ngithub.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=\ngithub.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=\ngithub.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=\ngithub.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=\ngithub.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=\ngithub.com/open-policy-agent/opa v1.4.0 h1:IGO3xt5HhQKQq2axfa9memIFx5lCyaBlG+fXcgHpd3A=\ngithub.com/open-policy-agent/opa v1.4.0/go.mod h1:DNzZPKqKh4U0n0ANxcCVlw8lCSv2c+h5G/3QvSYdWZ8=\ngithub.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=\ngithub.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=\ngithub.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=\ngithub.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=\ngithub.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=\ngithub.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=\ngithub.com/package-url/packageurl-go v0.1.2 h1:0H2DQt6DHd/NeRlVwW4EZ4oEI6Bn40XlNPRqegcxuo4=\ngithub.com/package-url/packageurl-go v0.1.2/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c=\ngithub.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=\ngithub.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=\ngithub.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=\ngithub.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=\ngithub.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=\ngithub.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=\ngithub.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=\ngithub.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=\ngithub.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4=\ngithub.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A=\ngithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=\ngithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=\ngithub.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=\ngithub.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=\ngithub.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=\ngithub.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=\ngithub.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=\ngithub.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=\ngithub.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=\ngithub.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=\ngithub.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=\ngithub.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=\ngithub.com/protocolbuffers/txtpbfmt v0.0.0-20240116145035-ef3ab179eed6 h1:MAzmm+JtFxQwTPb1cVMLkemw2OxLy5AB/d/rxtAwGQQ=\ngithub.com/protocolbuffers/txtpbfmt v0.0.0-20240116145035-ef3ab179eed6/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c=\ngithub.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=\ngithub.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=\ngithub.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=\ngithub.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=\ngithub.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=\ngithub.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=\ngithub.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=\ngithub.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=\ngithub.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=\ngithub.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=\ngithub.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=\ngithub.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=\ngithub.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=\ngithub.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=\ngithub.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k=\ngithub.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk=\ngithub.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=\ngithub.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=\ngithub.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI=\ngithub.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=\ngithub.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b h1:jUK33OXuZP/l6babJtnLo1qsGvq6G9so9KMflGAm4YA=\ngithub.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b/go.mod h1:8458kAagoME2+LN5//WxE71ysZ3B7r22fdgb7qVmXSY=\ngithub.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522 h1:fOCp11H0yuyAt2wqlbJtbyPzSgaxHTv8uN1pMpkG1t8=\ngithub.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522/go.mod h1:tQTYKOQgxoH3v6dEmdHiz4JG+nbxWwM5fgPQUpSZqVQ=\ngithub.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A=\ngithub.com/sassoftware/relic v7.2.1+incompatible/go.mod h1:CWfAxv73/iLZ17rbyhIEq3K9hs5w6FpNMdUT//qR+zk=\ngithub.com/sassoftware/relic/v7 v7.6.2 h1:rS44Lbv9G9eXsukknS4mSjIAuuX+lMq/FnStgmZlUv4=\ngithub.com/sassoftware/relic/v7 v7.6.2/go.mod h1:kjmP0IBVkJZ6gXeAu35/KCEfca//+PKM6vTAsyDPY+k=\ngithub.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14=\ngithub.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk=\ngithub.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=\ngithub.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=\ngithub.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=\ngithub.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=\ngithub.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=\ngithub.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=\ngithub.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=\ngithub.com/sigstore/cosign/v2 v2.4.1 h1:b8UXEfJFks3hmTwyxrRNrn6racpmccUycBHxDMkEPvU=\ngithub.com/sigstore/cosign/v2 v2.4.1/go.mod h1:GvzjBeUKigI+XYnsoVQDmMAsMMc6engxztRSuxE+x9I=\ngithub.com/sigstore/fulcio v1.6.3 h1:Mvm/bP6ELHgazqZehL8TANS1maAkRoM23CRAdkM4xQI=\ngithub.com/sigstore/fulcio v1.6.3/go.mod h1:5SDgLn7BOUVLKe1DwOEX3wkWFu5qEmhUlWm+SFf0GH8=\ngithub.com/sigstore/protobuf-specs v0.5.0 h1:F8YTI65xOHw70NrvPwJ5PhAzsvTnuJMGLkA4FIkofAY=\ngithub.com/sigstore/protobuf-specs v0.5.0/go.mod h1:+gXR+38nIa2oEupqDdzg4qSBT0Os+sP7oYv6alWewWc=\ngithub.com/sigstore/rekor v1.3.9 h1:sUjRpKVh/hhgqGMs0t+TubgYsksArZ6poLEC3MsGAzU=\ngithub.com/sigstore/rekor v1.3.9/go.mod h1:xThNUhm6eNEmkJ/SiU/FVU7pLY2f380fSDZFsdDWlcM=\ngithub.com/sigstore/sigstore v1.10.3 h1:s7fBYYOzW/2Vd0nND2ZdpWySb5vRF2u9eix/NZMHJm0=\ngithub.com/sigstore/sigstore v1.10.3/go.mod h1:T26vXIkpnGEg391v3TaZ8EERcXbnjtZb/1erh5jbIQk=\ngithub.com/sigstore/sigstore-go v0.6.1 h1:tGkkv1oDIER+QYU5MrjqlttQOVDWfSkmYwMqkJhB/cg=\ngithub.com/sigstore/sigstore-go v0.6.1/go.mod h1:Xe5GHmUeACRFbomUWzVkf/xYCn8xVifb9DgqJrV2dIw=\ngithub.com/sigstore/sigstore/pkg/signature/kms/aws v1.8.12 h1:EC3UmIaa7nV9sCgSpVevmvgvTYTkMqyrRbj5ojPp7tE=\ngithub.com/sigstore/sigstore/pkg/signature/kms/aws v1.8.12/go.mod h1:aw60vs3crnQdM/DYH+yF2P0MVKtItwAX34nuaMrY7Lk=\ngithub.com/sigstore/sigstore/pkg/signature/kms/azure v1.8.12 h1:FPpliDTywSy0woLHMAdmTSZ5IS/lVBZ0dY0I+2HmnSY=\ngithub.com/sigstore/sigstore/pkg/signature/kms/azure v1.8.12/go.mod h1:NkPiz4XA0JcBSXzJUrjMj7Xi7oSTew1Ip3Zmt56mHlw=\ngithub.com/sigstore/sigstore/pkg/signature/kms/gcp v1.8.12 h1:kweBChR6M9FEvmxN3BMEcl7SNnwxTwKF7THYFKLOE5U=\ngithub.com/sigstore/sigstore/pkg/signature/kms/gcp v1.8.12/go.mod h1:6+d+A6oYt1W5OgtzgEVb21V7tAZ/C2Ihtzc5MNJbayY=\ngithub.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.8.12 h1:jvY1B9bjP+tKzdKDyuq5K7O19CG2IKzGJNTy5tuL2Gs=\ngithub.com/sigstore/sigstore/pkg/signature/kms/hashivault v1.8.12/go.mod h1:2uEeOb8xE2RC6OvzxKux1wkS39Zv8gA27z92m49xUTc=\ngithub.com/sigstore/timestamp-authority v1.2.2 h1:X4qyutnCQqJ0apMewFyx+3t7Tws00JQ/JonBiu3QvLE=\ngithub.com/sigstore/timestamp-authority v1.2.2/go.mod h1:nEah4Eq4wpliDjlY342rXclGSO7Kb9hoRrl9tqLW13A=\ngithub.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=\ngithub.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=\ngithub.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=\ngithub.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=\ngithub.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=\ngithub.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262 h1:unQFBIznI+VYD1/1fApl1A+9VcBk+9dcqGfnePY87LY=\ngithub.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262/go.mod h1:MyOHs9Po2fbM1LHej6sBUT8ozbxmMOFG+E+rx/GSGuc=\ngithub.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=\ngithub.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=\ngithub.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=\ngithub.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=\ngithub.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=\ngithub.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=\ngithub.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=\ngithub.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE=\ngithub.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=\ngithub.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=\ngithub.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=\ngithub.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=\ngithub.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=\ngithub.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=\ngithub.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=\ngithub.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=\ngithub.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=\ngithub.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=\ngithub.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=\ngithub.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=\ngithub.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=\ngithub.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=\ngithub.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=\ngithub.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=\ngithub.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48=\ngithub.com/tchap/go-patricia/v2 v2.3.2 h1:xTHFutuitO2zqKAQ5rCROYgUb7Or/+IC3fts9/Yc7nM=\ngithub.com/tchap/go-patricia/v2 v2.3.2/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k=\ngithub.com/thales-e-security/pool v0.0.2 h1:RAPs4q2EbWsTit6tpzuvTFlgFRJ3S8Evf5gtvVDbmPg=\ngithub.com/thales-e-security/pool v0.0.2/go.mod h1:qtpMm2+thHtqhLzTwgDBj/OuNnMpupY8mv0Phz0gjhU=\ngithub.com/theupdateframework/go-tuf v0.7.0 h1:CqbQFrWo1ae3/I0UCblSbczevCCbS31Qvs5LdxRWqRI=\ngithub.com/theupdateframework/go-tuf v0.7.0/go.mod h1:uEB7WSY+7ZIugK6R1hiBMBjQftaFzn7ZCDJcp1tCUug=\ngithub.com/theupdateframework/go-tuf/v2 v2.3.1 h1:fReZUTLvPdqIL8Rd9xEKPmaxig8GIXe0kS4RSEaRfaM=\ngithub.com/theupdateframework/go-tuf/v2 v2.3.1/go.mod h1:9S0Srkf3c13FelsOyt5OyG3ZZDq9OJDA4IILavrt72Y=\ngithub.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=\ngithub.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=\ngithub.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=\ngithub.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=\ngithub.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=\ngithub.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=\ngithub.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=\ngithub.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=\ngithub.com/tink-crypto/tink-go-awskms/v2 v2.1.0 h1:N9UxlsOzu5mttdjhxkDLbzwtEecuXmlxZVo/ds7JKJI=\ngithub.com/tink-crypto/tink-go-awskms/v2 v2.1.0/go.mod h1:PxSp9GlOkKL9rlybW804uspnHuO9nbD98V/fDX4uSis=\ngithub.com/tink-crypto/tink-go-gcpkms/v2 v2.2.0 h1:3B9i6XBXNTRspfkTC0asN5W0K6GhOSgcujNiECNRNb0=\ngithub.com/tink-crypto/tink-go-gcpkms/v2 v2.2.0/go.mod h1:jY5YN2BqD/KSCHM9SqZPIpJNG/u3zwfLXHgws4x2IRw=\ngithub.com/tink-crypto/tink-go/v2 v2.5.0 h1:B8KLF6AofxdBIE4UJIaFbmoj5/1ehEtt7/MmzfI4Zpw=\ngithub.com/tink-crypto/tink-go/v2 v2.5.0/go.mod h1:2WbBA6pfNsAfBwDCggboaHeB2X29wkU8XHtGwh2YIk8=\ngithub.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w=\ngithub.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho=\ngithub.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE=\ngithub.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4=\ngithub.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A=\ngithub.com/urfave/sflags v0.4.1 h1:9BKteZiMaLlgfMm8eYbFge3eRAUsrJXs4HsCemdDl+A=\ngithub.com/urfave/sflags v0.4.1/go.mod h1:NCIz2mBC+woyrkl88PeiKAuQUKJdEre2Y4at5SreAeU=\ngithub.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4=\ngithub.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=\ngithub.com/vladimirvivien/gexe v0.5.0 h1:AWBVaYnrTsGYBktXvcO0DfWPeSiZxn6mnQ5nvL+A1/A=\ngithub.com/vladimirvivien/gexe v0.5.0/go.mod h1:3gjgTqE2c0VyHnU5UOIwk7gyNzZDGulPb/DJPgcw64E=\ngithub.com/weaveworks/eksctl v0.221.0 h1:sJEuVRU+8dia8rj/4VmB8DwKArmGhG7uwaqdUYJhqv0=\ngithub.com/weaveworks/eksctl v0.221.0/go.mod h1:fkWnFg8h/h24bl5DmyRgJIERB/7g5zqIeNgSklfeH5Q=\ngithub.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=\ngithub.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=\ngithub.com/xanzy/go-gitlab v0.109.0 h1:RcRme5w8VpLXTSTTMZdVoQWY37qTJWg+gwdQl4aAttE=\ngithub.com/xanzy/go-gitlab v0.109.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=\ngithub.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=\ngithub.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=\ngithub.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=\ngithub.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=\ngithub.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=\ngithub.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=\ngithub.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=\ngithub.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=\ngithub.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=\ngithub.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=\ngithub.com/yashtewari/glob-intersection v0.2.0 h1:8iuHdN88yYuCzCdjt0gDe+6bAhUwBeEWqThExu54RFg=\ngithub.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok=\ngithub.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=\ngithub.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=\ngithub.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=\ngithub.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=\ngithub.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q=\ngithub.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=\ngithub.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=\ngithub.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=\ngithub.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=\ngithub.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=\ngithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=\ngithub.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=\ngithub.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=\ngithub.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms=\ngithub.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk=\ngitlab.alpinelinux.org/alpine/go v0.10.0 h1:/ekBiNqDSXZpK+AfZx4lrtVwKTDrWz3N3ck0S+fCxwU=\ngitlab.alpinelinux.org/alpine/go v0.10.0/go.mod h1:LKzOqYjGTZNLwcHl+c2I5VNioQio7agzRFvlGB9Owk4=\ngo.mongodb.org/mongo-driver v1.17.2 h1:gvZyk8352qSfzyZ2UMWcpDpMSGEr1eqE4T793SqyhzM=\ngo.mongodb.org/mongo-driver v1.17.2/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=\ngo.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=\ngo.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=\ngo.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=\ngo.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=\ngo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=\ngo.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=\ngo.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=\ngo.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg=\ngo.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk=\ngo.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k=\ngo.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=\ngo.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=\ngo.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=\ngo.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=\ngo.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=\ngo.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=\ngo.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=\ngo.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=\ngo.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=\ngo.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=\ngo.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=\ngo.step.sm/crypto v0.57.0 h1:YjoRQDaJYAxHLVwjst0Bl0xcnoKzVwuHCJtEo2VSHYU=\ngo.step.sm/crypto v0.57.0/go.mod h1:+Lwp5gOVPaTa3H/Ul/TzGbxQPXZZcKIUGMS0lG6n9Go=\ngo.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=\ngo.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=\ngo.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=\ngo.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=\ngo.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=\ngo.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=\ngo.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=\ngo.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=\ngo.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=\ngo.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=\ngolang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=\ngolang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=\ngolang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=\ngolang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=\ngolang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=\ngolang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=\ngolang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=\ngolang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=\ngolang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0=\ngolang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=\ngolang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=\ngolang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=\ngolang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=\ngolang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=\ngolang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=\ngolang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=\ngolang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=\ngolang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=\ngolang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=\ngolang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=\ngolang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=\ngolang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=\ngolang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=\ngolang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=\ngolang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=\ngolang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=\ngolang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=\ngolang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=\ngolang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=\ngolang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=\ngolang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=\ngolang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=\ngolang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=\ngolang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=\ngolang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=\ngolang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=\ngolang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=\ngolang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=\ngolang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=\ngolang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=\ngolang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=\ngolang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=\ngolang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4=\ngolang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=\ngolang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk=\ngolang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=\ngomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=\ngomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=\ngonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=\ngonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=\ngoogle.golang.org/api v0.242.0 h1:7Lnb1nfnpvbkCiZek6IXKdJ0MFuAZNAJKQfA1ws62xg=\ngoogle.golang.org/api v0.242.0/go.mod h1:cOVEm2TpdAGHL2z+UwyS+kmlGr3bVWQQ6sYEqkKje50=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4=\ngoogle.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s=\ngoogle.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=\ngoogle.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=\ngoogle.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=\ngoogle.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=\ngoogle.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=\ngoogle.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=\ngoogle.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=\ngopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=\ngopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=\ngopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=\ngopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs=\ngopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=\ngopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=\ngopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=\ngopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=\ngopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=\ngopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=\ngopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=\ngotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nk8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY=\nk8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA=\nk8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI=\nk8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc=\nk8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8=\nk8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=\nk8s.io/cli-runtime v0.35.0 h1:PEJtYS/Zr4p20PfZSLCbY6YvaoLrfByd6THQzPworUE=\nk8s.io/cli-runtime v0.35.0/go.mod h1:VBRvHzosVAoVdP3XwUQn1Oqkvaa8facnokNkD7jOTMY=\nk8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE=\nk8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o=\nk8s.io/cloud-provider v0.35.0 h1:syiBCQbKh2gho/S1BkIl006Dc44pV8eAtGZmv5NMe7M=\nk8s.io/cloud-provider v0.35.0/go.mod h1:7grN+/Nt5Hf7tnSGPT3aErt4K7aQpygyCrGpbrQbzNc=\nk8s.io/cloud-provider-aws v1.35.0 h1:jlMZmc4JjJ6lkYj41xeKqZ8nw1ais00xQi8Nnz2lqkI=\nk8s.io/cloud-provider-aws v1.35.0/go.mod h1:6R9TIgQ/ecysPukSmEUs4kZIwqvju80+FjMAhtJ22Q0=\nk8s.io/component-base v0.35.0 h1:+yBrOhzri2S1BVqyVSvcM3PtPyx5GUxCK2tinZz1G94=\nk8s.io/component-base v0.35.0/go.mod h1:85SCX4UCa6SCFt6p3IKAPej7jSnF3L8EbfSyMZayJR0=\nk8s.io/component-helpers v0.35.0 h1:wcXv7HJRksgVjM4VlXJ1CNFBpyDHruRI99RrBtrJceA=\nk8s.io/component-helpers v0.35.0/go.mod h1:ahX0m/LTYmu7fL3W8zYiIwnQ/5gT28Ex4o2pymF63Co=\nk8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=\nk8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=\nk8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=\nk8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=\nk8s.io/kops v1.33.1 h1:MFrj3r6f+F9rL2DQQdfAXEyFJDdq0GAyu96woF6TOaQ=\nk8s.io/kops v1.33.1/go.mod h1:epTyN30uGaeRBmN1jmT993Kc4Wd/tti9snQDd5aivXc=\nk8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE=\nk8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ=\nk8s.io/kubelet v0.35.0 h1:8cgJHCBCKLYuuQ7/Pxb/qWbJfX1LXIw7790ce9xHq7c=\nk8s.io/kubelet v0.35.0/go.mod h1:ciRzAXn7C4z5iB7FhG1L2CGPPXLTVCABDlbXt/Zz8YA=\nk8s.io/release v0.18.0 h1:xn+ZU/8bDmtAcSZMh0K2HMa2+dYrD3Qqq+yqv3Uuk9k=\nk8s.io/release v0.18.0/go.mod h1:PJ4HhnTcmTKSakE475b4e3xJEVw+EVB5ycZM9vWFcTU=\nk8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKWdRm+BC+xt33FOY=\nk8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=\nmodernc.org/cc/v4 v4.19.3 h1:vE9kmJqUcyvNOf8F2Hn8od14SOMq34BiqcZ2tMzLk5c=\nmodernc.org/cc/v4 v4.19.3/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=\nmodernc.org/ccgo/v4 v4.11.0 h1:2uc2kRvZLC/oHylsrirRW6f1I4wljQST2BBbm+aKiXM=\nmodernc.org/ccgo/v4 v4.11.0/go.mod h1:GwrfAtnU6PdZkCWD4XI8wB1T5Xj3fSw9lO/40H1ldys=\nmodernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=\nmodernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=\nmodernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=\nmodernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=\nmodernc.org/libc v1.45.2 h1:oRlBu8xlBen2awVAWuLOkvYNBPaIKFxFOj9wA/jaXHM=\nmodernc.org/libc v1.45.2/go.mod h1:YkRHLoN4L70OdO1cVmM83KZhRbRvsc3XogfVzbTXBwE=\nmodernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=\nmodernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=\nmodernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=\nmodernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=\nmodernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=\nmodernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=\nmodernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=\nmodernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=\nmodernc.org/sqlite v1.29.5 h1:8l/SQKAjDtZFo9lkJLdk8g9JEOeYRG4/ghStDCCTiTE=\nmodernc.org/sqlite v1.29.5/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U=\nmodernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=\nmodernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=\nmodernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=\nmodernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=\nsigs.k8s.io/bom v0.6.0 h1:IPMPHx6XdmMeW2oEeF66DgNyP5d4RxfuXwiC1qn+n9o=\nsigs.k8s.io/bom v0.6.0/go.mod h1:MV0D3vdGlkaPgi5EwpwMBeQ8n8QS8Q2u1lJ5LyE7RLM=\nsigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A=\nsigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8=\nsigs.k8s.io/e2e-framework v0.6.1-0.20250909060333-8677714ff9a6 h1:5saOTCrwclRdFJLj5zDMJITisRmR0HuG8SU6ts9z5IY=\nsigs.k8s.io/e2e-framework v0.6.1-0.20250909060333-8677714ff9a6/go.mod h1:MUvWdQO9AGg4/yP9Y0kOcmX+KIOXI0UR6Xw6xz11ULw=\nsigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=\nsigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=\nsigs.k8s.io/karpenter v1.8.0 h1:AmTHUPtnuL8IX9mbcD3NOohyk62idrBCBtM+8Wn6Jvk=\nsigs.k8s.io/karpenter v1.8.0/go.mod h1:nDDVB5873dVVuyTam3oJrllSv0sAgp6as6/5HRTcV4o=\nsigs.k8s.io/kubetest2 v0.0.0-20260108084739-2f9a9397f033 h1:+HmjjgPGGqvYRBErxVSbguBnp7hILyuwHHDKUXRCDA4=\nsigs.k8s.io/kubetest2 v0.0.0-20260108084739-2f9a9397f033/go.mod h1:pBd0cFaT0hDqmwQg+TIhyLgPMYaH66QMLcKd09XnKTI=\nsigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I=\nsigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM=\nsigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78=\nsigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po=\nsigs.k8s.io/promo-tools/v3 v3.6.0 h1:C2L08ezrWm1aZI8Emd3iZPZQserLPRgzuqQVxvI0PUI=\nsigs.k8s.io/promo-tools/v3 v3.6.0/go.mod h1:XJ3jy0hJYs+hWKt8XsLHFzGQV8PUtvllvbxjN/E5RXI=\nsigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=\nsigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=\nsigs.k8s.io/release-sdk v0.12.2 h1:ncuHwUu8VWcZVVrNkjoUR8xGo6ibHg+AM6uMMD+IwuQ=\nsigs.k8s.io/release-sdk v0.12.2/go.mod h1:tlJgWPJLeRbWOvcyq1XrCZmLe8Yfn3H5U/LNtmBa0Nc=\nsigs.k8s.io/release-utils v0.12.0 h1:+Z8cEUAaxItrMcTOJ0jtUg3Fm1uNgPNol+VIL6XtQqQ=\nsigs.k8s.io/release-utils v0.12.0/go.mod h1:TveYRPK4Mq6qXA0PJiUMEOlWvvIQG0Mh5APQmHD5JpA=\nsigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=\nsigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=\nsigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=\nsigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=\nsoftware.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k=\nsoftware.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=\n" }, { "path": "hack/download-kubernetes-binaries.sh", "content": "#!/usr/bin/env bash\n\nset -o errexit\nset -o nounset\n\nBUNDLES=(\n \"kubernetes-client\"\n \"kubernetes-test\"\n)\n\nif [ \"$#\" -ne 3 ]; then\n echo >&2 \"usage: $0 (KUBERNETES_MINOR_VERSION|latest) OS ARCH\"\n exit 1\nfi\n\nif [ \"$1\" = \"latest\" ]; then\n RELEASE_MARKER=\"latest.txt\"\nelse\n RELEASE_MARKER=\"latest-$1.txt\"\nfi\n\necho >&2 \"Release marker: ${RELEASE_MARKER}\"\n\nOS=\"$2\"\nARCH=\"$3\"\n\nfunction download_binaries() {\n local basePath=$1\n\n local KUBERNETES_VERSION=$(curl --silent \"${basePath}/${RELEASE_MARKER}\")\n\n echo \"Kubernetes version: ${KUBERNETES_VERSION}\"\n echo \"${KUBERNETES_VERSION}\" > kubernetes-version.txt\n\n for BUNDLE in ${BUNDLES[@]}; do\n echo >&2 \"Downloading bundle: ${BUNDLE}\"\n local TARBALL=\"${BUNDLE}.tar.gz\"\n if ! wget --quiet --output-document=${TARBALL} $basePath/${KUBERNETES_VERSION}/${BUNDLE}-${OS}-${ARCH}.tar.gz; then\n return 1\n fi\n tar xzf ${TARBALL}\n rm ${TARBALL}\n done\n}\n\nif ! download_binaries https://storage.googleapis.com/kubernetes-release/release; then\n echo >&2 \"binary download failed from release bucket, falling back to ci dev release\"\n download_binaries https://storage.googleapis.com/k8s-release-dev/ci\nfi\n" }, { "path": "hack/free-disk-space.sh", "content": "#!/usr/bin/env bash\n\nset -o nounset\nset -o errexit\nset -o pipefail\n\n# hack to free up disk space for build\n# ref: https://github.com/easimon/maximize-build-space/blob/master/action.yml\n\n# storage before\nsudo df -h\n\nsudo rm -rf \\\n /usr/share/dotnet \\\n /usr/local/lib/android \\\n /opt/ghc \\\n /opt/hostedtoolcache/CodeQL\n\ndocker image prune --all --force\ndocker builder prune -a\n\n# storage after\nsudo df -h\n" }, { "path": "hack/update-go-dependencies.sh", "content": "#!/usr/bin/env bash\n\nset -o nounset\nset -o errexit\nset -o pipefail\n\necho \"Updating go modules...\"\ngo get $(go list -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' -mod=mod -m all) && go mod tidy\n\necho \"Updating kubetest2 image go version...\"\nMODULE_GO_VERSION=$(go list -m -f \"{{if .Main}}{{.GoVersion}}{{end}}\" | cut -d'.' -f1-2)\nfind . -type f -name Dockerfile -exec sed -i \"s/\\(GO_MINOR_VERSION\\)=.*/\\1=${MODULE_GO_VERSION}/g\" {} +\n" }, { "path": "hack/update-image-tags.sh", "content": "#!/usr/bin/env bash\n\nset -o nounset\nset -o errexit\nset -o pipefail\n\nECR_PUBLIC_REGISTRY=\"public.ecr.aws\"\nEKS_CONTAINER_REGISTRY=\"602401143452.dkr.ecr.us-west-2.amazonaws.com\"\n\n# get_ecr_image_tags \n# e.g. get_ecr_image_tags $ECR_PUBLIC_REGISTRY amazonlinux/amazonlinux\nget_ecr_image_tags() {\n set -e\n local REGISTRY=$1 \n local REPOSITORY=$2\n local TOKEN\n\n # Get ECR public token if image is from a public registry, otherwise use a private token\n # An authorization token is required for every ECR HTTP request\n if [ \"$REGISTRY\" = \"$ECR_PUBLIC_REGISTRY\" ]; then\n TOKEN=$(aws ecr-public get-authorization-token --region us-east-1 --output=text --query 'authorizationData.authorizationToken')\n local AUTHORIZATION_TYPE=\"Bearer\"\n else \n TOKEN=$(aws ecr get-authorization-token --output text --query 'authorizationData[].authorizationToken')\n local AUTHORIZATION_TYPE=\"Basic\"\n fi\n\n curl -s -H \"Authorization: ${AUTHORIZATION_TYPE} $TOKEN\" \"https://$REGISTRY/v2/$REPOSITORY/tags/list\" | jq '.tags'\n}\n\n# update_image_uris REPOSITORY IMAGE_TAG\nupdate_image_uris() {\n local REPOSITORY=$1\n local NEW_TAG=$2\n PREFIX=\"image: ${REPOSITORY}\"\n find ./test/manifests -type f -exec sed -i \"s#$PREFIX:.*#$PREFIX:$NEW_TAG#g\" {} +\n}\n\n# update the nvidia k8s device plugin\necho \"Updating Nvidia device plugin image\"\nNVIDIA_DEVICE_PLUGIN_TAG=$(curl -s 'https://catalog.ngc.nvidia.com/api/containers/images?orgName=nvidia&name=k8s-device-plugin&isPublic=true' | jq -r '.images | sort_by(.updatedDate) | reverse | map(select(.tag | test(\"^v[0-9]+.[0-9]+.[0-9]+$\"))) | first | .tag')\nupdate_image_uris nvcr.io/nvidia/k8s-device-plugin $NVIDIA_DEVICE_PLUGIN_TAG\n\n# below updates require authentication and should not exit early with a failure.\n# TODO: remove this once the aws credentials are setup and the paths are expected to succeed.\nset +e\n\n# update the neuron k8s device plugin\necho \"Updating Neuron device plugin image\"\nNEURON_DEVICE_PLUGIN_REPOSITORY_NAME=\"neuron/neuron-device-plugin\"\nNEURON_DEVICE_PLUGIN_TAGS=$(get_ecr_image_tags $ECR_PUBLIC_REGISTRY $NEURON_DEVICE_PLUGIN_REPOSITORY_NAME)\nif [ $? -eq 0 ]; then\n LATEST_NEURON_DEVICE_PLUGIN_TAG=$(echo $NEURON_DEVICE_PLUGIN_TAGS | jq -r 'max_by(split(\".\") | map(tonumber))')\n update_image_uris \"${ECR_PUBLIC_REGISTRY}/${NEURON_DEVICE_PLUGIN_REPOSITORY_NAME}\" $LATEST_NEURON_DEVICE_PLUGIN_TAG\nfi\n\n# update the efa k8s device plugin\necho \"Updating EFA device plugin image\"\nEFA_DEVICE_PLUGIN_REPOSITORY_NAME=\"eks/aws-efa-k8s-device-plugin\"\nEFA_DEVICE_PLUGIN_TAGS=$(get_ecr_image_tags $EKS_CONTAINER_REGISTRY $EFA_DEVICE_PLUGIN_REPOSITORY_NAME)\nif [ $? -eq 0 ]; then\n LATEST_EFA_DEVICE_PLUGIN_TAG=$(echo $EFA_DEVICE_PLUGIN_TAGS | jq -r 'map(split(\"-\") | .[0]) | max_by(sub(\"^v\"; \"\") | split(\".\") | map(tonumber))')\n update_image_uris \"${EKS_CONTAINER_REGISTRY}/${EFA_DEVICE_PLUGIN_REPOSITORY_NAME}\" $LATEST_EFA_DEVICE_PLUGIN_TAG\nfi" }, { "path": "hack/update-neuron-dependencies.sh", "content": "#!/usr/bin/env bash\n\nset -o nounset\nset -o errexit\nset -o pipefail\n\n# pip_versionsearch takes exactly 1 argument and returns its latest available version from the neuron pip repo\n# usage: pip_versionsearch PACKAGE\npip_versionsearch() {\n local PACKAGE_INDEX_NAME=$(echo $1 | tr -s '_' '-')\n local PACKAGE_VERSION_NAME=$(echo $PACKAGE_INDEX_NAME | tr -s '-' '_')\n curl -s https://pip.repos.neuron.amazonaws.com/${PACKAGE_INDEX_NAME} | grep -o -G \"${PACKAGE_VERSION_NAME}-[0-9\\.]*+[a-f0-9]*\" | sed \"s/$PACKAGE_VERSION_NAME-//\" | sort -V | tail -n 1 \n}\n\n# versionsearch takes exactly 1 argument and returns its latest available version from the neuron amd64 apt repo\n# usage: versionsearch PACKAGE\nversionsearch() {\n local PACKAGE_NAME=$1\n curl -s https://apt.repos.neuron.amazonaws.com/dists/focal/main/binary-amd64/Packages | grep -o \"${PACKAGE_NAME}_[0-9\\.]*-*[a-f0-9]*\" | sed \"s/${PACKAGE_NAME}_//\" | sort -V | tail -n 1 \n}\n\n# update_arg ARG NEW_VALUE\nupdate_arg() {\n local ARG=$1\n local NEW_VALUE=$2\n echo \"setting $ARG to $NEW_VALUE\"\n find . -type f -name Dockerfile -exec sed -i \"s/${ARG}=.*/${ARG}=$NEW_VALUE/g\" {} +\n}\n\nupdate_arg NEURONX_RUNTIME_LIB_VERSION $(versionsearch aws-neuronx-runtime-lib)\nupdate_arg NEURONX_COLLECTIVES_LIB_VERSION $(versionsearch aws-neuronx-collectives)\nupdate_arg NEURONX_TOOLS_VERSION $(versionsearch aws-neuronx-tools)\nupdate_arg NEURONX_FRAMEWORK_VERSION $(pip_versionsearch torch-neuronx)\nupdate_arg NEURONX_CC_VERSION $(pip_versionsearch neuronx-cc)\nupdate_arg NEURONX_DISTRIBUTED_VERSION $(pip_versionsearch neuronx_distributed)" }, { "path": "hack/update-nvidia-dependencies.sh", "content": "#!/usr/bin/env bash\n\n# following from the last updated dependency:\n# 1. get the latest release of aws-ofi-nccl\n# 2. get the supported version of libnccl\n# 3. get the latest correct cuda version used for libnccl\n\nset -o nounset\nset -o errexit\nset -o pipefail\n\necho \"Updating aws-ofi-nccl\"\nAWS_OFI_NCCL_TAG=$(curl -s https://api.github.com/repos/aws/aws-ofi-nccl/releases/latest | jq -r .tag_name | sed 's/^v//')\nfind . -type f -name Dockerfile -exec sed -i \"s/AWS_OFI_NCCL_VERSION=.*/AWS_OFI_NCCL_VERSION=$AWS_OFI_NCCL_TAG/g\" {} +\n\necho \"Updating nccl\"\nLIB_NCCL_TAG=$(curl -s https://api.github.com/repos/aws/aws-ofi-nccl/releases/latest | jq -r .body | grep -oP '\\[NCCL \\K(\\S*)(?=\\])' | head -n 1 | sed 's/^v//')\nfind . -type f -name Dockerfile -exec sed -i \"s/LIBNCCL_VERSION=.*/LIBNCCL_VERSION=$LIB_NCCL_TAG/g\" {} +\n\necho \"Updating nvbandwidth\"\nNVBANDWIDTH_TAG=$(curl -s https://api.github.com/repos/NVIDIA/nvbandwidth/releases/latest | jq -r .tag_name)\nfind . -type f -name Dockerfile -exec sed -i \"s/NVBANDWIDTH_VERSION=.*/NVBANDWIDTH_VERSION=$NVBANDWIDTH_TAG/g\" {} +\n\n" }, { "path": "internal/awssdk/config.go", "content": "package awssdk\n\nimport (\n\t\"context\"\n\t\"log/slog\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n)\n\n// NewConfig returns an AWS SDK config\n// It will panic if the cnfig cannot be created\nfunc NewConfig() aws.Config {\n\tc, err := config.LoadDefaultConfig(context.TODO())\n\tif err != nil {\n\t\tslog.Error(\"failed to create AWS SDK config\", \"error\", err)\n\t\tpanic(err)\n\t}\n\treturn c\n}\n" }, { "path": "internal/deployers/eksapi/addons.go", "content": "package eksapi\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks\"\n)\n\nconst (\n\taddonCreationTimeout = 5 * time.Minute\n)\n\ntype AddonManager struct {\n\tclients *awsClients\n}\n\nfunc NewAddonManager(clients *awsClients) *AddonManager {\n\treturn &AddonManager{\n\t\tclients: clients,\n\t}\n}\n\nfunc (m *AddonManager) createAddons(infra *Infrastructure, cluster *Cluster, opts *deployerOptions) error {\n\tctx := context.TODO()\n\n\taddonMap := map[string]string{}\n\tfor _, addon := range opts.Addons {\n\t\taddonParts := strings.Split(addon, \":\")\n\t\tif len(addonParts) != 2 {\n\t\t\treturn fmt.Errorf(\"invalid addon format: %s\", addon)\n\t\t}\n\t\tname := addonParts[0]\n\t\tversion := addonParts[1]\n\t\tslog.Info(\"resolving addon version\", \"addon\", name, \"version\", version)\n\t\tresolvedVersion, err := m.resolveAddonVersion(name, version, opts.KubernetesVersion)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// dedupe addons with the same name. last provided entry wins.\n\t\taddonMap[name] = resolvedVersion\n\t}\n\n\tfor addonName, addonVersion := range addonMap {\n\t\tslog.Info(\"creating addon\", \"addon\", addonName, \"version\", addonVersion)\n\t\tinput := eks.CreateAddonInput{\n\t\t\tAddonName: aws.String(addonName),\n\t\t\tAddonVersion: aws.String(addonVersion),\n\t\t\tClusterName: aws.String(cluster.name),\n\t\t}\n\t\t_, err := m.clients.EKS().CreateAddon(ctx, &input)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to create addon: %v\", err)\n\t\t}\n\t\tslog.Info(\"waiting for addon to be active\", \"addon\", addonName)\n\t\terr = eks.NewAddonActiveWaiter(m.clients.EKS()).\n\t\t\tWait(ctx, &eks.DescribeAddonInput{\n\t\t\t\tAddonName: aws.String(addonName),\n\t\t\t\tClusterName: aws.String(cluster.name),\n\t\t\t}, addonCreationTimeout)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to wait for addon to be active: %v\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (m *AddonManager) resolveAddonVersion(name string, versionMarker string, kubernetesVersion string) (string, error) {\n\tinput := eks.DescribeAddonVersionsInput{\n\t\tAddonName: aws.String(name),\n\t\tKubernetesVersion: aws.String(kubernetesVersion),\n\t}\n\tdescOutput, err := m.clients.EKS().DescribeAddonVersions(context.TODO(), &input)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tfor _, addon := range descOutput.Addons {\n\t\tfor _, versionInfo := range addon.AddonVersions {\n\t\t\tswitch versionMarker {\n\t\t\tcase \"latest\":\n\t\t\t\treturn *versionInfo.AddonVersion, nil\n\t\t\tcase \"default\":\n\t\t\t\tfor _, compatibility := range versionInfo.Compatibilities {\n\t\t\t\t\tif compatibility.DefaultVersion {\n\t\t\t\t\t\treturn *versionInfo.AddonVersion, nil\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tif *versionInfo.AddonVersion == versionMarker {\n\t\t\t\t\treturn *versionInfo.AddonVersion, nil\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn \"\", fmt.Errorf(\"failed to resolve addon version: %s=%s\", name, versionMarker)\n}\n" }, { "path": "internal/deployers/eksapi/ami_resolver.go", "content": "package eksapi\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log/slog\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ec2\"\n\tec2types \"github.com/aws/aws-sdk-go-v2/service/ec2/types\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ssm\"\n)\n\nfunc NewAMIResolver(awsClients *awsClients) *amiResolver {\n\treturn &amiResolver{\n\t\tclients: awsClients,\n\t}\n}\n\ntype amiResolver struct {\n\tclients *awsClients\n}\n\nfunc (r *amiResolver) Resolve(ctx context.Context, opts *deployerOptions) (string, error) {\n\tswitch opts.UserDataFormat {\n\tcase UserDataBootstrapSh:\n\t\t// TODO: AL2 is not a high priority, skipping for now.\n\t\treturn \"\", fmt.Errorf(\"%s is not handled\", opts.UserDataFormat)\n\tcase UserDataNodeadm:\n\t\treturn r.ResolveAL2023(ctx, opts)\n\tcase UserDataBottlerocket:\n\t\treturn r.ResolveBottlerocket(ctx, opts)\n\tdefault:\n\t\treturn \"\", fmt.Errorf(\"unhandled userdata format: %s\", opts.UserDataFormat)\n\t}\n}\n\nfunc (r *amiResolver) ResolveAL2023(ctx context.Context, opts *deployerOptions) (string, error) {\n\tdescribeInstanceTypesResponse, err := r.clients.EC2().DescribeInstanceTypes(ctx, &ec2.DescribeInstanceTypesInput{\n\t\tInstanceTypes: []ec2types.InstanceType{ec2types.InstanceType(r.getInstance(opts))},\n\t})\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tinstanceTypeInfo := describeInstanceTypesResponse.InstanceTypes[0]\n\n\tarch, err := r.resolveArch(instanceTypeInfo)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tvariant := \"standard\"\n\tif instanceTypeInfo.NeuronInfo != nil {\n\t\tif len(instanceTypeInfo.NeuronInfo.NeuronDevices) > 0 {\n\t\t\tvariant = \"neuron\"\n\t\t}\n\t} else if instanceTypeInfo.GpuInfo != nil {\n\t\tfor _, gpu := range instanceTypeInfo.GpuInfo.Gpus {\n\t\t\tif aws.ToString(gpu.Manufacturer) == \"NVIDIA\" {\n\t\t\t\tvariant = \"nvidia\"\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tgetParameterReponse, err := r.clients.SSM().GetParameter(ctx, &ssm.GetParameterInput{\n\t\tName: aws.String(fmt.Sprintf(\"/aws/service/eks/optimized-ami/%s/amazon-linux-2023/%s/%s/recommended/image_id\", opts.KubernetesVersion, arch, variant)),\n\t})\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn aws.ToString(getParameterReponse.Parameter.Value), nil\n}\n\nfunc (r *amiResolver) ResolveBottlerocket(ctx context.Context, opts *deployerOptions) (string, error) {\n\tdescribeInstanceTypesResponse, err := r.clients.EC2().DescribeInstanceTypes(ctx, &ec2.DescribeInstanceTypesInput{\n\t\tInstanceTypes: []ec2types.InstanceType{ec2types.InstanceType(r.getInstance(opts))},\n\t})\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tinstanceTypeInfo := describeInstanceTypesResponse.InstanceTypes[0]\n\n\tarch, err := r.resolveArch(instanceTypeInfo)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\t// TODO: enable fips\n\tflavorSuffix := \"\"\n\tif instanceTypeInfo.GpuInfo != nil {\n\t\tfor _, gpu := range instanceTypeInfo.GpuInfo.Gpus {\n\t\t\tif aws.ToString(gpu.Manufacturer) == \"NVIDIA\" {\n\t\t\t\tflavorSuffix = \"-nvidia\"\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tgetParameterResponse, err := r.clients.SSM().GetParameter(ctx, &ssm.GetParameterInput{\n\t\tName: aws.String(fmt.Sprintf(\"/aws/service/bottlerocket/aws-k8s-%s%s/%s/latest/image_id\", opts.KubernetesVersion, flavorSuffix, arch)),\n\t})\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn aws.ToString(getParameterResponse.Parameter.Value), nil\n}\n\nfunc (r *amiResolver) getInstance(opts *deployerOptions) string {\n\tinstanceType := opts.InstanceTypes[0]\n\tif len(opts.InstanceTypes) > 1 {\n\t\tslog.Warn(\"only resolving AMI based on first instance type\", \"instanceType\", instanceType)\n\t}\n\treturn instanceType\n}\n\nfunc (r *amiResolver) resolveArch(instanceTypeInfo ec2types.InstanceTypeInfo) (string, error) {\n\t// TODO: the ordering might be weird because old instances might support\n\t// both i386 and x8664.\n\tswitch arch := instanceTypeInfo.ProcessorInfo.SupportedArchitectures[0]; arch {\n\tcase ec2types.ArchitectureTypeArm64, ec2types.ArchitectureTypeX8664:\n\t\treturn string(arch), nil\n\tdefault:\n\t\treturn \"\", fmt.Errorf(\"unhandled arch: %s\", arch)\n\t}\n}\n" }, { "path": "internal/deployers/eksapi/ami_resolver_test.go", "content": "//go:build integration\n\npackage eksapi\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestAMIResolver(t *testing.T) {\n\tctx := context.Background()\n\tawsCfg, err := config.LoadDefaultConfig(ctx)\n\tassert.NoError(t, err)\n\n\tamiResolver := NewAMIResolver(newAWSClients(awsCfg, \"\"))\n\n\tt.Run(\"AL2023-nvidia\", func(t *testing.T) {\n\t\topts := deployerOptions{\n\t\t\tUserDataFormat: UserDataNodeadm,\n\t\t\tKubernetesVersion: \"1.33\",\n\t\t}\n\t\tt.Run(\"nvidia\", func(t *testing.T) {\n\t\t\topts := opts\n\t\t\topts.InstanceTypes = []string{\"g5.xlarge\"}\n\n\t\t\tami, err := amiResolver.Resolve(ctx, &opts)\n\t\t\tassert.NoError(t, err)\n\t\t\tassert.Regexp(t, \"ami-.*\", ami)\n\t\t})\n\t\tt.Run(\"standard\", func(t *testing.T) {\n\t\t\topts := opts\n\t\t\topts.InstanceTypes = []string{\"m5.xlarge\"}\n\n\t\t\tami, err := amiResolver.Resolve(ctx, &opts)\n\t\t\tassert.NoError(t, err)\n\t\t\tassert.Regexp(t, \"ami-.*\", ami)\n\t\t})\n\t})\n\n\tt.Run(\"Bottlerocket\", func(t *testing.T) {\n\t\topts := deployerOptions{\n\t\t\tUserDataFormat: UserDataBottlerocket,\n\t\t\tKubernetesVersion: \"1.33\",\n\t\t}\n\t\tt.Run(\"nvidia\", func(t *testing.T) {\n\t\t\topts := opts\n\t\t\topts.InstanceTypes = []string{\"g5.xlarge\"}\n\n\t\t\tami, err := amiResolver.Resolve(ctx, &opts)\n\t\t\tassert.NoError(t, err)\n\t\t\tassert.Regexp(t, \"ami-.*\", ami)\n\t\t})\n\t\tt.Run(\"standard\", func(t *testing.T) {\n\t\t\topts := opts\n\t\t\topts.InstanceTypes = []string{\"m5.xlarge\"}\n\n\t\t\tami, err := amiResolver.Resolve(ctx, &opts)\n\t\t\tassert.NoError(t, err)\n\t\t\tassert.Regexp(t, \"ami-.*\", ami)\n\t\t})\n\t})\n}\n" }, { "path": "internal/deployers/eksapi/auth_map_role.go", "content": "package eksapi\n\nimport (\n\t\"bytes\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates\"\n)\n\nfunc generateAuthMapRole(nodeNameStrategy string, rolearn string) (string, error) {\n\ttemplate := templates.AuthMapRole\n\tbuf := bytes.Buffer{}\n\tif err := template.Execute(&buf, templates.AuthMapRoleTemplateData{\n\t\tNodeNameStrategy: nodeNameStrategy,\n\t\tRolearn: rolearn,\n\t}); err != nil {\n\t\treturn \"\", err\n\t}\n\treturn buf.String(), nil\n}\n" }, { "path": "internal/deployers/eksapi/auth_map_role_test.go", "content": "package eksapi\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst rolearn = \"mock-role-arn\"\n\nconst sessionNamedAuthMapRole = `\n- username: system:node:{{SessionName}} \n groups:\n - system:bootstrappers\n - system:nodes\n rolearn: mock-role-arn`\n\nconst privateDNSNamedAuthMapRole = `\n- username: system:node:{{EC2PrivateDNSName}} \n groups:\n - system:bootstrappers\n - system:nodes\n rolearn: mock-role-arn`\n\nfunc Test_generateAuthRoleMap(t *testing.T) {\n\tcases := []struct {\n\t\tnodeNameStrategy string\n\t\texpected string\n\t}{\n\t\t{\n\t\t\tnodeNameStrategy: \"SessionName\",\n\t\t\texpected: sessionNamedAuthMapRole,\n\t\t},\n\t\t{\n\t\t\tnodeNameStrategy: \"EC2PrivateDNSName\",\n\t\t\texpected: privateDNSNamedAuthMapRole,\n\t\t},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.nodeNameStrategy, func(t *testing.T) {\n\t\t\tactual, err := generateAuthMapRole(c.nodeNameStrategy, rolearn)\n\t\t\tif err != nil {\n\t\t\t\tt.Log(err)\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tassert.Equal(t, c.expected, actual)\n\t\t})\n\t}\n}\n" }, { "path": "internal/deployers/eksapi/aws.go", "content": "package eksapi\n\nimport (\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/autoscaling\"\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudformation\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ec2\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks\"\n\t\"github.com/aws/aws-sdk-go-v2/service/iam\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ssm\"\n)\n\ntype awsClients struct {\n\t_eks *eks.Client\n\t_cfn *cloudformation.Client\n\t_ec2 *ec2.Client\n\t_asg *autoscaling.Client\n\t_ssm *ssm.Client\n\t_iam *iam.Client\n\t_s3 *s3.Client\n\t_s3Presign *s3.PresignClient\n}\n\nfunc newAWSClients(config aws.Config, eksEndpointURL string) *awsClients {\n\tclients := awsClients{\n\t\t_cfn: cloudformation.NewFromConfig(config),\n\t\t_ec2: ec2.NewFromConfig(config),\n\t\t_asg: autoscaling.NewFromConfig(config),\n\t\t_ssm: ssm.NewFromConfig(config),\n\t\t_iam: iam.NewFromConfig(config),\n\t\t_s3: s3.NewFromConfig(config),\n\t}\n\tclients._s3Presign = s3.NewPresignClient(clients._s3)\n\tif eksEndpointURL != \"\" {\n\t\tclients._eks = eks.NewFromConfig(config, func(o *eks.Options) {\n\t\t\to.BaseEndpoint = aws.String(eksEndpointURL)\n\t\t})\n\t} else {\n\t\tclients._eks = eks.NewFromConfig(config)\n\t}\n\treturn &clients\n}\n\nfunc (c *awsClients) EKS() *eks.Client {\n\treturn c._eks\n}\n\nfunc (c *awsClients) CFN() *cloudformation.Client {\n\treturn c._cfn\n}\n\nfunc (c *awsClients) EC2() *ec2.Client {\n\treturn c._ec2\n}\n\nfunc (c *awsClients) ASG() *autoscaling.Client {\n\treturn c._asg\n}\n\nfunc (c *awsClients) SSM() *ssm.Client {\n\treturn c._ssm\n}\n\nfunc (c *awsClients) IAM() *iam.Client {\n\treturn c._iam\n}\n\nfunc (c *awsClients) S3() *s3.Client {\n\treturn c._s3\n}\n\nfunc (c *awsClients) S3Presign() *s3.PresignClient {\n\treturn c._s3Presign\n}\n" }, { "path": "internal/deployers/eksapi/cluster.go", "content": "package eksapi\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/util\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks\"\n\tekstypes \"github.com/aws/aws-sdk-go-v2/service/eks/types\"\n\t\"github.com/aws/smithy-go/ptr\"\n)\n\ntype ClusterManager struct {\n\tclients *awsClients\n\tresourceID string\n}\n\nfunc NewClusterManager(clients *awsClients, resourceID string) *ClusterManager {\n\treturn &ClusterManager{\n\t\tclients: clients,\n\t\tresourceID: resourceID,\n\t}\n}\n\ntype Cluster struct {\n\tendpoint string\n\tcertificateAuthorityData string\n\tsecurityGroupId string\n\tarn string\n\tname string\n\tcidr string\n}\n\nfunc (m *ClusterManager) getOrCreateCluster(infra *Infrastructure, opts *deployerOptions) (*Cluster, error) {\n\ttargetClusterName := opts.StaticClusterName\n\tif targetClusterName == \"\" {\n\t\tslog.Info(\"creating cluster...\")\n\t\tinput := eks.CreateClusterInput{\n\t\t\tName: aws.String(m.resourceID),\n\t\t\tResourcesVpcConfig: &ekstypes.VpcConfigRequest{\n\t\t\t\tEndpointPrivateAccess: aws.Bool(true),\n\t\t\t\tEndpointPublicAccess: aws.Bool(true),\n\t\t\t\tSubnetIds: infra.subnets(),\n\t\t\t},\n\t\t\tRoleArn: aws.String(infra.clusterRoleARN),\n\t\t\tKubernetesNetworkConfig: &ekstypes.KubernetesNetworkConfigRequest{\n\t\t\t\tIpFamily: ekstypes.IpFamily(opts.IPFamily),\n\t\t\t},\n\t\t\tVersion: aws.String(opts.KubernetesVersion),\n\t\t}\n\t\tif opts.AutoMode {\n\t\t\tinput.ComputeConfig = &ekstypes.ComputeConfigRequest{\n\t\t\t\t// we don't enable any of the default node pools, we'll create our own\n\t\t\t\tEnabled: aws.Bool(true),\n\t\t\t\tNodeRoleArn: aws.String(infra.nodeRoleARN),\n\t\t\t\t// TODO: we can't currently enable managed compute without a default NodePool\n\t\t\t\t// the system NodePool is tainted for critical addons only, so will be ignored for our test workloads\n\t\t\t\tNodePools: []string{\"system\"},\n\t\t\t}\n\t\t\tinput.StorageConfig = &ekstypes.StorageConfigRequest{\n\t\t\t\tBlockStorage: &ekstypes.BlockStorage{\n\t\t\t\t\tEnabled: aws.Bool(true),\n\t\t\t\t},\n\t\t\t}\n\t\t\tinput.KubernetesNetworkConfig.ElasticLoadBalancing = &ekstypes.ElasticLoadBalancing{\n\t\t\t\tEnabled: aws.Bool(true),\n\t\t\t}\n\t\t\tinput.AccessConfig = &ekstypes.CreateAccessConfigRequest{\n\t\t\t\tAuthenticationMode: ekstypes.AuthenticationModeApi,\n\t\t\t}\n\t\t\tinput.BootstrapSelfManagedAddons = aws.Bool(false)\n\t\t}\n\t\tif opts.EnableClusterLogging {\n\t\t\tinput.Logging = &ekstypes.Logging{\n\t\t\t\tClusterLogging: []ekstypes.LogSetup{\n\t\t\t\t\t{\n\t\t\t\t\t\tEnabled: ptr.Bool(true),\n\t\t\t\t\t\tTypes: ekstypes.LogTypeApi.Values(),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t}\n\t\t}\n\t\tapiOpts, err := util.NewHTTPHeaderAPIOptions(opts.UpClusterHeaders)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to create API options: %v\", err)\n\t\t}\n\t\tcreateOutput, err := m.clients.EKS().CreateCluster(context.TODO(), &input,\n\t\t\tfunc(o *eks.Options) {\n\t\t\t\to.APIOptions = apiOpts\n\t\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to create cluster: %v\", err)\n\t\t}\n\t\ttargetClusterName = aws.ToString(createOutput.Cluster.Name)\n\t} else {\n\t\tslog.Info(\"reusing existing static cluster\", \"clusterName\", opts.StaticClusterName)\n\t}\n\tcluster, waitErr := m.waitForClusterActive(targetClusterName, opts.ClusterCreationTimeout)\n\tif waitErr != nil {\n\t\treturn nil, fmt.Errorf(\"failed to wait for cluster to become active: %v\", waitErr)\n\t}\n\treturn cluster, nil\n}\n\nfunc (m *ClusterManager) waitForClusterActive(clusterName string, timeout time.Duration) (*Cluster, error) {\n\tslog.Info(\"waiting for cluster to be active\", \"clusterName\", clusterName)\n\tout, err := eks.NewClusterActiveWaiter(m.clients.EKS()).WaitForOutput(context.TODO(), &eks.DescribeClusterInput{\n\t\tName: aws.String(clusterName),\n\t}, timeout)\n\t// log when possible, whether there was an error or not\n\tif out != nil {\n\t\tslog.Info(\"cluster details\", \"cluster\", out.Cluster)\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed waiting for cluster be active: %v\", err)\n\t}\n\tslog.Info(\"cluster is active\", \"arn\", *out.Cluster.Arn)\n\tvar cidr string\n\tswitch out.Cluster.KubernetesNetworkConfig.IpFamily {\n\tcase ekstypes.IpFamilyIpv4:\n\t\tcidr = *out.Cluster.KubernetesNetworkConfig.ServiceIpv4Cidr\n\tcase ekstypes.IpFamilyIpv6:\n\t\tcidr = *out.Cluster.KubernetesNetworkConfig.ServiceIpv6Cidr\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unknown cluster IP family: '%v'\", out.Cluster.KubernetesNetworkConfig.IpFamily)\n\t}\n\treturn &Cluster{\n\t\tarn: *out.Cluster.Arn,\n\t\tcertificateAuthorityData: *out.Cluster.CertificateAuthority.Data,\n\t\tcidr: cidr,\n\t\tendpoint: *out.Cluster.Endpoint,\n\t\tname: *out.Cluster.Name,\n\t\tsecurityGroupId: *out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId,\n\t}, nil\n}\n\nfunc (m *ClusterManager) isClusterActive() (bool, error) {\n\tresult, err := m.clients.EKS().DescribeCluster(context.TODO(), &eks.DescribeClusterInput{\n\t\tName: aws.String(m.resourceID),\n\t})\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tswitch result.Cluster.Status {\n\tcase ekstypes.ClusterStatusActive:\n\t\treturn true, nil\n\tcase ekstypes.ClusterStatusCreating:\n\t\treturn false, nil\n\tdefault:\n\t\treturn false, fmt.Errorf(\"cluster status is: %v\", result.Cluster.Status)\n\t}\n}\n\nfunc (m *ClusterManager) deleteCluster() error {\n\tconst (\n\t\tretryInterval = 2 * time.Minute\n\t\tmaxAttempts = 5\n\t)\n\n\tfor attempt := 1; attempt <= maxAttempts; attempt++ {\n\t\tinput := eks.DeleteClusterInput{\n\t\t\tName: aws.String(m.resourceID),\n\t\t}\n\n\t\tslog.Info(\"deleting cluster...\", \"attempt\", attempt)\n\t\tout, err := m.clients.EKS().DeleteCluster(context.TODO(), &input)\n\t\tif err != nil {\n\t\t\tvar notFound *ekstypes.ResourceNotFoundException\n\t\t\tif errors.As(err, ¬Found) {\n\t\t\t\tslog.Info(\"cluster does not exist\", \"resourceID\", m.resourceID)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif attempt == maxAttempts {\n\t\t\t\treturn fmt.Errorf(\"failed to delete cluster after %d attempts: %v\", maxAttempts, err)\n\t\t\t}\n\t\t\tslog.Info(\"deletion failed, retrying...\", \"error\", err, \"retryInterval\", retryInterval)\n\t\t\ttime.Sleep(retryInterval)\n\t\t\tcontinue\n\t\t}\n\n\t\tslog.Info(\"waiting for cluster to be deleted\", \"arn\", *out.Cluster.Arn)\n\t\terr = eks.NewClusterDeletedWaiter(m.clients.EKS()).\n\t\t\tWait(context.TODO(), &eks.DescribeClusterInput{\n\t\t\t\tName: aws.String(m.resourceID),\n\t\t\t}, time.Minute*15)\n\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to wait for cluster to be deleted: %v\", err)\n\t\t}\n\t\treturn nil\n\t}\n\n\treturn fmt.Errorf(\"failed to delete cluster after %d attempts\", maxAttempts)\n}\n" }, { "path": "internal/deployers/eksapi/common.go", "content": "package eksapi\n\nimport (\n\t\"os\"\n\t\"slices\"\n\t\"strings\"\n)\n\nconst AvailabilityZonePriorityEnv = \"EKSAPI_AZ_PRIORITY\"\n\nfunc availabilityZoneHintedOrder(availabilityZones []string) []string {\n\tvar priorityAZs []string\n\tif priorityAZsString, ok := os.LookupEnv(AvailabilityZonePriorityEnv); ok {\n\t\tpriorityAZs = strings.Split(priorityAZsString, \",\")\n\t}\n\tif len(priorityAZs) == 0 {\n\t\treturn availabilityZones\n\t}\n\treturn slices.SortedStableFunc(slices.Values(availabilityZones), func(az1, az2 string) int {\n\t\tif slices.Contains(priorityAZs, az1) {\n\t\t\tif slices.Contains(priorityAZs, az2) {\n\t\t\t\treturn 0\n\t\t\t}\n\t\t\treturn -1\n\t\t}\n\t\treturn 0\n\t})\n}\n" }, { "path": "internal/deployers/eksapi/common_test.go", "content": "package eksapi\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc Test_AZ_PRIORITY(t *testing.T) {\n\tt.Setenv(AvailabilityZonePriorityEnv, \"us-west-2d\")\n\tassert.Equal(t,\n\t\t[]string{\"us-west-2d\", \"us-west-2b\", \"us-west-2c\"},\n\t\tavailabilityZoneHintedOrder([]string{\"us-west-2b\", \"us-west-2c\", \"us-west-2d\"}),\n\t)\n}\n" }, { "path": "internal/deployers/eksapi/deployer.go", "content": "package eksapi\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal\"\n\t\"github.com/aws/aws-k8s-tester/internal/awssdk\"\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates\"\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/internal/metrics\"\n\t\"github.com/aws/aws-k8s-tester/internal/util\"\n\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudwatch\"\n\tekstypes \"github.com/aws/aws-sdk-go-v2/service/eks/types\"\n\t\"github.com/spf13/pflag\"\n\t\"github.com/urfave/sflags/gen/gpflag\"\n\t\"golang.org/x/exp/slices\"\n\t\"sigs.k8s.io/kubetest2/pkg/types\"\n)\n\n// DeployerName is the name of the deployer\nconst DeployerName = \"eksapi\"\n\nconst ResourcePrefix = \"kubetest2-\" + DeployerName\n\nvar SupportedNodeNameStrategy = []string{\"SessionName\", \"EC2PrivateDNSName\"}\n\n// assert that deployer implements optional interfaces\nvar _ types.DeployerWithKubeconfig = &deployer{}\nvar _ types.DeployerWithInit = &deployer{}\nvar _ types.DeployerWithFinish = &deployer{}\n\ntype deployer struct {\n\tcommonOptions types.Options\n\tdeployerOptions\n\n\tmetrics metrics.MetricRegistry\n\tinfraManager *InfrastructureManager\n\tclusterManager *ClusterManager\n\taddonManager *AddonManager\n\tnodeManager *nodeManager\n\tlogManager *logManager\n\tstaticClusterManager *StaticClusterManager\n\n\tawsClients *awsClients\n\n\tinfra *Infrastructure\n\tcluster *Cluster\n\n\tk8sClient *k8sClient\n\n\tinitTime time.Time\n}\n\ntype deployerOptions struct {\n\tAddons []string `flag:\"addons\" desc:\"Managed addons (name:version pairs) to create in the cluster. Use 'latest' for the most recent version, or 'default' for the default version.\"`\n\tAMI string `flag:\"ami\" desc:\"AMI for unmanaged nodes\"`\n\tAMIType string `flag:\"ami-type\" desc:\"AMI type for managed nodes\"`\n\tAutoMode bool `flag:\"auto-mode\" desc:\"Enable EKS Auto Mode\"`\n\tCapacityReservation bool `flag:\"capacity-reservation\" desc:\"Use capacity reservation for the unmanaged nodegroup\"`\n\tTargetCapacityReservationId string `flag:\"target-capacity-reservation-id\" desc:\"CapacityReservation ID to use for targeted launches. Implies --capacity-reservation.\"`\n\tClusterCreationTimeout time.Duration `flag:\"cluster-creation-timeout\" desc:\"Time to wait for cluster to be created and become active.\"`\n\tClusterRoleServicePrincipal string `flag:\"cluster-role-service-principal\" desc:\"Additional service principal that can assume the cluster role\"`\n\tDeployCloudwatchInfra bool `flag:\"deploy-cloudwatch-infra\" desc:\"Deploy required infrastructure for emitting metrics to CloudWatch\"`\n\tEFA bool `flag:\"efa\" desc:\"Create EFA interfaces on the node of an unmanaged nodegroup. One instance type must be passed if set. Requires --unmanaged-nodes and --instance-types.\"`\n\tEKSEndpointURL string `flag:\"endpoint-url\" desc:\"Endpoint URL for the EKS API\"`\n\tEmitMetrics bool `flag:\"emit-metrics\" desc:\"Record and emit metrics to CloudWatch\"`\n\tEnableClusterLogging bool `flag:\"enable-cluster-logging\" desc:\"Enable sending EKS control plane logs to an /aws/eks/ 0 && len(d.InstanceTypeArchs) > 0 {\n\t\treturn fmt.Errorf(\"--instance-types and --instance-type-archs are mutually exclusive\")\n\t}\n\tif d.TargetCapacityReservationId != \"\" {\n\t\td.CapacityReservation = true\n\t}\n\tif d.UnmanagedNodes {\n\t\tif d.AMIType != \"\" {\n\t\t\treturn fmt.Errorf(\"--ami-type should not be provided with --unmanaged-nodes\")\n\t\t}\n\t\tif d.NodeNameStrategy == \"\" {\n\t\t\td.NodeNameStrategy = \"EC2PrivateDNSName\"\n\t\t\tslog.Info(\"using default node name strategy\", \"strategy\", \"EC2PrivateDNSName\")\n\t\t} else {\n\t\t\tif !slices.Contains(SupportedNodeNameStrategy, d.NodeNameStrategy) {\n\t\t\t\treturn fmt.Errorf(\"--node-name-strategy must be one of the following values: ['SessionName', 'EC2PrivateDNSName']\")\n\t\t\t}\n\t\t}\n\t\tif d.UserDataFormat == \"\" {\n\t\t\td.UserDataFormat = UserDataBootstrapSh\n\t\t\tslog.Info(\"using default user data format\", \"format\", d.UserDataFormat)\n\t\t}\n\t\t// AMI ID check must come after user-data format resolution because we\n\t\t// can try to infer the AMI type for unmanaged nodes.\n\t\tif d.AMI == \"\" {\n\t\t\tami, err := NewAMIResolver(d.awsClients).Resolve(context.TODO(), &d.deployerOptions)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to automatically resolve ami for unmanaged nodegroup (provide --ami to short circuit this): %w\", err)\n\t\t\t}\n\t\t\td.AMI = ami\n\t\t}\n\n\t\tif d.EFA && len(d.InstanceTypes) != 1 {\n\t\t\treturn fmt.Errorf(\"--efa requires a single instance type\")\n\t\t}\n\t} else {\n\t\tif d.AMI != \"\" {\n\t\t\treturn fmt.Errorf(\"--ami should not be provided without --unmanaged-nodes\")\n\t\t}\n\t\tif d.AMIType == \"\" {\n\t\t\td.AMIType = \"AL2023_x86_64_STANDARD\"\n\t\t\tslog.Info(\"using default AMI type\", \"amiType\", d.AMIType)\n\t\t}\n\t}\n\tif d.EKSEndpointURL != \"\" && d.ClusterRoleServicePrincipal == \"\" {\n\t\tspType := \"beta\"\n\t\tif strings.Contains(d.EKSEndpointURL, \"gamma\") {\n\t\t\tspType = \"gamma\"\n\t\t}\n\t\td.ClusterRoleServicePrincipal = fmt.Sprintf(\"eks-%s.aws.internal\", spType)\n\t}\n\tif d.DeployCloudwatchInfra {\n\t\tslog.Info(\"prepending pod identity agent to addons for cloudwatch infrastructure\")\n\t\t// this must be prepended to the list in order to respect user overrides.\n\t\td.deployerOptions.Addons = slices.Insert(d.deployerOptions.Addons, 0, \"eks-pod-identity-agent:default\")\n\t}\n\treturn nil\n}\n\nfunc detectKubernetesVersion() (string, error) {\n\tdetectedVersion, err := util.DetectKubernetesVersion()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tminorVersion, err := util.ParseMinorVersion(detectedVersion)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn minorVersion, nil\n}\n\nfunc (d *deployer) IsUp() (up bool, err error) {\n\treturn d.clusterManager.isClusterActive()\n}\n\nfunc (d *deployer) Down() error {\n\tif err := d.logManager.gatherLogsFromNodes(d.k8sClient, &d.deployerOptions, deployerPhaseDown); err != nil {\n\t\tslog.Warn(\"failed to gather logs from nodes\", \"error\", err)\n\t\t// don't return err, this isn't critical\n\t}\n\tif d.deployerOptions.StaticClusterName != \"\" {\n\t\treturn d.staticClusterManager.TearDownNodeForStaticCluster()\n\t}\n\treturn deleteResources(d.infraManager, d.clusterManager, d.nodeManager, d.k8sClient, &d.deployerOptions)\n}\n\nfunc deleteResources(im *InfrastructureManager, cm *ClusterManager, nm *nodeManager, k8sClient *k8sClient /* nillable */, opts *deployerOptions /* nillable */) error {\n\tif err := im.deleteCloudWatchInfrastructureStack(); err != nil {\n\t\treturn err\n\t}\n\tif err := nm.deleteNodes(k8sClient, opts); err != nil {\n\t\treturn err\n\t}\n\t// the EKS-managed cluster security group may be associated with a leaked ENI\n\t// so we need to make sure we've deleted leaked ENIs before we delete the cluster\n\t// otherwise, the cluster security group will be left behind and will block deletion of our VPC\n\tif err := im.deleteLeakedENIs(); err != nil {\n\t\treturn err\n\t}\n\tif err := cm.deleteCluster(); err != nil {\n\t\treturn err\n\t}\n\treturn im.deleteInfrastructureStack()\n}\n" }, { "path": "internal/deployers/eksapi/infra.go", "content": "package eksapi\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"errors\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"path\"\n\t\"slices\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudformation\"\n\tcloudformationtypes \"github.com/aws/aws-sdk-go-v2/service/cloudformation/types\"\n\tcloudwatchtypes \"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ec2\"\n\tec2types \"github.com/aws/aws-sdk-go-v2/service/ec2/types\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks\"\n\t\"github.com/aws/aws-sdk-go-v2/service/iam\"\n\tiamtypes \"github.com/aws/aws-sdk-go-v2/service/iam/types\"\n\t\"github.com/aws/aws-sdk-go/aws/arn\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates\"\n\t\"github.com/aws/aws-k8s-tester/internal/metrics\"\n\t\"github.com/aws/aws-k8s-tester/internal/util\"\n)\n\nconst (\n\tinfraStackCreationTimeout = time.Minute * 15\n\tinfraStackDeletionTimeout = time.Minute * 30\n\tnetworkInterfaceDetachmentTimeout = time.Minute * 10\n)\n\nconst (\n\t// the VPC CNI will always add this tag to ENI's that it creates\n\tvpcCNIENITagKey = \"node.k8s.amazonaws.com/createdAt\"\n\t// the IPAM controller will add this tag to the ENI's that it creates\n\tipamControllerENITagKey = \"eks:kubernetes-cni-node-name\"\n)\n\n// eksEndpointURLTag is the key for an optional tag on the infrastructure CloudFormation stack,\n// which indicates which EKS environment is associated with the stack's resources.\n// The tag is only added when --endpoint-url is passed to the deployer.\nconst eksEndpointURLTag = \"eks-endpoint-url\"\n\nvar (\n\tinfraMetricNamespace = path.Join(DeployerMetricNamespace, \"infrastructure\")\n\tinfraStackDeletionFailed = &metrics.MetricSpec{\n\t\tNamespace: infraMetricNamespace,\n\t\tMetric: \"StackDeletionFailed\",\n\t\tUnit: cloudwatchtypes.StandardUnitCount,\n\t}\n\tinfraLeakedENIs = &metrics.MetricSpec{\n\t\tNamespace: infraMetricNamespace,\n\t\tMetric: \"LeakedENIs\",\n\t\tUnit: cloudwatchtypes.StandardUnitCount,\n\t}\n)\n\ntype InfrastructureManager struct {\n\tclients *awsClients\n\tresourceID string\n\tmetrics metrics.MetricRegistry\n}\n\nfunc NewInfrastructureManager(clients *awsClients, resourceID string, metrics metrics.MetricRegistry) *InfrastructureManager {\n\treturn &InfrastructureManager{\n\t\tclients: clients,\n\t\tresourceID: resourceID,\n\t\tmetrics: metrics,\n\t}\n}\n\ntype Infrastructure struct {\n\tavailabilityZones []string\n\tvpc string\n\tsubnetsPublic []string\n\tsubnetsPrivate []string\n\tclusterRoleARN string\n\tnodeRoleARN string\n\tnodeRoleName string\n\tcloudwatchRoleArn string\n}\n\nfunc (i *Infrastructure) subnets() []string {\n\treturn append(i.subnetsPublic, i.subnetsPrivate...)\n}\n\nfunc (m *InfrastructureManager) createInfrastructureStack(opts *deployerOptions) (*Infrastructure, error) {\n\tvar subnetAzs []string\n\tif opts.CapacityReservation {\n\t\tazs, err := m.getAZsWithCapacity(opts)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tsubnetAzs = azs\n\t} else if len(opts.InstanceTypes) > 0 {\n\t\tazs, err := m.getRankedAZsForInstanceTypes(opts)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif len(azs) == 0 {\n\t\t\treturn nil, fmt.Errorf(\"no AZs support any of the provided instance types (%v)\", opts.InstanceTypes)\n\t\t}\n\t\tsubnetAzs = azs\n\t}\n\n\t// this value is not currently configurable, the infra stack is hardcoded to create 2\n\t// TODO: create a subnet in every AZ. today we need exactly 2 AZs for the subnets.\n\tconst numInfraAZs = 2\n\n\tsubnetAzs, err := m.normalizeAZs(opts, subnetAzs, numInfraAZs)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tslog.Info(\"creating infrastructure stack\", \"availabilityZones\", subnetAzs)\n\tinput := cloudformation.CreateStackInput{\n\t\tStackName: aws.String(m.resourceID),\n\t\tTemplateBody: aws.String(templates.Infrastructure),\n\t\tCapabilities: []cloudformationtypes.Capability{cloudformationtypes.CapabilityCapabilityIam},\n\t\tParameters: []cloudformationtypes.Parameter{\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"ResourceId\"),\n\t\t\t\tParameterValue: aws.String(m.resourceID),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"Subnet01AZ\"),\n\t\t\t\tParameterValue: aws.String(subnetAzs[0]),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"Subnet02AZ\"),\n\t\t\t\tParameterValue: aws.String(subnetAzs[1]),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"AutoMode\"),\n\t\t\t\tParameterValue: aws.String(fmt.Sprintf(\"%t\", opts.AutoMode)),\n\t\t\t},\n\t\t},\n\t}\n\tif opts.ClusterRoleServicePrincipal != \"\" {\n\t\tinput.Parameters = append(input.Parameters, cloudformationtypes.Parameter{\n\t\t\tParameterKey: aws.String(\"AdditionalClusterRoleServicePrincipal\"),\n\t\t\tParameterValue: aws.String(opts.ClusterRoleServicePrincipal),\n\t\t})\n\t}\n\tif opts.EKSEndpointURL != \"\" {\n\t\tinput.Tags = []cloudformationtypes.Tag{\n\t\t\t{\n\t\t\t\tKey: aws.String(eksEndpointURLTag),\n\t\t\t\tValue: aws.String(opts.EKSEndpointURL),\n\t\t\t},\n\t\t}\n\t}\n\tslog.Info(\"creating infrastructure stack...\")\n\tout, err := m.clients.CFN().CreateStack(context.TODO(), &input)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tslog.Info(\"waiting for infrastructure stack to be created\", \"stackId\", *out.StackId)\n\terr = cloudformation.NewStackCreateCompleteWaiter(m.clients.CFN()).\n\t\tWait(context.TODO(),\n\t\t\t&cloudformation.DescribeStacksInput{\n\t\t\t\tStackName: out.StackId,\n\t\t\t},\n\t\t\tinfraStackCreationTimeout)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to wait for infrastructure stack creation: %w\", err)\n\t}\n\tslog.Info(\"getting infrastructure stack resources\", \"stackId\", *out.StackId)\n\tinfra, err := m.getInfrastructureStackResources()\n\tinfra.availabilityZones = subnetAzs\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to get infrastructure stack resources: %w\", err)\n\t}\n\tslog.Info(\"created infrastructure\", \"infra\", infra)\n\n\treturn infra, nil\n}\n\nfunc (m *InfrastructureManager) getInfrastructureStackResources() (*Infrastructure, error) {\n\tstack, err := m.clients.CFN().DescribeStacks(context.TODO(), &cloudformation.DescribeStacksInput{\n\t\tStackName: aws.String(m.resourceID),\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tinfra := Infrastructure{}\n\tfor _, output := range stack.Stacks[0].Outputs {\n\t\tvalue := *output.OutputValue\n\t\tswitch *output.OutputKey {\n\t\tcase \"VPC\":\n\t\t\tinfra.vpc = value\n\t\tcase \"SubnetsPublic\":\n\t\t\tinfra.subnetsPublic = strings.Split(value, \",\")\n\t\tcase \"SubnetsPrivate\":\n\t\t\tinfra.subnetsPrivate = strings.Split(value, \",\")\n\t\tcase \"ClusterRole\":\n\t\t\tarn, err := arn.Parse(value)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"infrastructure stack ClusterRole output is not a valid ARN: '%s': %v\", value, err)\n\t\t\t}\n\t\t\tinfra.clusterRoleARN = arn.String()\n\t\tcase \"NodeRole\":\n\t\t\tarn, err := arn.Parse(value)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"infrastructure stack NodeRole output is not a valid ARN: '%s': %v\", value, err)\n\t\t\t}\n\t\t\tinfra.nodeRoleARN = arn.String()\n\t\t\t// Resource looks like 'role:/MyRole'\n\t\t\tresourceParts := strings.Split(arn.Resource, \"/\")\n\t\t\tinfra.nodeRoleName = resourceParts[len(resourceParts)-1]\n\t\t}\n\t}\n\treturn &infra, nil\n}\n\nfunc (m *InfrastructureManager) deleteInfrastructureStack() error {\n\tinfra, err := m.getInfrastructureStackResources()\n\tif err != nil {\n\t\tvar notFound *cloudformationtypes.StackNotFoundException\n\t\tif errors.As(err, ¬Found) {\n\t\t\tslog.Info(\"infrastructure stack does not exist\", \"resourceID\", m.resourceID)\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\tif err := m.deleteLeakedInstanceProfiles(infra); err != nil {\n\t\treturn err\n\t}\n\tinput := cloudformation.DeleteStackInput{\n\t\tStackName: aws.String(m.resourceID),\n\t}\n\tslog.Info(\"deleting infrastructure stack\", \"resourceID\", m.resourceID)\n\t_, err = m.clients.CFN().DeleteStack(context.TODO(), &input)\n\tif err != nil {\n\t\tvar notFound *cloudformationtypes.StackNotFoundException\n\t\tif errors.As(err, ¬Found) {\n\t\t\tslog.Info(\"infrastructure stack does not exist\", \"resourceID\", m.resourceID)\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to delete infrastructure stack: %w\", err)\n\t}\n\tslog.Info(\"waiting for infrastructure stack to be deleted\", \"resourceID\", m.resourceID)\n\terr = cloudformation.NewStackDeleteCompleteWaiter(m.clients.CFN()).\n\t\tWait(context.TODO(),\n\t\t\t&cloudformation.DescribeStacksInput{\n\t\t\t\tStackName: aws.String(m.resourceID),\n\t\t\t},\n\t\t\tinfraStackDeletionTimeout)\n\tif err != nil {\n\t\t// don't fail the overall test, the janitor can clean this up\n\t\tslog.Warn(\"failed to wait for infrastructure stack deletion\", \"error\", err)\n\t\tm.metrics.Record(infraStackDeletionFailed, 1, nil)\n\t\treturn nil\n\t}\n\tslog.Info(\"deleted infrastructure stack\", \"resourceID\", m.resourceID)\n\treturn nil\n}\n\n// deleteLeakedInstanceProfiles deletes any instance profiles to which the node role is attached,\n// because this will block node role deletion (and deletion of the infrastructure stack).\n// For example, when --auto-mode is used, an instance profile will be created for us and won't be deleted automatically with the cluster.\nfunc (m *InfrastructureManager) deleteLeakedInstanceProfiles(infra *Infrastructure) error {\n\tif infra.nodeRoleName == \"\" {\n\t\t// if the infra stack failed to create, it could end up in a weird state with no node role\n\t\t// we know there aren't any instance profiles in that case, so all good\n\t\treturn nil\n\t}\n\tout, err := m.clients.IAM().ListInstanceProfilesForRole(context.TODO(), &iam.ListInstanceProfilesForRoleInput{\n\t\tRoleName: aws.String(infra.nodeRoleName),\n\t})\n\tif err != nil {\n\t\tvar notFound *iamtypes.NoSuchEntityException\n\t\tif errors.As(err, ¬Found) {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to list instance profiles for role name: '%s': %v\", infra.nodeRoleName, err)\n\t} else if len(out.InstanceProfiles) > 0 {\n\t\tvar deletedInstanceProfiles []string\n\t\tfor _, instanceProfile := range out.InstanceProfiles {\n\t\t\t_, err := m.clients.IAM().RemoveRoleFromInstanceProfile(context.TODO(), &iam.RemoveRoleFromInstanceProfileInput{\n\t\t\t\tRoleName: aws.String(infra.nodeRoleName),\n\t\t\t\tInstanceProfileName: instanceProfile.InstanceProfileName,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tvar notFound *iamtypes.NoSuchEntityException\n\t\t\t\tif errors.As(err, ¬Found) {\n\t\t\t\t\tslog.Info(\"instance profile does not exist\", \"name\", aws.ToString(instanceProfile.InstanceProfileName))\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"failed to remove node role %s from instance profile: %s: %v\", infra.nodeRoleName, aws.ToString(instanceProfile.InstanceProfileName), err)\n\t\t\t}\n\t\t\t_, err = m.clients.IAM().DeleteInstanceProfile(context.TODO(), &iam.DeleteInstanceProfileInput{\n\t\t\t\tInstanceProfileName: instanceProfile.InstanceProfileName,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tvar notFound *iamtypes.NoSuchEntityException\n\t\t\t\tif errors.As(err, ¬Found) {\n\t\t\t\t\tslog.Info(\"instance profile does not exist\", \"name\", aws.ToString(instanceProfile.InstanceProfileName))\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"failed to delete instance profile: %s: %v\", aws.ToString(instanceProfile.InstanceProfileName), err)\n\t\t\t}\n\t\t\tdeletedInstanceProfiles = append(deletedInstanceProfiles, aws.ToString(instanceProfile.InstanceProfileName))\n\t\t}\n\t\tslog.Info(\"deleted leaked instance profiles\", \"count\", len(deletedInstanceProfiles), \"profiles\", deletedInstanceProfiles)\n\t}\n\treturn nil\n}\n\n// deleteLeakedENIs deletes Elastic Network Interfaces that may have been allocated (and left behind) by the VPC CNI.\n// These leaked ENIs will prevent deletion of their associated subnets and security groups.\nfunc (m *InfrastructureManager) deleteLeakedENIs() error {\n\tinfra, err := m.getInfrastructureStackResources()\n\tif err != nil {\n\t\tvar notFound *cloudformationtypes.StackNotFoundException\n\t\tif errors.As(err, ¬Found) {\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to get infrastructure stack resources: %w\", err)\n\t}\n\tenis, err := m.getVPCCNINetworkInterfaceIds(infra.vpc)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(enis) == 0 {\n\t\treturn nil\n\t}\n\tslog.Info(\"waiting for leaked ENIs to become available\", \"count\", len(enis), \"enis\", enis)\n\tif err := ec2.NewNetworkInterfaceAvailableWaiter(m.clients.EC2()).Wait(context.TODO(), &ec2.DescribeNetworkInterfacesInput{\n\t\tNetworkInterfaceIds: enis,\n\t}, networkInterfaceDetachmentTimeout); err != nil {\n\t\trefreshedENIs, err2 := m.getVPCCNINetworkInterfaceIds(infra.vpc)\n\t\tif err2 != nil {\n\t\t\treturn fmt.Errorf(\"waiter failed, and re-checking ENIs also failed: %w\", err2)\n\t\t}\n\t\tif len(refreshedENIs) == 0 {\n\t\t\tslog.Info(\"ENIs were deleted during waiter timeout, skipping delete\")\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to wait for ENI(s) to become available: %v\", err)\n\t}\n\tfor _, eni := range enis {\n\t\tslog.Info(\"deleting leaked ENI\", \"eni\", eni)\n\t\t_, err := m.clients.EC2().DeleteNetworkInterface(context.TODO(), &ec2.DeleteNetworkInterfaceInput{\n\t\t\tNetworkInterfaceId: aws.String(eni),\n\t\t})\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete leaked ENI: %w\", err)\n\t\t}\n\t}\n\tslog.Info(\"deleted leaked ENIs\", \"count\", len(enis))\n\tm.metrics.Record(infraLeakedENIs, float64(len(enis)), nil)\n\treturn nil\n}\n\n// getVPCCNINetworkInterfaceIds returns the IDs of ENIs in the specified VPC that were created by the VPC CNI\nfunc (m *InfrastructureManager) getVPCCNINetworkInterfaceIds(vpcId string) ([]string, error) {\n\tpaginator := ec2.NewDescribeNetworkInterfacesPaginator(m.clients.EC2(), &ec2.DescribeNetworkInterfacesInput{\n\t\tFilters: []ec2types.Filter{\n\t\t\t{\n\t\t\t\tName: aws.String(\"vpc-id\"),\n\t\t\t\tValues: []string{vpcId},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: aws.String(\"interface-type\"),\n\t\t\t\tValues: []string{\"interface\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: aws.String(\"tag-key\"),\n\t\t\t\tValues: []string{vpcCNIENITagKey, ipamControllerENITagKey},\n\t\t\t},\n\t\t},\n\t})\n\tvar enis []string\n\tfor paginator.HasMorePages() {\n\t\tpage, err := paginator.NextPage(context.TODO())\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to describe ENIs: %w\", err)\n\t\t}\n\t\tfor _, eni := range page.NetworkInterfaces {\n\t\t\tenis = append(enis, *eni.NetworkInterfaceId)\n\t\t}\n\t}\n\treturn enis, nil\n}\n\n// normalizeAZs removes availability zones that don't meet launch requirements\n// for instances and ensures that the resulting list containers enough AZs to\n// satisfy the deployment.\nfunc (m *InfrastructureManager) normalizeAZs(opts *deployerOptions, subnetAZs []string, expectedCount int) ([]string, error) {\n\tazs, err := m.clients.EC2().DescribeAvailabilityZones(context.TODO(), &ec2.DescribeAvailabilityZonesInput{\n\t\tFilters: []ec2types.Filter{\n\t\t\t{\n\t\t\t\tName: aws.String(\"zone-type\"),\n\t\t\t\tValues: []string{opts.ZoneType},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar supporttedAZs []string\n\tfor _, az := range azs.AvailabilityZones {\n\t\tsupporttedAZs = append(supporttedAZs, aws.ToString(az.ZoneName))\n\t}\n\n\tvar filteredAZs []string\n\tfor _, az := range subnetAZs {\n\t\tif slices.Contains(supporttedAZs, az) {\n\t\t\tfilteredAZs = append(filteredAZs, az)\n\t\t}\n\t}\n\n\t// enforce users' preferred ordering over AZs\n\tfilteredAZs = availabilityZoneHintedOrder(filteredAZs)\n\t// truncate the list if we went over the max\n\tfilteredAZs = filteredAZs[:min(len(filteredAZs), expectedCount)]\n\n\t// pad the availability zones with supported entries if we end up not having\n\t// enough after filtering.\n\tif len(filteredAZs) < expectedCount {\n\t\tfor _, az := range supporttedAZs {\n\t\t\tif len(filteredAZs) == expectedCount {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif !slices.Contains(filteredAZs, az) {\n\t\t\t\tslog.Info(\"padding infra stack with AZ\", \"az\", az)\n\t\t\t\tfilteredAZs = append(filteredAZs, az)\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(filteredAZs) != expectedCount {\n\t\treturn nil, fmt.Errorf(\"failed to provide AZs with expected count %d: %v\", expectedCount, filteredAZs)\n\t}\n\n\treturn filteredAZs, nil\n}\n\n// getAZsWithInstanceTypes returns the availability zones ordered decreasingly by the number of\n// requested instance types they support\nfunc (m *InfrastructureManager) getRankedAZsForInstanceTypes(opts *deployerOptions) ([]string, error) {\n\tofferings, err := m.clients.EC2().DescribeInstanceTypeOfferings(context.TODO(), &ec2.DescribeInstanceTypeOfferingsInput{\n\t\tLocationType: ec2types.LocationTypeAvailabilityZone,\n\t\tFilters: []ec2types.Filter{\n\t\t\t{\n\t\t\t\tName: aws.String(\"instance-type\"),\n\t\t\t\tValues: opts.InstanceTypes,\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to describe instance type offerings: %v\", err)\n\t}\n\tcounts := make(map[string]int)\n\tfor _, offering := range offerings.InstanceTypeOfferings {\n\t\tcounts[aws.ToString(offering.Location)]++\n\t}\n\tvar azs []string\n\tfor az := range counts {\n\t\tazs = append(azs, az)\n\t}\n\tsort.Slice(azs, func(i, j int) bool {\n\t\treturn counts[azs[i]] > counts[azs[j]]\n\t})\n\treturn azs, nil\n}\n\nfunc (m *InfrastructureManager) getAZsWithCapacity(opts *deployerOptions) ([]string, error) {\n\t// TODO: consolidate this with the CapacityReservation logic in node.go\n\tvar subnetAzs []string\n\tdescribeReservationsInput := ec2.DescribeCapacityReservationsInput{\n\t\tFilters: []ec2types.Filter{\n\t\t\t{\n\t\t\t\tName: aws.String(\"instance-type\"),\n\t\t\t\tValues: opts.InstanceTypes,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: aws.String(\"state\"),\n\t\t\t\tValues: []string{\"active\"},\n\t\t\t},\n\t\t},\n\t}\n\tif opts.TargetCapacityReservationId != \"\" {\n\t\tdescribeReservationsInput.CapacityReservationIds = []string{opts.TargetCapacityReservationId}\n\t}\n\tcapacityReservations, err := m.clients.EC2().DescribeCapacityReservations(context.TODO(), &describeReservationsInput)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, cr := range capacityReservations.CapacityReservations {\n\t\tif *cr.AvailableInstanceCount >= int32(opts.Nodes) {\n\t\t\tsubnetAzs = append(subnetAzs, *cr.AvailabilityZone)\n\t\t\tbreak\n\t\t}\n\t}\n\treturn subnetAzs, nil\n}\n\nfunc getCloudWatchStackName(resourceID string) (string, string) {\n\tclusterUUID := strings.TrimPrefix(resourceID, ResourcePrefix+\"-\")\n\treturn fmt.Sprintf(\"%s-cw\", resourceID), clusterUUID\n}\n\nfunc (m *InfrastructureManager) createCloudWatchInfrastructureStack(clusterName string) (string, error) {\n\tstackName, clusterUUID := getCloudWatchStackName(clusterName)\n\tslog.Info(\"creating CloudWatch infrastructure stack\", \"stackName\", stackName)\n\tout, err := m.clients.CFN().CreateStack(context.TODO(), &cloudformation.CreateStackInput{\n\t\tStackName: aws.String(stackName),\n\t\tTemplateBody: aws.String(templates.CloudWatchInfra),\n\t\tCapabilities: []cloudformationtypes.Capability{cloudformationtypes.CapabilityCapabilityNamedIam},\n\t\tParameters: []cloudformationtypes.Parameter{\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"ClusterUUID\"),\n\t\t\t\tParameterValue: aws.String(clusterUUID),\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to create CloudWatch infrastructure stack: %w\", err)\n\t}\n\n\tslog.Info(\"waiting for CloudWatch infrastructure stack to be created\", \"stackId\", *out.StackId)\n\tif err := cloudformation.NewStackCreateCompleteWaiter(m.clients.CFN()).\n\t\tWait(context.TODO(),\n\t\t\t&cloudformation.DescribeStacksInput{\n\t\t\t\tStackName: out.StackId,\n\t\t\t},\n\t\t\tinfraStackCreationTimeout); err != nil {\n\t\treturn \"\", util.WrapCFNStackFailure(context.TODO(), m.clients.CFN(), fmt.Errorf(\"failed to wait for CloudWatch infrastructure stack creation: %w\", err), stackName)\n\t}\n\n\tstack, err := m.clients.CFN().DescribeStacks(context.TODO(), &cloudformation.DescribeStacksInput{\n\t\tStackName: out.StackId,\n\t})\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to describe CloudWatch infrastructure stack: %w\", err)\n\t}\n\n\t// Get the CloudWatch role ARN from stack outputs\n\tvar roleArn string\n\tfor _, output := range stack.Stacks[0].Outputs {\n\t\tif aws.ToString(output.OutputKey) == \"CloudWatchRoleArn\" {\n\t\t\troleArn = aws.ToString(output.OutputValue)\n\t\t\tbreak\n\t\t}\n\t}\n\tif roleArn == \"\" {\n\t\treturn \"\", fmt.Errorf(\"CloudWatch role ARN not found in stack outputs\")\n\t}\n\n\tslog.Info(\"CloudWatch infrastructure stack created successfully\", \"roleArn\", roleArn)\n\treturn roleArn, nil\n}\n\n// createCloudWatchPodIdentityAssociation creates a PodIdentityAssociation\n// via the EKS API directly, rather than through CloudFormation, to ensure\n// the correct EKS endpoint is used when a custom endpoint URL is configured.\n// The association is automatically cleaned up when the cluster is deleted.\nfunc (m *InfrastructureManager) createCloudWatchPodIdentityAssociation(clusterName string, roleArn string) error {\n\tslog.Info(\"creating PodIdentityAssociation\", \"clusterName\", clusterName)\n\t_, err := m.clients.EKS().CreatePodIdentityAssociation(context.TODO(), &eks.CreatePodIdentityAssociationInput{\n\t\tClusterName: aws.String(clusterName),\n\t\tNamespace: aws.String(\"amazon-cloudwatch\"),\n\t\tServiceAccount: aws.String(\"cwagent\"),\n\t\tRoleArn: aws.String(roleArn),\n\t})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create PodIdentityAssociation: %w\", err)\n\t}\n\tslog.Info(\"PodIdentityAssociation created successfully\", \"clusterName\", clusterName)\n\treturn nil\n}\n\nfunc (m *InfrastructureManager) deleteCloudWatchInfrastructureStack() error {\n\tstackName, _ := getCloudWatchStackName(m.resourceID)\n\n\t// The PodIdentityAssociation created via the EKS API is automatically\n\t// cleaned up when the cluster is deleted, so no explicit deletion is needed.\n\n\tslog.Info(\"deleting CloudWatch infrastructure stack\", \"stackName\", stackName)\n\tif _, err := m.clients.CFN().DeleteStack(context.TODO(), &cloudformation.DeleteStackInput{\n\t\tStackName: aws.String(stackName),\n\t}); err != nil {\n\t\tvar notFound *cloudformationtypes.StackNotFoundException\n\t\tif errors.As(err, ¬Found) {\n\t\t\tslog.Info(\"CloudWatch infrastructure stack does not exist\", \"stackName\", stackName)\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to delete CloudWatch infrastructure stack: %w\", err)\n\t}\n\n\tslog.Info(\"initiated deletion of CloudWatch infrastructure stack\", \"stackName\", stackName)\n\treturn nil\n}\n" }, { "path": "internal/deployers/eksapi/janitor.go", "content": "package eksapi\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/awssdk\"\n\t\"github.com/aws/aws-k8s-tester/internal/metrics\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudformation\"\n\tcloudformationtypes \"github.com/aws/aws-sdk-go-v2/service/cloudformation/types\"\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudwatch\"\n)\n\nfunc NewJanitor(maxResourceAge time.Duration, emitMetrics bool, workers int, stackStatus string) *janitor {\n\tawsConfig := awssdk.NewConfig()\n\tvar metricRegistry metrics.MetricRegistry\n\tif emitMetrics {\n\t\tmetricRegistry = metrics.NewCloudWatchRegistry(cloudwatch.NewFromConfig(awsConfig))\n\t} else {\n\t\tmetricRegistry = metrics.NewNoopMetricRegistry()\n\t}\n\tif workers <= 0 {\n\t\tworkers = 1\n\t}\n\treturn &janitor{\n\t\tmaxResourceAge: maxResourceAge,\n\t\tworkers: workers,\n\t\tstackStatus: stackStatus,\n\t\tawsConfig: awsConfig,\n\t\tcfnClient: cloudformation.NewFromConfig(awsConfig),\n\t\tmetrics: metricRegistry,\n\t}\n}\n\ntype janitor struct {\n\tmaxResourceAge time.Duration\n\tworkers int\n\tstackStatus string\n\n\tawsConfig aws.Config\n\tcfnClient *cloudformation.Client\n\tmetrics metrics.MetricRegistry\n}\n\nfunc (j *janitor) Sweep(ctx context.Context) error {\n\tawsConfig := awssdk.NewConfig()\n\tcfnClient := cloudformation.NewFromConfig(awsConfig)\n\tstacks, err := j.getStacks(ctx, cfnClient)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to get stacks: %v\", err)\n\t}\n\tvar wg sync.WaitGroup\n\tstackQueue := make(chan cloudformationtypes.Stack, len(stacks))\n\terrChan := make(chan error, len(stacks))\n\tfor i := 1; i <= j.workers; i++ {\n\t\twg.Add(1)\n\t\tgo j.sweepWorker(&wg, stackQueue, errChan)\n\t}\n\n\tfor _, stack := range stacks {\n\t\tstackQueue <- stack\n\t}\n\tclose(stackQueue)\n\n\twg.Wait()\n\tclose(errChan)\n\tvar errs []error\n\tfor err := range errChan {\n\t\terrs = append(errs, err)\n\t}\n\treturn errors.Join(errs...)\n}\n\nfunc (j *janitor) getStacks(ctx context.Context, cfnClient *cloudformation.Client) ([]cloudformationtypes.Stack, error) {\n\tvar stacks []cloudformationtypes.Stack\n\tstackPaginator := cloudformation.NewDescribeStacksPaginator(cfnClient, &cloudformation.DescribeStacksInput{})\n\tfor stackPaginator.HasMorePages() {\n\t\tpage, err := stackPaginator.NextPage(ctx)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tstacks = append(stacks, page.Stacks...)\n\t}\n\treturn stacks, nil\n}\n\nfunc (j *janitor) sweepWorker(wg *sync.WaitGroup, stackQueue <-chan cloudformationtypes.Stack, errChan chan<- error) {\n\tdefer wg.Done()\n\tfor stack := range stackQueue {\n\t\tresourceID := *stack.StackName\n\t\tif !strings.HasPrefix(resourceID, ResourcePrefix) {\n\t\t\tcontinue\n\t\t}\n\t\tif stack.StackStatus == \"DELETE_COMPLETE\" {\n\t\t\tcontinue\n\t\t}\n\t\tif j.stackStatus != \"\" && j.stackStatus != string(stack.StackStatus) {\n\t\t\tslog.Info(\"skipping resources\", \"status\", stack.StackStatus, \"resourceID\", resourceID)\n\t\t\tcontinue\n\t\t}\n\t\tresourceAge := time.Since(*stack.CreationTime)\n\t\tif resourceAge < j.maxResourceAge {\n\t\t\tslog.Info(\"skipping resources\", \"age\", resourceAge, \"resourceID\", resourceID)\n\t\t\tcontinue\n\t\t}\n\t\tclients := j.awsClientsForStack(stack)\n\t\tinfraManager := NewInfrastructureManager(clients, resourceID, j.metrics)\n\t\tclusterManager := NewClusterManager(clients, resourceID)\n\t\tnodeManager := NewNodeManager(clients, resourceID)\n\t\tslog.Info(\"deleting resources\", \"age\", resourceAge, \"resourceID\", resourceID)\n\t\tif err := deleteResources(infraManager, clusterManager, nodeManager, nil /* k8sClient */, nil /* deployerOptions */); err != nil {\n\t\t\terrChan <- fmt.Errorf(\"failed to delete resources: %s: %v\", resourceID, err)\n\t\t}\n\t}\n}\n\nfunc (j *janitor) awsClientsForStack(stack cloudformationtypes.Stack) *awsClients {\n\tvar eksEndpointURL string\n\tfor _, tag := range stack.Tags {\n\t\tif *tag.Key == eksEndpointURLTag {\n\t\t\teksEndpointURL = *tag.Value\n\t\t}\n\t}\n\treturn newAWSClients(j.awsConfig, eksEndpointURL)\n}\n" }, { "path": "internal/deployers/eksapi/k8s.go", "content": "package eksapi\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"net\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/metrics\"\n\t\"github.com/aws/aws-k8s-tester/internal/util\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ec2\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/util/sets\"\n\t\"k8s.io/apimachinery/pkg/util/wait\"\n\t\"k8s.io/apimachinery/pkg/watch\"\n\t\"k8s.io/client-go/dynamic\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n\t\"k8s.io/client-go/tools/clientcmd\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n\tcrlog \"sigs.k8s.io/controller-runtime/pkg/log\"\n\t\"sigs.k8s.io/controller-runtime/pkg/log/zap\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n)\n\nfunc init() {\n\t// controller-runtime will complain loudly if this isn't set, even though we don't use this logger\n\tcrlog.SetLogger(zap.New())\n}\n\nconst (\n\trequestRetryInterval = 10 * time.Second\n\trequestRetryTimeout = 10 * time.Minute\n)\n\ntype k8sClient struct {\n\tconfig *rest.Config\n\tclientset kubernetes.Interface\n\tclient client.Client\n\tdclient *dynamic.DynamicClient\n}\n\nfunc newK8sClient(kubeconfigPath string) (*k8sClient, error) {\n\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigPath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &k8sClient{\n\t\tconfig: config,\n\t\tclientset: kubernetes.NewForConfigOrDie(config),\n\t\tclient: util.Must(client.New(config, client.Options{})),\n\t\tdclient: util.Must(dynamic.NewForConfig(config)),\n\t}, nil\n}\n\nfunc (k *k8sClient) waitForReadyNodes(nodeCount int, timeout time.Duration) error {\n\tslog.Info(\"waiting for nodes to be ready\", \"nodeCount\", nodeCount, \"timeout\", timeout)\n\treadyNodes := sets.NewString()\n\twatcher, err := k.clientset.CoreV1().Nodes().Watch(context.TODO(), metav1.ListOptions{})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create node watcher: %v\", err)\n\t}\n\tdefer watcher.Stop()\n\tinitialReadyNodes, err := k.getReadyNodes()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to get ready nodes: %v\", err)\n\t}\n\tcounter := len(initialReadyNodes)\n\tctx, cancel := context.WithTimeout(context.Background(), timeout)\n\tdefer cancel()\n\tfor {\n\t\tselect {\n\t\tcase event, ok := <-watcher.ResultChan():\n\t\t\tif !ok {\n\t\t\t\treturn fmt.Errorf(\"the watcher channel for the nodes was closed by Kubernetes due to an unknown error\")\n\t\t\t}\n\t\t\tif event.Type == watch.Error {\n\t\t\t\tmsg := \"unexpected error event type from node watcher\"\n\t\t\t\tif statusErr, ok := event.Object.(*metav1.Status); ok {\n\t\t\t\t\treturn fmt.Errorf(\"%s: %s\", msg, statusErr.String())\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"%s: %+v\", msg, event.Object)\n\t\t\t}\n\t\t\tif event.Object != nil && event.Type != watch.Deleted {\n\t\t\t\tif node, ok := event.Object.(*corev1.Node); ok {\n\t\t\t\t\tif isNodeReady(node) {\n\t\t\t\t\t\treadyNodes.Insert(node.Name)\n\t\t\t\t\t\tcounter = readyNodes.Len()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tcase <-ctx.Done():\n\t\t\treturn fmt.Errorf(\"timed out waiting for %d nodes to be ready: %w\", nodeCount, ctx.Err())\n\t\t}\n\t\tif counter >= nodeCount {\n\t\t\tbreak\n\t\t}\n\t}\n\tslog.Info(\"nodes are ready\", \"count\", readyNodes.Len(), \"nodes\", readyNodes)\n\treturn nil\n}\n\nfunc (k *k8sClient) waitForNodeDeletion(timeout time.Duration) error {\n\tslog.Info(\"waiting for nodes to be deleted\", \"timeout\", timeout)\n\tnodes := sets.NewString()\n\twatcher, err := k.clientset.CoreV1().Nodes().Watch(context.TODO(), metav1.ListOptions{})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create node watcher: %v\", err)\n\t}\n\tdefer watcher.Stop()\n\tinitialNodes, err := k.clientset.CoreV1().Nodes().List(context.TODO(), v1.ListOptions{})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to list nodes: %v\", err)\n\t}\n\tfor _, node := range initialNodes.Items {\n\t\tnodes.Insert(node.Name)\n\t}\n\tctx, cancelFunc := context.WithTimeout(context.Background(), timeout)\n\tdefer cancelFunc()\n\tfor {\n\t\tselect {\n\t\tcase event, ok := <-watcher.ResultChan():\n\t\t\tif !ok {\n\t\t\t\treturn fmt.Errorf(\"the watcher channel for the nodes was closed by Kubernetes due to an unknown error\")\n\t\t\t}\n\t\t\tif event.Type == watch.Error {\n\t\t\t\tmsg := \"unexpected error event type from node watcher\"\n\t\t\t\tif statusErr, ok := event.Object.(*metav1.Status); ok {\n\t\t\t\t\treturn fmt.Errorf(\"%s: %s\", msg, statusErr.String())\n\t\t\t\t}\n\t\t\t\treturn fmt.Errorf(\"%s: %+v\", msg, event.Object)\n\t\t\t}\n\t\t\tif event.Object != nil {\n\t\t\t\tif node, ok := event.Object.(*corev1.Node); !ok {\n\t\t\t\t\treturn fmt.Errorf(\"node watcher received an object that isn't a Node: %+v\", event.Object)\n\t\t\t\t} else {\n\t\t\t\t\tswitch event.Type {\n\t\t\t\t\tcase watch.Added:\n\t\t\t\t\t\tnodes.Insert(node.Name)\n\t\t\t\t\tcase watch.Deleted:\n\t\t\t\t\t\tnodes.Delete(node.Name)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\tcase <-ctx.Done():\n\t\t\treturn fmt.Errorf(\"timed out waiting for nodes to be deleted: %w\", ctx.Err())\n\t\t}\n\t\tif len(nodes) == 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\tslog.Info(\"all nodes deleted!\")\n\treturn nil\n}\n\nfunc (k *k8sClient) getReadyNodes() ([]corev1.Node, error) {\n\tnodes, err := k.clientset.CoreV1().Nodes().List(context.TODO(), v1.ListOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar readyNodes []corev1.Node\n\tfor _, node := range nodes.Items {\n\t\tif isNodeReady(&node) {\n\t\t\treadyNodes = append(readyNodes, node)\n\t\t}\n\t}\n\treturn readyNodes, nil\n}\n\nfunc isNodeReady(node *corev1.Node) bool {\n\tc := getNodeReadyCondition(node)\n\tif c == nil {\n\t\treturn false\n\t}\n\treturn c.Status == corev1.ConditionTrue\n}\n\nfunc getNodeReadyCondition(node *corev1.Node) *corev1.NodeCondition {\n\tfor _, c := range node.Status.Conditions {\n\t\tif c.Type == corev1.NodeReady {\n\t\t\treturn &c\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (k *k8sClient) createAWSAuthConfigMap(nodeNameStrategy string, nodeRoleARN string) error {\n\tmapRoles, err := generateAuthMapRole(nodeNameStrategy, nodeRoleARN)\n\tif err != nil {\n\t\treturn err\n\t}\n\tslog.Info(\"generated AuthMapRole\", \"mapRoles\", mapRoles)\n\terr = wait.PollUntilContextTimeout(context.TODO(), requestRetryInterval, requestRetryTimeout, true, func(ctx context.Context) (bool, error) {\n\t\t_, err := k.clientset.CoreV1().ConfigMaps(\"kube-system\").Create(ctx, &corev1.ConfigMap{\n\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\tName: \"aws-auth\",\n\t\t\t\tNamespace: \"kube-system\",\n\t\t\t},\n\t\t\tData: map[string]string{\n\t\t\t\t\"mapRoles\": mapRoles,\n\t\t\t},\n\t\t}, metav1.CreateOptions{})\n\t\tif err != nil {\n\t\t\tvar dnsErr *net.DNSError\n\t\t\tif errors.As(err, &dnsErr) {\n\t\t\t\tslog.Warn(\"failed to create aws-auth configmap due to DNS error, retrying\", \"error\", err)\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t\treturn false, err\n\t\t}\n\t\treturn true, nil\n\t})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"retry loop failed: %w\", err)\n\t}\n\treturn nil\n}\n\nfunc getNodeInstanceIDs(nodes []corev1.Node) ([]string, error) {\n\tvar instanceIds []string\n\tvar errs []error\n\tfor _, node := range nodes {\n\t\tproviderId, err := parseKubernetesProviderID(node.Spec.ProviderID)\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t\tcontinue\n\t\t}\n\t\tinstanceIds = append(instanceIds, providerId.InstanceID)\n\t}\n\tif len(errs) > 0 {\n\t\treturn nil, errors.Join(errs...)\n\t}\n\treturn instanceIds, nil\n}\n\nfunc (k *k8sClient) emitNodeMetrics(metricRegistry metrics.MetricRegistry, ec2Client *ec2.Client) error {\n\tnodes, err := k.getReadyNodes()\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar errs []error\n\tfor _, node := range nodes {\n\t\tproviderId, err := parseKubernetesProviderID(node.Spec.ProviderID)\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t\tcontinue\n\t\t}\n\t\tinstanceInfo, err := ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{\n\t\t\tInstanceIds: []string{providerId.InstanceID},\n\t\t})\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t\tcontinue\n\t\t}\n\t\tinstance := instanceInfo.Reservations[0].Instances[0]\n\t\tlaunchTime := *instance.LaunchTime\n\t\ttimeToRegistration := node.ObjectMeta.CreationTimestamp.Time.Sub(launchTime)\n\t\ttimeToReady := getNodeReadyCondition(&node).LastTransitionTime.Time.Sub(launchTime)\n\n\t\tnodeDimensions := map[string]string{\n\t\t\t\"instanceType\": string(instance.InstanceType),\n\t\t\t\"os\": node.Status.NodeInfo.OperatingSystem,\n\t\t\t\"osImage\": node.Status.NodeInfo.OSImage,\n\t\t\t\"arch\": node.Status.NodeInfo.Architecture,\n\t\t}\n\n\t\t// we'll emit the metrics with different subset(s) of dimensions, to make aggregation simpler\n\t\tvar nodeDimensionSets []map[string]string\n\t\tnodeDimensionSets = append(nodeDimensionSets, nodeDimensions)\n\n\t\tvar osDistro string\n\t\tif strings.HasPrefix(node.Status.NodeInfo.OSImage, \"Amazon Linux\") {\n\t\t\t// on al2: \"Amazon Linux 2\"\n\t\t\t// on al2023: \"Amazon Linux 2023.6.20241010\"\n\t\t\tparts := strings.Split(node.Status.NodeInfo.OSImage, \".\")\n\t\t\tamazonLinuxMajorVersion := parts[0]\n\t\t\tosDistro = amazonLinuxMajorVersion\n\t\t}\n\n\t\tif osDistro != \"\" {\n\t\t\tnodeDimensions[\"osDistro\"] = osDistro\n\n\t\t\t// if we have an osDistro, add a pared-down dimension set that includes it\n\t\t\tnodeDimensionSets = append(nodeDimensionSets, map[string]string{\n\t\t\t\t\"osDistro\": nodeDimensions[\"osDistro\"],\n\t\t\t\t\"instanceType\": nodeDimensions[\"instanceType\"],\n\t\t\t\t\"arch\": nodeDimensions[\"arch\"],\n\t\t\t})\n\t\t}\n\n\t\tfor _, nodeDimensionSet := range nodeDimensionSets {\n\t\t\tmetricRegistry.Record(nodeTimeToRegistrationSeconds, timeToRegistration.Seconds(), nodeDimensionSet)\n\t\t\tmetricRegistry.Record(nodeTimeToReadySeconds, timeToReady.Seconds(), nodeDimensionSet)\n\t\t}\n\t}\n\treturn errors.Join(errs...)\n}\n\ntype KubernetesProviderID struct {\n\tAvailabilityZone string\n\tInstanceID string\n}\n\nfunc parseKubernetesProviderID(rawProviderId string) (*KubernetesProviderID, error) {\n\turl, err := url.Parse(rawProviderId)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"malformed provider ID: %s\", rawProviderId)\n\t}\n\tif url.Scheme != \"aws\" {\n\t\treturn nil, fmt.Errorf(\"usupported provider ID scheme: %s\", url.Scheme)\n\t}\n\tif url.Path == \"\" {\n\t\treturn nil, fmt.Errorf(\"provider ID path is empty: %s\", rawProviderId)\n\t}\n\t// example: /us-west-2a/i-12345abcdefg\n\tparts := strings.Split(url.Path, \"/\")\n\tif len(parts) != 3 {\n\t\treturn nil, fmt.Errorf(\"provider ID path does not have 3 parts: %s\", url.Path)\n\t}\n\treturn &KubernetesProviderID{\n\t\tAvailabilityZone: parts[1],\n\t\tInstanceID: parts[2],\n\t}, nil\n}\n" }, { "path": "internal/deployers/eksapi/kubeconfig.go", "content": "package eksapi\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"os\"\n\t\"text/template\"\n)\n\nconst kubeconfigPerm = 0666\n\nvar kubeconfigTemplate = `---\napiVersion: v1\nkind: Config\nclusters:\n- cluster:\n certificate-authority-data: {{ .ClusterCertificateAuthority }}\n server: {{ .ClusterEndpoint }}\n name: {{ .ClusterARN }}\ncontexts:\n- context:\n cluster: {{ .ClusterARN }}\n user: {{ .ClusterARN }}\n name: {{ .ClusterARN }}\ncurrent-context: {{ .ClusterARN }}\npreferences: {}\nusers:\n- name: {{ .ClusterARN }}\n user:\n exec:\n apiVersion: client.authentication.k8s.io/v1beta1\n command: aws\n args:\n - eks\n - get-token\n - --cluster-name\n - {{ .ClusterName }}\n`\n\ntype kubeconfigTemplateParameters struct {\n\tClusterCertificateAuthority string\n\tClusterARN string\n\tClusterEndpoint string\n\tClusterName string\n}\n\nfunc writeKubeconfig(cluster *Cluster, kubeconfigPath string) error {\n\tif cluster == nil {\n\t\treturn fmt.Errorf(\"Cluster is nil, you might need set --static-cluster-name or set --up to initial cluster resrouces\")\n\t}\n\tslog.Info(\"writing kubeconfig\", \"path\", kubeconfigPath, \"clusterArn\", cluster.arn)\n\ttemplateParams := kubeconfigTemplateParameters{\n\t\tClusterCertificateAuthority: cluster.certificateAuthorityData,\n\t\tClusterARN: cluster.arn,\n\t\tClusterEndpoint: cluster.endpoint,\n\t\tClusterName: cluster.name,\n\t}\n\n\tkubeconfig := bytes.Buffer{}\n\n\tt, err := template.New(\"kubeconfig\").Parse(kubeconfigTemplate)\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = t.Execute(&kubeconfig, templateParams)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = os.WriteFile(kubeconfigPath, kubeconfig.Bytes(), kubeconfigPerm)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tslog.Info(\"wrote kubeconfig\", \"path\", kubeconfigPath, \"content\", kubeconfig.String())\n\treturn nil\n}\n" }, { "path": "internal/deployers/eksapi/logs.go", "content": "package eksapi\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"errors\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"slices\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/autoscaling\"\n\t\"github.com/aws/aws-sdk-go-v2/service/s3\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ssm\"\n\tssmtypes \"github.com/aws/aws-sdk-go-v2/service/ssm/types\"\n\tv1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured\"\n\t\"k8s.io/apimachinery/pkg/util/wait\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n)\n\ntype logManager struct {\n\tclients *awsClients\n\tresourceID string\n}\n\ntype deployerPhase string\n\nconst (\n\tdeployerPhaseUp = \"up\"\n\tdeployerPhaseDown = \"down\"\n)\n\nfunc NewLogManager(clients *awsClients, resourceID string) *logManager {\n\treturn &logManager{\n\t\tclients: clients,\n\t\tresourceID: resourceID,\n\t}\n}\n\nfunc (m *logManager) gatherLogsFromNodes(k8sClient *k8sClient, opts *deployerOptions, phase deployerPhase) error {\n\tif opts.LogBucket == \"\" {\n\t\tslog.Info(\"--log-bucket is empty, no logs will be gathered!\")\n\t\treturn nil\n\t}\n\tif k8sClient == nil {\n\t\tslog.Info(\"no k8s client available, no logs will be gathered!\")\n\t\treturn nil\n\t}\n\tif opts.AutoMode {\n\t\treturn m.gatherLogsUsingNodeDiagnostic(k8sClient, opts, phase)\n\t}\n\tswitch opts.UserDataFormat {\n\tcase \"bootstrap.sh\", \"nodeadm\", \"\": // if no --user-data-format was passed, we must be using managed nodes, which default to AL-based AMIs\n\t\treturn m.gatherLogsUsingScript(k8sClient, opts, phase)\n\tdefault:\n\t\tslog.Warn(\"unable to gather logs for userDataFormat\", \"format\", opts.UserDataFormat)\n\t\treturn nil\n\t}\n}\n\n//go:embed logs_ssm_doc.json\nvar logCollectorScriptSsmDocumentContent string\n\nconst logCollectorSsmDocumentTimeout = 5 * time.Minute\n\nfunc (m *logManager) gatherLogsUsingScript(k8sClient *k8sClient, opts *deployerOptions, phase deployerPhase) error {\n\tslog.Info(\"gathering logs using script...\")\n\tnodes, err := k8sClient.clientset.CoreV1().Nodes().List(context.TODO(), v1.ListOptions{})\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar instanceIds []string\n\tif len(nodes.Items) > 0 {\n\t\tinstanceIds, err = getNodeInstanceIDs(nodes.Items)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\tslog.Warn(\"no nodes found in cluster!\")\n\t\t// if we're using unmanaged nodes, we can track down the instances in the ASG even if they didn't join the cluster\n\t\tif opts.UnmanagedNodes {\n\t\t\tslog.Info(\"fetching instances from unmanaged nodegroup...\")\n\t\t\tout, err := m.clients.ASG().DescribeAutoScalingGroups(context.TODO(), &autoscaling.DescribeAutoScalingGroupsInput{\n\t\t\t\tAutoScalingGroupNames: []string{m.resourceID},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tslog.Warn(\"failed to describe unmanaged nodegroup ASG\", \"error\", err)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif len(out.AutoScalingGroups) != 1 {\n\t\t\t\tslog.Warn(\"autoscaling group not found\", \"resourceID\", m.resourceID)\n\t\t\t} else {\n\t\t\t\tfor _, asg := range out.AutoScalingGroups {\n\t\t\t\t\tfor _, instance := range asg.Instances {\n\t\t\t\t\t\tinstanceIds = append(instanceIds, aws.ToString(instance.InstanceId))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif len(instanceIds) == 0 {\n\t\tslog.Warn(\"no nodes to gather logs from!\")\n\t\treturn nil\n\t}\n\tdoc, err := m.clients.SSM().CreateDocument(context.TODO(), &ssm.CreateDocumentInput{\n\t\tContent: aws.String(logCollectorScriptSsmDocumentContent),\n\t\tName: aws.String(fmt.Sprintf(\"%s-log-collector\", m.resourceID)),\n\t\tDocumentType: ssmtypes.DocumentTypeCommand,\n\t\tDocumentFormat: ssmtypes.DocumentFormatJson,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tm.clients.SSM().DeleteDocument(context.TODO(), &ssm.DeleteDocumentInput{\n\t\t\tName: doc.DocumentDescription.Name,\n\t\t})\n\t}()\n\tcommand, err := m.clients.SSM().SendCommand(context.TODO(), &ssm.SendCommandInput{\n\t\tDocumentName: doc.DocumentDescription.Name,\n\t\tInstanceIds: instanceIds,\n\t\tParameters: map[string][]string{\n\t\t\t\"s3Destination\": {fmt.Sprintf(\"s3://%s/node-logs/%s/%s/\", opts.LogBucket, m.resourceID, phase)},\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar errs []error\n\tfor _, instanceId := range instanceIds {\n\t\tout, err := ssm.NewCommandExecutedWaiter(m.clients.SSM()).WaitForOutput(context.TODO(), &ssm.GetCommandInvocationInput{\n\t\t\tCommandId: command.Command.CommandId,\n\t\t\tInstanceId: aws.String(instanceId),\n\t\t}, logCollectorSsmDocumentTimeout)\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t} else {\n\t\t\tslog.Info(\"log collection command completed\", \"instanceId\", instanceId, \"status\", out.Status)\n\t\t}\n\t}\n\tif len(errs) > 0 {\n\t\treturn errors.Join(errs...)\n\t}\n\tslog.Info(\"gathered logs from nodes\", \"instanceIds\", instanceIds)\n\treturn nil\n}\n\nconst logCollectorNodeDiagnosticTimeout = 5 * time.Minute\n\nfunc (m *logManager) gatherLogsUsingNodeDiagnostic(k8sClient *k8sClient, opts *deployerOptions, phase deployerPhase) error {\n\tslog.Info(\"gathering logs using NodeDiagnostic...\")\n\tnodes, err := k8sClient.clientset.CoreV1().Nodes().List(context.TODO(), v1.ListOptions{})\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(nodes.Items) == 0 {\n\t\tslog.Warn(\"no nodes to gather logs from!\")\n\t\treturn nil\n\t}\n\tinstanceIds, err := getNodeInstanceIDs(nodes.Items)\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar errs []error\n\tvar nodeDiagnostics []unstructured.Unstructured\n\tfor _, instanceId := range instanceIds {\n\t\tpresignedPut, err := m.clients.S3Presign().PresignPutObject(context.TODO(), &s3.PutObjectInput{\n\t\t\tBucket: aws.String(opts.LogBucket),\n\t\t\tKey: aws.String(fmt.Sprintf(\"node-logs/%s/%s/%s.tar.gz\", m.resourceID, phase, instanceId)),\n\t\t})\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to create presigned PUT for %s: %v\", instanceId, err))\n\t\t\tcontinue\n\t\t}\n\t\tnodeDiagnostic := unstructured.Unstructured{\n\t\t\tObject: map[string]interface{}{\n\t\t\t\t\"apiVersion\": \"eks.amazonaws.com/v1alpha1\",\n\t\t\t\t\"kind\": \"NodeDiagnostic\",\n\t\t\t\t\"metadata\": v1.ObjectMeta{\n\t\t\t\t\tName: instanceId,\n\t\t\t\t},\n\t\t\t\t\"spec\": map[string]interface{}{\n\t\t\t\t\t\"logCapture\": map[string]interface{}{\n\t\t\t\t\t\t\"destination\": presignedPut.URL,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t\tif err := k8sClient.client.Create(context.TODO(), &nodeDiagnostic); err != nil {\n\t\t\terrs = append(errs, err)\n\t\t} else {\n\t\t\tnodeDiagnostics = append(nodeDiagnostics, nodeDiagnostic)\n\t\t}\n\t}\n\toutcomes, err := m.waitForNodeDiagnostics(k8sClient, nodeDiagnostics)\n\tif err != nil {\n\t\terrs = append(errs, fmt.Errorf(\"failed to wait for node diagnostics: %v\", err))\n\t}\n\tfor instanceId, reasons := range outcomes {\n\t\tfor _, reason := range reasons {\n\t\t\t// consider SuccessWithErrors a success, this isn't high stakes\n\t\t\tif !slices.Contains([]string{\"Success\", \"SuccessWithErrors\"}, reason) {\n\t\t\t\terrs = append(errs, fmt.Errorf(\"node diagnostic outcome reason for %s: %s\", instanceId, reason))\n\t\t\t}\n\t\t}\n\t}\n\tfor _, nodeDiagnostic := range nodeDiagnostics {\n\t\tif err := k8sClient.client.Delete(context.TODO(), &nodeDiagnostic); err != nil {\n\t\t\terrs = append(errs, err)\n\t\t}\n\t}\n\tif len(errs) > 0 {\n\t\treturn errors.Join(errs...)\n\t}\n\tslog.Info(\"gathered logs from nodes\", \"instanceIds\", instanceIds)\n\treturn nil\n}\n\n// waitForNodeDiagnostics polls each node diagnostic until it reaches a terminal state, or the timeout is reached\n// a map of node diagnostic names to their outcome reason(s) is returned if no error occurred\nfunc (m *logManager) waitForNodeDiagnostics(k8sClient *k8sClient, nodeDiagnostics []unstructured.Unstructured) (map[string][]string, error) {\n\toutcomes := make(map[string][]string)\n\terr := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, logCollectorNodeDiagnosticTimeout, false, func(ctx context.Context) (done bool, err error) {\n\t\tfor _, nodeDiagnostic := range nodeDiagnostics {\n\t\t\tobjectKey := client.ObjectKeyFromObject(&nodeDiagnostic)\n\t\t\tif _, ok := outcomes[objectKey.Name]; ok {\n\t\t\t\t// we already have an outcome for this node diagnostic\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err := k8sClient.client.Get(ctx, objectKey, &nodeDiagnostic); err != nil {\n\t\t\t\treturn false, fmt.Errorf(\"failed to get node diagnostic: %+v: %v\", objectKey, err)\n\t\t\t}\n\t\t\tcomplete, reasons := m.isNodeDiagnosticComplete(&nodeDiagnostic)\n\t\t\tif !complete {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\toutcomes[objectKey.Name] = reasons\n\t\t}\n\t\tif len(outcomes) == len(nodeDiagnostics) {\n\t\t\t// we're done!\n\t\t\treturn true, nil\n\t\t}\n\t\treturn false, nil\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn outcomes, nil\n}\n\nfunc (m *logManager) isNodeDiagnosticComplete(nodeDiagnostic *unstructured.Unstructured) (bool, []string) {\n\tcaptureStatuses, found, err := unstructured.NestedSlice(nodeDiagnostic.Object, \"status\", \"captureStatuses\")\n\tif err != nil {\n\t\tslog.Error(\"NodeDiagnostic captureStatuses does not match expected type\", \"nodeDiagnostic\", nodeDiagnostic)\n\t\treturn false, nil\n\t}\n\tif !found {\n\t\treturn false, nil\n\t}\n\tvar reasons []string\n\tfor _, captureStatus := range captureStatuses {\n\t\tcaptureStatusMap, ok := captureStatus.(map[string]interface{})\n\t\tif !ok {\n\t\t\tslog.Error(\"NodeDiagnostic captureStatus does not match expected type\", \"nodeDiagnostic\", nodeDiagnostic)\n\t\t\treturn false, nil\n\t\t}\n\t\treason, found, err := unstructured.NestedString(captureStatusMap, \"state\", \"completed\", \"reason\")\n\t\tif err != nil {\n\t\t\tslog.Error(\"NodeDiagnostic captureStatus.reason does not match expected type\", \"nodeDiagnostic\", nodeDiagnostic)\n\t\t\treturn false, nil\n\t\t}\n\t\tif !found {\n\t\t\treturn false, nil\n\t\t}\n\t\treasons = append(reasons, reason)\n\t}\n\treturn true, reasons\n}\n" }, { "path": "internal/deployers/eksapi/logs_ssm_doc.json", "content": "{\n \"schemaVersion\": \"2.2\",\n \"description\": \"Collect logs from an Amazon Linux EKS node\",\n \"parameters\": {\n \"s3Destination\": {\n \"type\": \"String\"\n }\n },\n \"mainSteps\": [\n {\n \"action\": \"aws:runShellScript\",\n \"name\": \"collectAndUploadLogs\",\n \"precondition\": {\n \"StringEquals\": [\n \"platformType\",\n \"Linux\"\n ]\n },\n \"inputs\": {\n \"runCommand\": [\n \"bash /etc/eks/log-collector-script/eks-log-collector.sh >/dev/null 2>&1\",\n \"aws s3 cp /var/log/eks_i* {{s3Destination}}\"\n ]\n }\n }\n ]\n}" }, { "path": "internal/deployers/eksapi/metrics.go", "content": "package eksapi\n\nimport (\n\t\"path\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/metrics\"\n\tcloudwatchtypes \"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types\"\n)\n\nvar DeployerMetricNamespace = path.Join(\"kubetest2\", DeployerName)\n\nvar (\n\ttotalRuntimeSeconds = &metrics.MetricSpec{\n\t\tNamespace: DeployerMetricNamespace,\n\t\tMetric: \"TotalRuntimeSeconds\",\n\t\tUnit: cloudwatchtypes.StandardUnitSeconds,\n\t}\n\n\tnodeTimeToRegistrationSeconds = &metrics.MetricSpec{\n\t\tNamespace: DeployerMetricNamespace,\n\t\tMetric: \"NodeTimeToRegistrationSeconds\",\n\t\tUnit: cloudwatchtypes.StandardUnitSeconds,\n\t}\n\n\tnodeTimeToReadySeconds = &metrics.MetricSpec{\n\t\tNamespace: DeployerMetricNamespace,\n\t\tMetric: \"NodeTimeToReadySeconds\",\n\t\tUnit: cloudwatchtypes.StandardUnitSeconds,\n\t}\n)\n" }, { "path": "internal/deployers/eksapi/node.go", "content": "package eksapi\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t_ \"embed\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/autoscaling\"\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudformation\"\n\tcloudformationtypes \"github.com/aws/aws-sdk-go-v2/service/cloudformation/types\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ec2\"\n\tec2types \"github.com/aws/aws-sdk-go-v2/service/ec2/types\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks\"\n\tekstypes \"github.com/aws/aws-sdk-go-v2/service/eks/types\"\n\t\"github.com/aws/smithy-go\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/runtime/schema\"\n\t\"log/slog\"\n\t\"k8s.io/utils/pointer\"\n\tkarpv1 \"sigs.k8s.io/karpenter/pkg/apis/v1\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates\"\n\t\"github.com/aws/aws-k8s-tester/internal/util\"\n\tapierrors \"k8s.io/apimachinery/pkg/api/errors\"\n)\n\nconst (\n\tnodeDeletionTimeout = time.Minute * 20\n)\n\nvar (\n\tdefaultInstanceTypes_x86_64 = []string{\n\t\t\"m6i.xlarge\",\n\t\t\"m6i.large\",\n\t\t\"m5.large\",\n\t\t\"t3.large\",\n\t}\n\n\tdefaultInstanceTypes_arm64 = []string{\n\t\t\"m7g.xlarge\",\n\t\t\"m7g.large\",\n\t\t\"m6g.xlarge\",\n\t\t\"m6g.large\",\n\t\t\"t4g.xlarge\",\n\t\t\"t4g.large\",\n\t}\n\n\tdefaultInstanceTypesByEC2ArchitectureValues = map[ec2types.ArchitectureValues][]string{\n\t\tec2types.ArchitectureValuesX8664: defaultInstanceTypes_x86_64,\n\t\tec2types.ArchitectureValuesArm64: defaultInstanceTypes_arm64,\n\t}\n\n\tdefaultInstanceTypesByEKSAMITypes = map[ekstypes.AMITypes][]string{\n\t\tekstypes.AMITypesAl2X8664: defaultInstanceTypes_x86_64,\n\t\tekstypes.AMITypesAl2Arm64: defaultInstanceTypes_arm64,\n\t\tekstypes.AMITypesAl2023X8664Standard: defaultInstanceTypes_x86_64,\n\t\tekstypes.AMITypesAl2023Arm64Standard: defaultInstanceTypes_arm64,\n\t}\n\tnodeClassResource = schema.GroupVersionResource{Group: \"eks.amazonaws.com\", Version: \"v1\", Resource: \"nodeclasses\"}\n)\n\ntype nodeManager struct {\n\tclients *awsClients\n\tresourceID string\n}\n\nfunc NewNodeManager(clients *awsClients, resourceID string) *nodeManager {\n\treturn &nodeManager{\n\t\tclients: clients,\n\t\tresourceID: resourceID,\n\t}\n}\n\nfunc (m *nodeManager) createNodes(infra *Infrastructure, cluster *Cluster, opts *deployerOptions, k8sClient *k8sClient) error {\n\tif err := m.resolveInstanceTypes(opts); err != nil {\n\t\treturn fmt.Errorf(\"failed to resolve instance types: %v\", err)\n\t}\n\tif opts.AutoMode {\n\t\tif err := m.createNodeClass(opts, k8sClient); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := m.createNodePool(opts, k8sClient); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, err := m.createPlaceholderDeployment(opts, k8sClient)\n\t\treturn err\n\t} else if opts.UnmanagedNodes {\n\t\treturn m.createUnmanagedNodegroup(infra, cluster, opts)\n\t} else {\n\t\treturn m.createManagedNodegroup(infra, cluster, opts)\n\t}\n}\n\nfunc (m *nodeManager) resolveInstanceTypes(opts *deployerOptions) (err error) {\n\tinstanceTypes := opts.InstanceTypes\n\tif len(instanceTypes) == 0 {\n\t\tif len(opts.InstanceTypeArchs) > 0 {\n\t\t\tslog.Info(\"choosing instance types based on architectures\", \"archs\", opts.InstanceTypeArchs)\n\t\t\tfor _, arch := range opts.InstanceTypeArchs {\n\t\t\t\tvar ec2Arch ec2types.ArchitectureValues\n\t\t\t\tswitch arch {\n\t\t\t\tcase \"x86_64\", \"amd64\":\n\t\t\t\t\tec2Arch = ec2types.ArchitectureValuesX8664\n\t\t\t\tcase \"aarch64\", \"arm64\":\n\t\t\t\t\tec2Arch = ec2types.ArchitectureValuesArm64\n\t\t\t\tdefault:\n\t\t\t\t\treturn fmt.Errorf(\"unknown architecture: '%s'\", arch)\n\t\t\t\t}\n\t\t\t\tinstanceTypesForArch, ok := defaultInstanceTypesByEC2ArchitectureValues[ec2Arch]\n\t\t\t\tif !ok {\n\t\t\t\t\treturn fmt.Errorf(\"no default instance types known for architecture: '%s'\", arch)\n\t\t\t\t}\n\t\t\t\tinstanceTypes = append(instanceTypes, instanceTypesForArch...)\n\t\t\t}\n\t\t} else if opts.UnmanagedNodes {\n\t\t\tslog.Info(\"choosing instance types based on AMI architecture...\")\n\t\t\tif out, err := m.clients.EC2().DescribeImages(context.TODO(), &ec2.DescribeImagesInput{\n\t\t\t\tImageIds: []string{opts.AMI},\n\t\t\t}); err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to describe AMI: %s: %v\", opts.AMI, err)\n\t\t\t} else {\n\t\t\t\tamiArch := out.Images[0].Architecture\n\t\t\t\tinstanceTypesForAMIArchitecture, ok := defaultInstanceTypesByEC2ArchitectureValues[amiArch]\n\t\t\t\tif !ok {\n\t\t\t\t\treturn fmt.Errorf(\"no default instance types known for AMI architecture: %v\", amiArch)\n\t\t\t\t}\n\t\t\t\tinstanceTypes = instanceTypesForAMIArchitecture\n\t\t\t}\n\t\t} else {\n\t\t\t// we don't rely on the service's default instance types, because they're a bit too small for the k8s e2e suite\n\t\t\tslog.Info(\"choosing instance types based on managed nodegroup's AMI type...\")\n\t\t\tinstanceTypesForAMIType, ok := defaultInstanceTypesByEKSAMITypes[ekstypes.AMITypes(opts.AMIType)]\n\t\t\tif !ok {\n\t\t\t\treturn fmt.Errorf(\"no default instance types known for AMI type: %v\", opts.AMIType)\n\t\t\t}\n\t\t\tinstanceTypes = instanceTypesForAMIType\n\t\t}\n\t}\n\tvalidInstanceTypes, err := m.getValidInstanceTypes(instanceTypes)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(validInstanceTypes) == 0 {\n\t\treturn fmt.Errorf(\"none of the instance types %v were valid\", instanceTypes)\n\t}\n\topts.InstanceTypes = validInstanceTypes\n\tslog.Info(\"using instance types\", \"instanceTypes\", opts.InstanceTypes)\n\treturn nil\n}\n\nfunc (m *nodeManager) createNodeClass(opts *deployerOptions, k8sClient *k8sClient) error {\n\tnodeclass, err := k8sClient.dclient.Resource(nodeClassResource).Get(context.Background(), \"default\", metav1.GetOptions{})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"getting default nodeclass, %w\", err)\n\t}\n\tslog.Info(\"got existing default nodeclass for template..\")\n\n\t// clear out the metadata and set the name only\n\tnodeclass.Object[\"metadata\"] = map[string]interface{}{}\n\tnodeclass.SetName(m.resourceID)\n\n\t// clear out the status\n\tdelete(nodeclass.Object, \"status\")\n\n\t// update the ephemeral storage spec to be 500Gi\n\tif spec, ok := nodeclass.Object[\"spec\"].(map[string]interface{}); ok {\n\t\tif ephemeralStorage, ok := spec[\"ephemeralStorage\"].(map[string]interface{}); ok {\n\t\t\tephemeralStorage[\"size\"] = \"500Gi\"\n\t\t}\n\n\t\t// configure capacity reservation selector terms if capacity reservation is enabled\n\t\tif opts.CapacityReservation {\n\t\t\tcapacityReservation, err := m.getCapacityReservation(opts)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to get capacity reservation: %w\", err)\n\t\t\t}\n\t\t\tspec[\"capacityReservationSelectorTerms\"] = []map[string]interface{}{\n\t\t\t\t{\n\t\t\t\t\t\"id\": aws.ToString(capacityReservation.CapacityReservationId),\n\t\t\t\t},\n\t\t\t}\n\t\t}\n\t}\n\n\tslog.Info(\"creating new node class...\")\n\t_, err = k8sClient.dclient.Resource(nodeClassResource).Create(context.Background(), nodeclass, metav1.CreateOptions{})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"creating new nodeclass, %w\", err)\n\t}\n\tslog.Info(\"node class created!\")\n\treturn nil\n}\n\nfunc (m *nodeManager) createNodePool(opts *deployerOptions, k8sClient *k8sClient) error {\n\tnodePool := karpv1.NodePool{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: m.resourceID,\n\t\t},\n\t\tSpec: karpv1.NodePoolSpec{\n\t\t\tWeight: pointer.Int32(100), // max\n\t\t\tDisruption: karpv1.Disruption{\n\t\t\t\tBudgets: []karpv1.Budget{\n\t\t\t\t\t{\n\t\t\t\t\t\tNodes: \"10%\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tConsolidateAfter: karpv1.MustParseNillableDuration(\"Never\"),\n\t\t\t},\n\t\t\tTemplate: karpv1.NodeClaimTemplate{\n\t\t\t\tSpec: karpv1.NodeClaimTemplateSpec{\n\t\t\t\t\tExpireAfter: karpv1.MustParseNillableDuration(\"24h\"),\n\t\t\t\t\tNodeClassRef: &karpv1.NodeClassReference{\n\t\t\t\t\t\tGroup: \"eks.amazonaws.com\",\n\t\t\t\t\t\tKind: \"NodeClass\",\n\t\t\t\t\t\tName: m.resourceID,\n\t\t\t\t\t},\n\t\t\t\t\tRequirements: []karpv1.NodeSelectorRequirementWithMinValues{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tNodeSelectorRequirement: corev1.NodeSelectorRequirement{\n\t\t\t\t\t\t\t\tKey: \"kubernetes.io/os\",\n\t\t\t\t\t\t\t\tOperator: corev1.NodeSelectorOpIn,\n\t\t\t\t\t\t\t\tValues: []string{\"linux\"},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tNodeSelectorRequirement: corev1.NodeSelectorRequirement{\n\t\t\t\t\t\t\t\tKey: \"karpenter.sh/capacity-type\",\n\t\t\t\t\t\t\t\tOperator: corev1.NodeSelectorOpIn,\n\t\t\t\t\t\t\t\tValues: []string{\"reserved\", \"on-demand\"},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tNodeSelectorRequirement: corev1.NodeSelectorRequirement{\n\t\t\t\t\t\t\t\tKey: \"node.kubernetes.io/instance-type\",\n\t\t\t\t\t\t\t\tOperator: corev1.NodeSelectorOpIn,\n\t\t\t\t\t\t\t\tValues: opts.InstanceTypes,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tslog.Info(\"creating node pool...\")\n\tif err := k8sClient.client.Create(context.TODO(), &nodePool); err != nil {\n\t\treturn fmt.Errorf(\"failed to create node pool: %v\", err)\n\t}\n\tslog.Info(\"created node pool\", \"nodePool\", nodePool)\n\treturn nil\n}\n\nfunc (m *nodeManager) deleteNodeClass(k8sClient *k8sClient) error {\n\tslog.Info(\"deleting node class...\")\n\tif err := k8sClient.dclient.Resource(nodeClassResource).Delete(context.Background(), m.resourceID, metav1.DeleteOptions{}); err != nil {\n\t\tif apierrors.IsNotFound(err) {\n\t\t\tslog.Info(\"node class does not exist\", \"resourceID\", m.resourceID)\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to delete node class, %w\", err)\n\t}\n\tslog.Info(\"deleted node class!\")\n\treturn nil\n}\n\nfunc (m *nodeManager) deleteNodePool(k8sClient *k8sClient) error {\n\tnodePool := karpv1.NodePool{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: m.resourceID,\n\t\t},\n\t}\n\tslog.Info(\"deleting node pool...\")\n\tif err := k8sClient.client.Delete(context.TODO(), &nodePool); err != nil {\n\t\tif apierrors.IsNotFound(err) {\n\t\t\tslog.Info(\"node pool does not exist\", \"resourceID\", m.resourceID)\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to delete node pool: %w\", err)\n\t}\n\tslog.Info(\"deleted node pool!\")\n\treturn nil\n}\n\n// createPlaceholderDeployment creates a Deployment with the specified number of replicas that requires\n// each replica to be scheduled on different nodes.\n// This ensures that (at least) the specified number of nodes exist in an EKS Auto cluster\nfunc (m *nodeManager) createPlaceholderDeployment(opts *deployerOptions, k8sClient *k8sClient) (*appsv1.Deployment, error) {\n\tif opts.Nodes == 0 {\n\t\tslog.Info(\"not creating placeholder deployment!\")\n\t\treturn nil, nil\n\t}\n\tlabels := map[string]string{\n\t\t\"app\": m.resourceID,\n\t}\n\td := &appsv1.Deployment{\n\t\tObjectMeta: metav1.ObjectMeta{Name: m.resourceID, Namespace: \"default\"},\n\t\tSpec: appsv1.DeploymentSpec{\n\t\t\tReplicas: pointer.Int32(int32(opts.Nodes)),\n\t\t\tSelector: &metav1.LabelSelector{\n\t\t\t\tMatchLabels: labels,\n\t\t\t},\n\t\t\tTemplate: corev1.PodTemplateSpec{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tLabels: labels,\n\t\t\t\t},\n\t\t\t\tSpec: corev1.PodSpec{\n\t\t\t\t\tAffinity: &corev1.Affinity{\n\t\t\t\t\t\tPodAntiAffinity: &corev1.PodAntiAffinity{\n\t\t\t\t\t\t\tRequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tLabelSelector: &metav1.LabelSelector{\n\t\t\t\t\t\t\t\t\t\tMatchLabels: labels,\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\tTopologyKey: \"kubernetes.io/hostname\",\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tContainers: []corev1.Container{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName: \"main\",\n\t\t\t\t\t\t\tImage: \"public.ecr.aws/amazonlinux/amazonlinux:2023\",\n\t\t\t\t\t\t\tCommand: []string{\"sleep\", \"infinity\"},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tslog.Info(\"creating placeholder deployment...\")\n\td, err := k8sClient.clientset.AppsV1().Deployments(\"default\").Create(context.TODO(), d, metav1.CreateOptions{})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create placeholder deployment: %v\", err)\n\t}\n\tslog.Info(\"created placeholder deployment\", \"deployment\", d)\n\treturn d, nil\n}\n\nfunc (m *nodeManager) deletePlaceholderDeployment(k8sClient *k8sClient) error {\n\tslog.Info(\"deleting placeholder deployment...\")\n\tif err := k8sClient.clientset.AppsV1().Deployments(\"default\").Delete(context.TODO(), m.resourceID, *metav1.NewDeleteOptions( /* no grace period */ 0)); err != nil {\n\t\tif apierrors.IsNotFound(err) {\n\t\t\tslog.Info(\"placeholder deployment does not exist\", \"resourceID\", m.resourceID)\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to delete placeholder deployment: %v\", err)\n\t}\n\tslog.Info(\"deleted placeholder deployment!\")\n\treturn nil\n}\n\nfunc (m *nodeManager) createManagedNodegroup(infra *Infrastructure, cluster *Cluster, opts *deployerOptions) error {\n\tslog.Info(\"creating nodegroup...\")\n\tinput := eks.CreateNodegroupInput{\n\t\tClusterName: aws.String(m.resourceID),\n\t\tNodegroupName: aws.String(m.resourceID),\n\t\tNodeRole: aws.String(infra.nodeRoleARN),\n\t\tSubnets: infra.subnets(),\n\t\tDiskSize: aws.Int32(100),\n\t\tCapacityType: ekstypes.CapacityTypesOnDemand,\n\t\tScalingConfig: &ekstypes.NodegroupScalingConfig{\n\t\t\tMinSize: aws.Int32(int32(opts.Nodes)),\n\t\t\tMaxSize: aws.Int32(int32(opts.Nodes)),\n\t\t\tDesiredSize: aws.Int32(int32(opts.Nodes)),\n\t\t},\n\t\tAmiType: ekstypes.AMITypes(opts.AMIType),\n\t\tInstanceTypes: opts.InstanceTypes,\n\t}\n\tout, err := m.clients.EKS().CreateNodegroup(context.TODO(), &input)\n\tif err != nil {\n\t\treturn err\n\t}\n\tslog.Info(\"waiting for nodegroup to be active\", \"arn\", *out.Nodegroup.NodegroupArn)\n\terr = eks.NewNodegroupActiveWaiter(m.clients.EKS()).\n\t\tWait(context.TODO(), &eks.DescribeNodegroupInput{\n\t\t\tClusterName: input.ClusterName,\n\t\t\tNodegroupName: input.NodegroupName,\n\t\t}, opts.NodeCreationTimeout)\n\tif err != nil {\n\t\treturn err\n\t}\n\tslog.Info(\"nodegroup is active\", \"arn\", *out.Nodegroup.NodegroupArn)\n\tif opts.ExpectedAMI != \"\" {\n\t\tout, err := m.clients.EKS().DescribeNodegroup(context.TODO(), &eks.DescribeNodegroupInput{\n\t\t\tClusterName: input.ClusterName,\n\t\t\tNodegroupName: input.NodegroupName,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tasgName := out.Nodegroup.Resources.AutoScalingGroups[0].Name\n\t\tif ok, err := m.verifyASGAMI(*asgName, opts.ExpectedAMI); err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn fmt.Errorf(\"ASG %s is not using expected AMI: %s\", *asgName, opts.ExpectedAMI)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *nodeManager) createUnmanagedNodegroup(infra *Infrastructure, cluster *Cluster, opts *deployerOptions) error {\n\tvar availabilityZoneFilter []string\n\tvar capacityReservationId string\n\tstackName := m.getUnmanagedNodegroupStackName()\n\tslog.Info(\"creating unmanaged nodegroup stack\", \"stackName\", stackName)\n\tuserData, userDataIsMimePart, err := generateUserData(cluster, opts)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif opts.CapacityReservation {\n\t\tcapacityReservation, err := m.getCapacityReservation(opts)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tcapacityReservationId = aws.ToString(capacityReservation.CapacityReservationId)\n\t\tavailabilityZoneFilter = []string{aws.ToString(capacityReservation.AvailabilityZone)}\n\t} else {\n\t\tavailabilityZoneFilter, err = m.getValidAvailabilityZonesFilter(opts, infra)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\ttargetSubnets, err := m.getValidSubnets(opts, infra, availabilityZoneFilter)\n\tif err != nil {\n\t\treturn err\n\t}\n\tnetworkInterfaces, err := m.getNetworkInterfaces(opts, []string{cluster.securityGroupId}, targetSubnets)\n\tif err != nil {\n\t\treturn err\n\t}\n\tvolumeMountPath := \"/dev/xvda\"\n\tif opts.UserDataFormat == \"bottlerocket\" {\n\t\tvolumeMountPath = \"/dev/xvdb\"\n\t}\n\ttemplateBuf := bytes.Buffer{}\n\terr = templates.UnmanagedNodegroup.Execute(&templateBuf, struct {\n\t\tNetworkInterfaces []templates.NetworkInterface\n\t\tInstanceTypes []string\n\t}{\n\t\tNetworkInterfaces: networkInterfaces,\n\t\tInstanceTypes: opts.InstanceTypes,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tinput := cloudformation.CreateStackInput{\n\t\tStackName: aws.String(stackName),\n\t\tTemplateBody: aws.String(templateBuf.String()),\n\t\tCapabilities: []cloudformationtypes.Capability{cloudformationtypes.CapabilityCapabilityIam},\n\t\tParameters: []cloudformationtypes.Parameter{\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"ResourceId\"),\n\t\t\t\tParameterValue: aws.String(m.resourceID),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"VpcId\"),\n\t\t\t\tParameterValue: aws.String(infra.vpc),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"SubnetIds\"),\n\t\t\t\tParameterValue: aws.String(strings.Join(targetSubnets, \",\")),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"UserData\"),\n\t\t\t\tParameterValue: aws.String(userData),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"UserDataIsMIMEPart\"),\n\t\t\t\tParameterValue: aws.String(strconv.FormatBool(userDataIsMimePart)),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"VolumeMountPath\"),\n\t\t\t\tParameterValue: aws.String(volumeMountPath),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"ClusterName\"),\n\t\t\t\tParameterValue: aws.String(cluster.name),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"NodeRoleName\"),\n\t\t\t\tParameterValue: aws.String(infra.nodeRoleName),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"NodeCount\"),\n\t\t\t\tParameterValue: aws.String(strconv.Itoa(opts.Nodes)),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"SecurityGroup\"),\n\t\t\t\tParameterValue: aws.String(cluster.securityGroupId),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"AMIId\"),\n\t\t\t\tParameterValue: aws.String(opts.AMI),\n\t\t\t},\n\t\t\t{\n\t\t\t\tParameterKey: aws.String(\"CapacityReservationId\"),\n\t\t\t\tParameterValue: aws.String(capacityReservationId),\n\t\t\t},\n\t\t},\n\t}\n\tout, err := m.clients.CFN().CreateStack(context.TODO(), &input)\n\tif err != nil {\n\t\treturn err\n\t}\n\tslog.Info(\"waiting for unmanaged nodegroup stack to be created\", \"stackId\", aws.ToString(out.StackId))\n\terr = cloudformation.NewStackCreateCompleteWaiter(m.clients.CFN()).\n\t\tWait(context.TODO(),\n\t\t\t&cloudformation.DescribeStacksInput{\n\t\t\t\tStackName: out.StackId,\n\t\t\t},\n\t\t\topts.NodeCreationTimeout)\n\tif err != nil {\n\t\treturn util.WrapCFNStackFailure(context.TODO(), m.clients.CFN(), fmt.Errorf(\"failed to wait for unmanaged nodegroup stack creation: %w\", err), stackName)\n\t}\n\tslog.Info(\"created unmanaged nodegroup stack\", \"stackId\", *out.StackId)\n\tif opts.ExpectedAMI != \"\" {\n\t\tif ok, err := m.verifyASGAMI(m.resourceID, opts.ExpectedAMI); err != nil {\n\t\t\treturn err\n\t\t} else if !ok {\n\t\t\treturn fmt.Errorf(\"ASG %s is not using expected AMI: %s\", m.resourceID, opts.ExpectedAMI)\n\t\t}\n\t}\n\treturn nil\n}\n\n// deleteNodes cleans up any nodes in the cluster\n// it will be called outside the context of a deployer run (by the janitor, for example)\n// so will try to delete nodes of any type\nfunc (m *nodeManager) deleteNodes(k8sClient *k8sClient, opts *deployerOptions) error {\n\tif err := m.deleteUnmanagedNodegroup(); err != nil {\n\t\treturn err\n\t}\n\tif err := m.deleteManagedNodegroup(); err != nil {\n\t\treturn err\n\t}\n\t// we only have a k8sClient when this is called by the deployer, not by the janitor\n\t// TODO implement cleanup of Auto nodes in the janitor\n\tif k8sClient != nil && opts != nil && opts.AutoMode {\n\t\tif err := m.deletePlaceholderDeployment(k8sClient); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := m.deleteNodeClass(k8sClient); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := m.deleteNodePool(k8sClient); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := k8sClient.waitForNodeDeletion(nodeDeletionTimeout); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *nodeManager) deleteManagedNodegroup() error {\n\tinput := eks.DeleteNodegroupInput{\n\t\tClusterName: aws.String(m.resourceID),\n\t\tNodegroupName: aws.String(m.resourceID),\n\t}\n\tslog.Info(\"deleting nodegroup...\")\n\tout, err := m.clients.EKS().DeleteNodegroup(context.TODO(), &input)\n\tif err != nil {\n\t\tvar notFound *ekstypes.ResourceNotFoundException\n\t\tif errors.As(err, ¬Found) {\n\t\t\tslog.Info(\"nodegroup does not exist\", \"resourceID\", m.resourceID)\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to delete nodegroup: %v\", err)\n\t}\n\tslog.Info(\"waiting for nodegroup deletion\", \"arn\", *out.Nodegroup.NodegroupArn)\n\terr = eks.NewNodegroupDeletedWaiter(m.clients.EKS()).\n\t\tWait(context.TODO(), &eks.DescribeNodegroupInput{\n\t\t\tClusterName: input.ClusterName,\n\t\t\tNodegroupName: input.NodegroupName,\n\t\t}, nodeDeletionTimeout)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to wait for nodegroup deletion: %v\", err)\n\t}\n\tslog.Info(\"nodegroup deleted\", \"arn\", *out.Nodegroup.NodegroupArn)\n\treturn nil\n}\n\nfunc (m *nodeManager) deleteUnmanagedNodegroup() error {\n\tstackName := m.getUnmanagedNodegroupStackName()\n\tinput := cloudformation.DeleteStackInput{\n\t\tStackName: aws.String(stackName),\n\t}\n\tslog.Info(\"deleting unmanaged nodegroup stack\", \"stackName\", stackName)\n\t_, err := m.clients.CFN().DeleteStack(context.TODO(), &input)\n\tif err != nil {\n\t\tvar notFound *cloudformationtypes.StackNotFoundException\n\t\tif errors.As(err, ¬Found) {\n\t\t\tslog.Info(\"unmanaged nodegroup stack does not exist\", \"stackName\", stackName)\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to delete unmanaged nodegroup stack: %w\", err)\n\t}\n\tslog.Info(\"waiting for unmanaged nodegroup stack to be deleted\", \"stackName\", stackName)\n\terr = cloudformation.NewStackDeleteCompleteWaiter(m.clients.CFN()).\n\t\tWait(context.TODO(),\n\t\t\t&cloudformation.DescribeStacksInput{\n\t\t\t\tStackName: aws.String(stackName),\n\t\t\t},\n\t\t\tinfraStackDeletionTimeout)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to wait for unmanaged nodegroup stack deletion: %w\", err)\n\t}\n\tslog.Info(\"deleted unmanaged nodegroup stack\", \"stackName\", stackName)\n\treturn nil\n}\n\nfunc (m *nodeManager) getUnmanagedNodegroupStackName() string {\n\treturn fmt.Sprintf(\"%s-unmanaged-nodegroup\", m.resourceID)\n}\n\nfunc (m *nodeManager) verifyASGAMI(asgName string, amiId string) (bool, error) {\n\tslog.Info(\"verifying AMI for ASG\", \"amiId\", amiId, \"asgName\", asgName)\n\tasgOut, err := m.clients.ASG().DescribeAutoScalingGroups(context.TODO(), &autoscaling.DescribeAutoScalingGroupsInput{\n\t\tAutoScalingGroupNames: []string{asgName},\n\t})\n\tif err != nil {\n\t\treturn false, nil\n\t}\n\tif len(asgOut.AutoScalingGroups) != 1 {\n\t\treturn false, fmt.Errorf(\"autoscaling group not found: %s\", asgName)\n\t}\n\tvar instanceIds []string\n\tfor _, instance := range asgOut.AutoScalingGroups[0].Instances {\n\t\tinstanceIds = append(instanceIds, *instance.InstanceId)\n\t}\n\tslog.Info(\"verifying AMI for instances\", \"instanceIds\", instanceIds)\n\tec2Out, err := m.clients.EC2().DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{\n\t\tInstanceIds: instanceIds,\n\t})\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tvar errs []error\n\tfor _, reservation := range ec2Out.Reservations {\n\t\tfor _, instance := range reservation.Instances {\n\t\t\tif *instance.ImageId != amiId {\n\t\t\t\terrs = append(errs, fmt.Errorf(\"instance %s using wrong AMI: %s\", *instance.InstanceId, *instance.ImageId))\n\t\t\t}\n\t\t}\n\t}\n\tif len(errs) > 0 {\n\t\treturn false, errors.Join(errs...)\n\t}\n\tslog.Info(\"ASG instances are using expected AMI\", \"amiId\", amiId)\n\treturn true, nil\n}\n\nfunc (m *nodeManager) getCapacityReservation(opts *deployerOptions) (*ec2types.CapacityReservation, error) {\n\tdescribeReservationsInput := ec2.DescribeCapacityReservationsInput{\n\t\tFilters: []ec2types.Filter{\n\t\t\t{\n\t\t\t\tName: aws.String(\"instance-type\"),\n\t\t\t\tValues: opts.InstanceTypes,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: aws.String(\"state\"),\n\t\t\t\tValues: []string{\"active\"},\n\t\t\t},\n\t\t},\n\t}\n\tif opts.TargetCapacityReservationId != \"\" {\n\t\tdescribeReservationsInput.CapacityReservationIds = []string{opts.TargetCapacityReservationId}\n\t}\n\tcapacityReservations, err := m.clients.EC2().DescribeCapacityReservations(context.TODO(), &describeReservationsInput)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to describe capacity reservation: %v\", err)\n\t}\n\tvar capacityReservation *ec2types.CapacityReservation\n\tfor _, cr := range capacityReservations.CapacityReservations {\n\t\tif aws.ToInt32(cr.AvailableInstanceCount) >= int32(opts.Nodes) {\n\t\t\tcapacityReservation = &cr\n\t\t\tbreak\n\t\t}\n\t}\n\tif capacityReservation == nil {\n\t\treturn nil, fmt.Errorf(\"no capacity reservation found for instance type %s with %d nodes count\", opts.InstanceTypes[0], opts.Nodes)\n\t}\n\tslog.Info(\"using capacity reservation\", \"id\", aws.ToString(capacityReservation.CapacityReservationId))\n\treturn capacityReservation, nil\n}\n\nfunc (m *nodeManager) getValidAvailabilityZonesFilter(opts *deployerOptions, infra *Infrastructure) ([]string, error) {\n\tif !opts.EFA {\n\t\t// no filter needed, leaves scheduling to EC2 provisioner\n\t\treturn []string{}, nil\n\t}\n\tdescribeFilters := []ec2types.Filter{\n\t\t{\n\t\t\tName: aws.String(\"instance-type\"),\n\t\t\tValues: opts.InstanceTypes,\n\t\t},\n\t\t{\n\t\t\tName: aws.String(\"location\"),\n\t\t\tValues: infra.availabilityZones,\n\t\t},\n\t}\n\tdescribeResponse, err := m.clients.EC2().DescribeInstanceTypeOfferings(context.TODO(), &ec2.DescribeInstanceTypeOfferingsInput{\n\t\tFilters: describeFilters,\n\t\tLocationType: ec2types.LocationTypeAvailabilityZone,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to describe instance type offerings: %v\", err)\n\t}\n\tif describeResponse == nil || len(describeResponse.InstanceTypeOfferings) == 0 {\n\t\treturn nil, fmt.Errorf(\"no instance type offerings in current region with filters %v\", describeFilters)\n\t}\n\tvar candidateAZs []string\n\tfor _, offering := range describeResponse.InstanceTypeOfferings {\n\t\tcandidateAZs = append(candidateAZs, aws.ToString(offering.Location))\n\t}\n\t// EFA traffic cannot cross an AZ https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html#efa-limits\n\ttargetAZ := availabilityZoneHintedOrder(candidateAZs)[0]\n\tslog.Info(\"found availability zone with offering\", \"az\", targetAZ, \"instanceTypes\", opts.InstanceTypes)\n\treturn []string{targetAZ}, nil\n}\n\nfunc formatFilters(filters []ec2types.Filter) string {\n\tvar parts []string\n\tfor _, f := range filters {\n\t\tparts = append(parts, fmt.Sprintf(\"{Name:%s Values:%v}\", aws.ToString(f.Name), f.Values))\n\t}\n\treturn \"[\" + strings.Join(parts, \",\") + \"]\"\n}\n\nfunc (m *nodeManager) getValidSubnets(opts *deployerOptions, infra *Infrastructure, availabilityZoneFilter []string) ([]string, error) {\n\tvar describeFilters []ec2types.Filter\n\tvar targetSubnets []string\n\tif opts.EFA {\n\t\t// EFA requires private subnets\n\t\ttargetSubnets = infra.subnetsPrivate\n\t} else {\n\t\ttargetSubnets = infra.subnets()\n\t}\n\tif len(availabilityZoneFilter) > 0 {\n\t\tdescribeFilters = append(describeFilters, ec2types.Filter{\n\t\t\tName: aws.String(\"availability-zone\"),\n\t\t\tValues: availabilityZoneFilter,\n\t\t})\n\t}\n\tdescribeResponse, err := m.clients.EC2().DescribeSubnets(context.TODO(), &ec2.DescribeSubnetsInput{\n\t\tFilters: describeFilters,\n\t\tSubnetIds: targetSubnets,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to describe subnets %v: %v\", targetSubnets, err)\n\t}\n\tif describeResponse == nil || len(describeResponse.Subnets) == 0 {\n\t\treturn nil, fmt.Errorf(\"no subnet in %v satisfied filters: %s\", targetSubnets, formatFilters(describeFilters))\n\t}\n\tvar subnetIds []string\n\tfor _, subnet := range describeResponse.Subnets {\n\t\tsubnetIds = append(subnetIds, *subnet.SubnetId)\n\t}\n\tslog.Info(\"using subnets\", \"subnetIds\", subnetIds)\n\treturn subnetIds, nil\n}\n\nfunc (m *nodeManager) getValidInstanceTypes(desiredInstanceTypes []string) ([]string, error) {\n\tvar validInstanceTypes []string\n\tfor _, instanceType := range desiredInstanceTypes {\n\t\tec2InstanceType := ec2types.InstanceType(instanceType)\n\t\t_, err := m.clients.EC2().DescribeInstanceTypes(context.TODO(), &ec2.DescribeInstanceTypesInput{\n\t\t\tInstanceTypes: []ec2types.InstanceType{ec2InstanceType},\n\t\t})\n\t\tif err != nil {\n\t\t\tvar apierr smithy.APIError\n\t\t\tif errors.As(err, &apierr) && apierr.ErrorCode() == \"InvalidInstanceType\" {\n\t\t\t\tslog.Info(\"eliminating instance type as an option\", \"instanceType\", instanceType)\n\t\t\t} else {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to describe instance type: %s: %v\", instanceType, err)\n\t\t\t}\n\t\t} else {\n\t\t\tvalidInstanceTypes = append(validInstanceTypes, instanceType)\n\t\t}\n\t}\n\treturn validInstanceTypes, nil\n}\n\nfunc (m *nodeManager) getNetworkInterfaces(opts *deployerOptions, securityGroups []string, subnetIDs []string) ([]templates.NetworkInterface, error) {\n\tif !opts.EFA {\n\t\t// create only the default primary network interface if not using EFA\n\t\tnetiface, err := getNetworkInterface(opts, 0, subnetIDs, securityGroups)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn []templates.NetworkInterface{netiface}, nil\n\t}\n\t// EFA option assumes a single instance type\n\tinstanceType := opts.InstanceTypes[0]\n\tec2InstanceType := ec2types.InstanceType(instanceType)\n\tdescribeInstanceTypeOutput, err := m.clients.EC2().DescribeInstanceTypes(context.TODO(), &ec2.DescribeInstanceTypesInput{\n\t\tInstanceTypes: []ec2types.InstanceType{ec2InstanceType},\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to describe instance type %s to get network interface support: %v\", instanceType, err)\n\t}\n\tnetworkInfo := describeInstanceTypeOutput.InstanceTypes[0].NetworkInfo\n\tif !aws.ToBool(networkInfo.EfaSupported) {\n\t\t// fail early for better transparency\n\t\treturn nil, fmt.Errorf(\"cannot generate efa interfaces for instance type %s because it does not support efa\", instanceType)\n\t}\n\n\t// 1 EFA interface is supported per network card\n\t// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html#efa-limits\n\tnumEfaInterfaces := int(aws.ToInt32(networkInfo.MaximumNetworkCards))\n\tvar networkInterfaces []templates.NetworkInterface\n\tfor cardIndex := range numEfaInterfaces {\n\t\tefaInterface, err := getNetworkInterface(opts, cardIndex, subnetIDs, securityGroups)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tnetworkInterfaces = append(networkInterfaces, efaInterface)\n\t}\n\treturn networkInterfaces, nil\n}\n\nfunc getNetworkInterface(opts *deployerOptions, networkCardIndex int, subnetIds []string, securityGroups []string) (templates.NetworkInterface, error) {\n\t// simplification that works with currently supported network interfaces based on\n\t// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#network-cards\n\t// and\n\t// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/create-efa.html#efa-launch\n\tdeviceIndex := 0\n\tif networkCardIndex > 0 {\n\t\tdeviceIndex = 1\n\t}\n\tvar description, interfaceType, subnetID *string\n\tif opts.EFA {\n\t\tif len(subnetIds) == 0 {\n\t\t\treturn templates.NetworkInterface{}, fmt.Errorf(\"EFA interfaces require a subnet but none were provided\")\n\t\t}\n\t\tsubnetID = &subnetIds[0]\n\t\tinterfaceType = aws.String(\"efa\")\n\t\tdescription = aws.String(\"EFA-enabled network interface\")\n\t} else {\n\t\t// no need to assign a subnet here, more restrictive than it is helpful\n\t\tinterfaceType = aws.String(\"interface\")\n\t\tdescription = aws.String(\"Standard network interface\")\n\t}\n\treturn templates.NetworkInterface{\n\t\tDescription: description,\n\t\tDeviceIndex: &deviceIndex,\n\t\tNetworkCardIndex: &networkCardIndex,\n\t\tInterfaceType: interfaceType,\n\t\tSubnetId: subnetID,\n\t\tGroups: securityGroups,\n\t\tDeleteOnTermination: aws.Bool(true),\n\t}, nil\n}\n" }, { "path": "internal/deployers/eksapi/static_cluster.go", "content": "package eksapi\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates\"\n\tv1 \"k8s.io/api/apps/v1\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/errors\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/util/wait\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/tools/clientcmd\"\n\t\"sigs.k8s.io/controller-runtime/pkg/client\"\n\tkarpv1 \"sigs.k8s.io/karpenter/pkg/apis/v1\"\n\t\"sigs.k8s.io/yaml\"\n)\n\ntype StaticClusterManager struct {\n\tk8sClient *kubernetes.Clientset\n\tkarpenterClient client.Client\n\toptions *deployerOptions\n}\n\ntype NodeCondition func(nodes []corev1.Node) bool\n\nfunc NewStaticClusterManager(options *deployerOptions) *StaticClusterManager {\n\treturn &StaticClusterManager{\n\t\toptions: options,\n\t}\n}\n\nfunc (s *StaticClusterManager) SetK8sClient(kubeconfig string) {\n\tcfg, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfig)\n\tif err != nil {\n\t\tslog.Error(\"failed to build kubeconfig\", \"error\", err)\n\t\tpanic(err)\n\t}\n\n\ts.k8sClient, err = kubernetes.NewForConfig(cfg)\n\tif err != nil {\n\t\tslog.Error(\"failed to create Kubernetes client\", \"error\", err)\n\t\tpanic(err)\n\t}\n\n\ts.karpenterClient, err = client.New(cfg, client.Options{})\n\tif err != nil {\n\t\tslog.Error(\"failed to create Karpenter client\", \"error\", err)\n\t\tpanic(err)\n\t}\n}\n\nfunc (s *StaticClusterManager) EnsureNodeForStaticCluster() error {\n\tif err := s.CreateNodePool(); err != nil {\n\t\treturn err\n\t}\n\treturn s.DeployBusyboxAndWaitForNodes()\n}\n\nfunc (s *StaticClusterManager) TearDownNodeForStaticCluster() error {\n\tif err := s.TearDownBusyboxAndNodes(); err != nil {\n\t\treturn err\n\t}\n\treturn s.TearDownNodePool()\n}\n\nfunc (s *StaticClusterManager) CreateNodePool() error {\n\tif !strings.Contains(strings.ToLower(s.options.StaticClusterName), \"nvidia\") {\n\t\tslog.Info(\"NVIDIA not in cluster name, skipping node pool creation\")\n\t\treturn nil\n\t}\n\n\tvar arch string\n\tif strings.Contains(s.options.StaticClusterName, \"x86_64\") {\n\t\tarch = \"amd64\"\n\t} else if strings.Contains(s.options.StaticClusterName, \"aarch64\") {\n\t\tarch = \"arm64\"\n\t} else {\n\t\treturn fmt.Errorf(\"unable to determine architecture from cluster name\")\n\t}\n\n\tt := templates.NvidiaStaticClusterNodepool\n\tvar buf bytes.Buffer\n\tif err := t.Execute(&buf, templates.NvidiaStaticClusterNodepoolTemplateData{\n\t\tArch: arch,\n\t\tInstanceTypes: s.options.InstanceTypes,\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tnodePool := &karpv1.NodePool{}\n\tif err := yaml.Unmarshal(buf.Bytes(), nodePool); err != nil {\n\t\treturn fmt.Errorf(\"failed to unmarshal nodepool YAML: %v\", err)\n\t}\n\n\tctx := context.TODO()\n\texisting := &karpv1.NodePool{}\n\terr := s.karpenterClient.Get(ctx, client.ObjectKey{Name: nodePool.Name}, existing)\n\tif client.IgnoreNotFound(err) != nil {\n\t\treturn err\n\t}\n\n\tif errors.IsNotFound(err) {\n\t\treturn s.karpenterClient.Create(ctx, nodePool)\n\t}\n\treturn nil\n}\n\nfunc (s *StaticClusterManager) TearDownNodePool() error {\n\tif !strings.Contains(strings.ToLower(s.options.StaticClusterName), \"nvidia\") {\n\t\tslog.Info(\"NVIDIA not in cluster name, skipping node pool deletion\")\n\t\treturn nil\n\t}\n\n\tnodePool := &karpv1.NodePool{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"nvidia\",\n\t\t},\n\t}\n\n\tif err := s.karpenterClient.Delete(context.TODO(), nodePool); err != nil {\n\t\tif errors.IsNotFound(err) {\n\t\t\tslog.Info(\"NodePool 'nvidia' not found, skipping deletion\")\n\t\t\treturn nil\n\t\t}\n\t\treturn fmt.Errorf(\"failed to delete nodepool: %v\", err)\n\t}\n\n\tslog.Info(\"NodePool deleted successfully\")\n\treturn nil\n}\n\nfunc (s *StaticClusterManager) DeployBusyboxAndWaitForNodes() error {\n\tslog.Info(\"deploying busybox pods\")\n\n\tt := templates.BusyboxDeployment\n\tvar buf bytes.Buffer\n\tif err := t.Execute(&buf, templates.BusyboxDeploymentTemplateData{\n\t\tNodes: s.options.Nodes,\n\t}); err != nil {\n\t\treturn err\n\t}\n\n\tdeployment := &v1.Deployment{}\n\terr := yaml.Unmarshal(buf.Bytes(), deployment)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to unmarshal deployment: %v\", err)\n\t}\n\n\tresult, err := s.k8sClient.AppsV1().Deployments(\"default\").Create(context.TODO(), deployment, metav1.CreateOptions{})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tslog.Info(\"created deployment\", \"name\", result.GetObjectMeta().GetName())\n\treturn waitForNodeCondition(s.k8sClient, func(nodes []corev1.Node) bool {\n\t\treadyNodes := 0\n\t\tfor _, node := range nodes {\n\t\t\tif isNodeReady(&node) {\n\t\t\t\treadyNodes++\n\t\t\t}\n\t\t}\n\t\tslog.Info(\"waiting for nodes\", \"readyNodes\", readyNodes, \"expectedNodes\", s.options.Nodes)\n\t\treturn readyNodes >= s.options.Nodes\n\t}, 15*time.Minute, \"Waiting for nodes to be ready\")\n}\n\nfunc (s *StaticClusterManager) TearDownBusyboxAndNodes() error {\n\tslog.Info(\"cleaning up busybox pods\")\n\n\terr := s.k8sClient.AppsV1().Deployments(\"default\").Delete(context.TODO(), \"busybox-deployment\", metav1.DeleteOptions{})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to delete deployment: %v\", err)\n\t}\n\tslog.Info(\"busybox deployment deleted successfully\")\n\n\treturn waitForNodeCondition(s.k8sClient, func(nodes []corev1.Node) bool {\n\t\treturn len(nodes) == 0\n\t}, 30*time.Minute, \"Waiting for nodes to be removed\")\n}\n\nfunc waitForNodeCondition(clientset *kubernetes.Clientset, condition NodeCondition, timeout time.Duration, description string) error {\n\tctx, cancel := context.WithTimeout(context.Background(), timeout)\n\tdefer cancel()\n\treturn wait.PollUntilContextTimeout(ctx, 15*time.Second, timeout, true, func(ctx context.Context) (bool, error) {\n\t\tnodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\tconditionMet := condition(nodes.Items)\n\t\tslog.Info(description, \"nodeCount\", len(nodes.Items))\n\t\treturn conditionMet, nil\n\t})\n}\n" }, { "path": "internal/deployers/eksapi/templates/auth_map_role.yaml.template", "content": "\n- username: system:node:{{\"{{\"}}{{.NodeNameStrategy}}{{\"}}\"}} \n groups:\n - system:bootstrappers\n - system:nodes\n rolearn: {{.Rolearn}}" }, { "path": "internal/deployers/eksapi/templates/busybox_deployment.yaml.template", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: busybox-deployment\nspec:\n replicas: {{.Nodes}}\n selector:\n matchLabels:\n app: busybox\n template:\n metadata:\n labels:\n app: busybox\n spec:\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchExpressions:\n - key: app\n operator: In\n values:\n - busybox\n topologyKey: \"kubernetes.io/hostname\"\n containers:\n - name: busybox\n image: busybox\n command: [\"sleep\", \"infinity\"]\n" }, { "path": "internal/deployers/eksapi/templates/cloudwatch-infra.yaml.template", "content": "AWSTemplateFormatVersion: '2010-09-09'\nDescription: kubetest2-eksapi CloudWatch using Pod Identity\n\nParameters:\n ClusterUUID:\n Description: UUID portion of the cluster name\n Type: String\n\nResources:\n CloudWatchRole:\n Type: AWS::IAM::Role\n Properties:\n RoleName: !Sub \"cloudwatch-role-${ClusterUUID}\"\n AssumeRolePolicyDocument:\n Version: '2012-10-17'\n Statement:\n - Sid: AllowEksAuthToAssumeRoleForPodIdentity\n Effect: Allow\n Principal:\n Service:\n - pods.eks.amazonaws.com\n - beta.pods.eks.aws.internal\n Action:\n - sts:AssumeRole\n - sts:TagSession\n ManagedPolicyArns:\n - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy\n Description: Role for CloudWatch Agent in EKS cluster\n\nOutputs:\n CloudWatchRoleArn:\n Description: ARN of the CloudWatch IAM role\n Value: !GetAtt CloudWatchRole.Arn\n Export:\n Name: !Sub \"${AWS::StackName}::CloudWatchRoleArn\"\n" }, { "path": "internal/deployers/eksapi/templates/cloudwatch_agent_infra.yaml", "content": "apiVersion: v1\nkind: Namespace\nmetadata:\n name: amazon-cloudwatch\n labels:\n name: amazon-cloudwatch\n\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: cwagent\n namespace: amazon-cloudwatch\n\n---\n# ClusterRole for cwagent\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: cwagent-role\nrules:\n - apiGroups: [\"\"]\n resources:\n - nodes\n - nodes/proxy\n - services\n - endpoints\n - pods\n verbs: [\"get\", \"list\", \"watch\"]\n - apiGroups: [\"extensions\"]\n resources:\n - ingresses\n verbs: [\"get\", \"list\", \"watch\"]\n - nonResourceURLs: [\"/metrics\"]\n verbs: [\"get\"]\n\n---\n# ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: cwagent-role-binding\nsubjects:\n - kind: ServiceAccount\n name: cwagent\n namespace: amazon-cloudwatch\nroleRef:\n kind: ClusterRole\n name: cwagent-role\n apiGroup: rbac.authorization.k8s.io" }, { "path": "internal/deployers/eksapi/templates/infra.yaml", "content": "---\nAWSTemplateFormatVersion: \"2010-09-09\"\nDescription: \"kubetest2-eksapi infrastructure\"\n\nParameters:\n VpcBlock:\n Type: String\n Default: 192.168.0.0/16\n Description: The CIDR range for the VPC. This should be a valid private (RFC 1918) CIDR range.\n\n PublicSubnet01Block:\n Type: String\n Default: 192.168.0.0/18\n Description: CidrBlock for public subnet 01 within the VPC\n\n PublicSubnet02Block:\n Type: String\n Default: 192.168.64.0/18\n Description: CidrBlock for public subnet 02 within the VPC\n\n PrivateSubnet01Block:\n Type: String\n Default: 192.168.128.0/18\n Description: CidrBlock for private subnet 01 within the VPC\n\n PrivateSubnet02Block:\n Type: String\n Default: 192.168.192.0/18\n Description: CidrBlock for private subnet 02 within the VPC\n\n AdditionalClusterRoleServicePrincipal:\n Type: String\n Default: \"\"\n Description: Additional service principal with sts:AssumeRole permissions on the ClusterRole\n\n ResourceId:\n Type: String\n\n Subnet01AZ:\n Type: String\n\n Subnet02AZ:\n Type: String\n\n AutoMode:\n Type: String\n AllowedValues:\n - \"true\"\n - \"false\"\n Default: \"false\"\n\nMetadata:\n AWS::CloudFormation::Interface:\n ParameterGroups:\n - Label:\n default: \"Worker Network Configuration\"\n Parameters:\n - VpcBlock\n - PublicSubnet01Block\n - PublicSubnet02Block\n - PrivateSubnet01Block\n - PrivateSubnet02Block\n\nConditions:\n HasAdditionalClusterRoleServicePrincipal:\n Fn::Not:\n - Fn::Equals:\n - \"\"\n - !Ref AdditionalClusterRoleServicePrincipal\n\n IsAutoMode: !Equals [!Ref AutoMode, \"true\"]\n\nResources:\n #\n # Public VPC\n #\n VPC:\n Type: AWS::EC2::VPC\n Properties:\n CidrBlock: !Ref VpcBlock\n EnableDnsHostnames: true\n EnableDnsSupport: true\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/VPC\"\n IPv6CidrBlock:\n Type: AWS::EC2::VPCCidrBlock\n Properties:\n AmazonProvidedIpv6CidrBlock: true\n VpcId:\n Ref: VPC\n\n #\n # Internet gateways (ipv4, and egress for ipv6)\n #\n InternetGateway:\n Type: AWS::EC2::InternetGateway\n Properties:\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/InternetGateway\"\n VPCGatewayAttachment:\n Type: AWS::EC2::VPCGatewayAttachment\n Properties:\n InternetGatewayId:\n Ref: InternetGateway\n VpcId:\n Ref: VPC\n EgressOnlyInternetGateway:\n Type: AWS::EC2::EgressOnlyInternetGateway\n Properties:\n VpcId:\n Ref: VPC\n\n #\n # Nat gateways\n #\n NATGateway01:\n Type: AWS::EC2::NatGateway\n DependsOn:\n - NatGatewayEIP1\n - SubnetPublic01\n - VPCGatewayAttachment\n Properties:\n AllocationId:\n Fn::GetAtt:\n - NatGatewayEIP1\n - AllocationId\n SubnetId:\n Ref: SubnetPublic01\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/NATGateway01\"\n NATGateway02:\n Type: AWS::EC2::NatGateway\n DependsOn:\n - NatGatewayEIP2\n - SubnetPublic02\n - VPCGatewayAttachment\n Properties:\n AllocationId:\n Fn::GetAtt:\n - NatGatewayEIP2\n - AllocationId\n SubnetId:\n Ref: SubnetPublic02\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/NATGateway02\"\n #\n # Nat Gateway IPs\n #\n NatGatewayEIP1:\n Type: AWS::EC2::EIP\n DependsOn:\n - VPCGatewayAttachment\n Properties:\n Domain: vpc\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/NatGatewayEIP1\"\n NatGatewayEIP2:\n Type: AWS::EC2::EIP\n DependsOn:\n - VPCGatewayAttachment\n Properties:\n Domain: vpc\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/NatGatewayEIP2\"\n\n #\n # Routing - public subnets\n #\n PublicRouteTable:\n Type: AWS::EC2::RouteTable\n Properties:\n VpcId:\n Ref: VPC\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/PublicRouteTable\"\n PublicSubnetDefaultRoute:\n Type: AWS::EC2::Route\n DependsOn:\n - InternetGateway\n - VPCGatewayAttachment\n Properties:\n DestinationCidrBlock: 0.0.0.0/0\n GatewayId:\n Ref: InternetGateway\n RouteTableId:\n Ref: PublicRouteTable\n PublicSubnetDefaultIpv6Route:\n Type: AWS::EC2::Route\n DependsOn:\n - InternetGateway\n - VPCGatewayAttachment\n Properties:\n DestinationIpv6CidrBlock: ::/0\n GatewayId:\n Ref: InternetGateway\n RouteTableId:\n Ref: PublicRouteTable\n\n #\n # Routing - private subnets\n # Route tables\n #\n PrivateRouteTable01:\n Type: AWS::EC2::RouteTable\n Properties:\n VpcId:\n Ref: VPC\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/PrivateRouteTable01\"\n PrivateRouteTable02:\n Type: AWS::EC2::RouteTable\n Properties:\n VpcId:\n Ref: VPC\n Tags:\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/PrivateRouteTable02\"\n #\n # Nat IPv4 Private Routes\n #\n PrivateSubnetDefaultRoute01:\n Type: AWS::EC2::Route\n DependsOn:\n - VPCGatewayAttachment\n - NATGateway01\n Properties:\n DestinationCidrBlock: 0.0.0.0/0\n NatGatewayId:\n Ref: NATGateway01\n RouteTableId:\n Ref: PrivateRouteTable01\n PrivateSubnetDefaultRoute02:\n Type: AWS::EC2::Route\n DependsOn:\n - VPCGatewayAttachment\n - NATGateway02\n Properties:\n DestinationCidrBlock: 0.0.0.0/0\n NatGatewayId:\n Ref: NATGateway02\n RouteTableId:\n Ref: PrivateRouteTable02\n\n #\n # EOIG IPv6 Private Routes\n #\n PrivateSubnetDefaultIpv6Route01:\n Type: AWS::EC2::Route\n Properties:\n DestinationIpv6CidrBlock: ::/0\n EgressOnlyInternetGatewayId:\n Ref: EgressOnlyInternetGateway\n RouteTableId:\n Ref: PrivateRouteTable01\n PrivateSubnetDefaultIpv6Route02:\n Type: AWS::EC2::Route\n Properties:\n DestinationIpv6CidrBlock: ::/0\n EgressOnlyInternetGatewayId:\n Ref: EgressOnlyInternetGateway\n RouteTableId:\n Ref: PrivateRouteTable02\n\n #\n # Public subnets\n SubnetPublic01:\n Type: AWS::EC2::Subnet\n Metadata:\n Comment: Subnet 01\n DependsOn: IPv6CidrBlock\n Properties:\n AvailabilityZone:\n Ref: Subnet01AZ\n CidrBlock:\n Ref: PublicSubnet01Block\n Ipv6CidrBlock:\n !Select [0, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 8, 64]]\n AssignIpv6AddressOnCreation: true\n MapPublicIpOnLaunch: true\n Tags:\n - Key: kubernetes.io/role/elb\n Value: \"1\"\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/SubnetPublic01\"\n VpcId:\n Ref: VPC\n SubnetPublic02:\n Type: AWS::EC2::Subnet\n DependsOn: IPv6CidrBlock\n Properties:\n AvailabilityZone:\n Ref: Subnet02AZ\n CidrBlock:\n Ref: PublicSubnet02Block\n Ipv6CidrBlock:\n !Select [1, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 8, 64]]\n AssignIpv6AddressOnCreation: true\n MapPublicIpOnLaunch: true\n Tags:\n - Key: kubernetes.io/role/elb\n Value: \"1\"\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/SubnetPublic02\"\n VpcId:\n Ref: VPC\n\n #\n # Public route table associations\n #\n RouteTableAssociationPublic01:\n Type: AWS::EC2::SubnetRouteTableAssociation\n Properties:\n RouteTableId:\n Ref: PublicRouteTable\n SubnetId:\n Ref: SubnetPublic01\n RouteTableAssociationPublic02:\n Type: AWS::EC2::SubnetRouteTableAssociation\n Properties:\n RouteTableId:\n Ref: PublicRouteTable\n SubnetId:\n Ref: SubnetPublic02\n\n #\n # Private subnets\n #\n SubnetPrivate01:\n Type: AWS::EC2::Subnet\n DependsOn: IPv6CidrBlock\n Properties:\n AvailabilityZone:\n Ref: Subnet01AZ\n CidrBlock:\n Ref: PrivateSubnet01Block\n Ipv6CidrBlock:\n !Select [2, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 8, 64]]\n AssignIpv6AddressOnCreation: true\n Tags:\n - Key: kubernetes.io/role/internal-elb\n Value: \"1\"\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/SubnetPrivate01\"\n VpcId:\n Ref: VPC\n SubnetPrivate02:\n Type: AWS::EC2::Subnet\n DependsOn: IPv6CidrBlock\n Properties:\n AvailabilityZone:\n Ref: Subnet02AZ\n CidrBlock:\n Ref: PrivateSubnet02Block\n Ipv6CidrBlock:\n !Select [3, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 8, 64]]\n AssignIpv6AddressOnCreation: true\n Tags:\n - Key: kubernetes.io/role/internal-elb\n Value: \"1\"\n - Key: Name\n Value:\n Fn::Sub: \"${AWS::StackName}/SubnetPrivate02\"\n VpcId:\n Ref: VPC\n\n #\n # Private route table associations\n #\n RouteTableAssociationPrivate01:\n Type: AWS::EC2::SubnetRouteTableAssociation\n Properties:\n RouteTableId:\n Ref: PrivateRouteTable01\n SubnetId:\n Ref: SubnetPrivate01\n RouteTableAssociationPrivate02:\n Type: AWS::EC2::SubnetRouteTableAssociation\n Properties:\n RouteTableId:\n Ref: PrivateRouteTable02\n SubnetId:\n Ref: SubnetPrivate02\n\n ClusterRole:\n Type: AWS::IAM::Role\n Properties:\n AssumeRolePolicyDocument:\n Version: 2012-10-17\n Statement:\n - Action:\n - \"sts:AssumeRole\"\n - \"sts:TagSession\"\n Effect: Allow\n Principal:\n Service:\n Fn::If:\n - HasAdditionalClusterRoleServicePrincipal\n - - \"eks.amazonaws.com\"\n - !Ref AdditionalClusterRoleServicePrincipal\n - - \"eks.amazonaws.com\"\n ManagedPolicyArns:\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonEKSClusterPolicy\"\n - !If\n - IsAutoMode\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonEKSBlockStoragePolicy\"\n - !Ref \"AWS::NoValue\"\n - !If\n - IsAutoMode\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonEKSComputePolicy\"\n - !Ref \"AWS::NoValue\"\n - !If\n - IsAutoMode\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonEKSLoadBalancingPolicy\"\n - !Ref \"AWS::NoValue\"\n - !If\n - IsAutoMode\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonEKSNetworkingPolicy\"\n - !Ref \"AWS::NoValue\"\n\n NodeRole:\n Type: AWS::IAM::Role\n Properties:\n AssumeRolePolicyDocument:\n Version: 2012-10-17\n Statement:\n - Action: \"sts:AssumeRole\"\n Effect: Allow\n Principal:\n Service: ec2.amazonaws.com\n ManagedPolicyArns:\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonEKSWorkerNodePolicy\"\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonEC2ContainerRegistryReadOnly\"\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonEKS_CNI_Policy\"\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonSSMManagedInstanceCore\"\n - !Join\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::aws:policy/AmazonS3FullAccess\"\n\n VPCCNIIPv6Policy:\n Type: AWS::IAM::Policy\n Properties:\n PolicyDocument: |\n {\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:AssignIpv6Addresses\",\n \"ec2:DescribeInstances\",\n \"ec2:DescribeTags\",\n \"ec2:DescribeNetworkInterfaces\",\n \"ec2:DescribeInstanceTypes\"\n ],\n \"Resource\": \"*\"\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"ec2:CreateTags\"\n ],\n \"Resource\": [\n \"arn:*:ec2:*:*:network-interface/*\"\n ]\n }\n ]\n }\n PolicyName: AmazonEKS_CNI_IPv6_Policy\n Roles:\n - !Ref NodeRole\n\nOutputs:\n SubnetsPrivate:\n Value:\n Fn::Join:\n - \",\"\n - - Ref: SubnetPrivate01\n - Ref: SubnetPrivate02\n Export:\n Name:\n Fn::Sub: \"${AWS::StackName}::SubnetsPrivate\"\n\n SubnetsPublic:\n Value:\n Fn::Join:\n - \",\"\n - - Ref: SubnetPublic01\n - Ref: SubnetPublic02\n Export:\n Name:\n Fn::Sub: \"${AWS::StackName}::SubnetsPublic\"\n\n VPC:\n Value:\n Ref: VPC\n Export:\n Name:\n Fn::Sub: \"${AWS::StackName}::VPC\"\n\n ClusterRole:\n Value:\n Fn::Join:\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::\"\n - !Ref \"AWS::AccountId\"\n - \":role/\"\n - !Ref ClusterRole\n Export:\n Name:\n Fn::Sub: \"${AWS::StackName}::ClusterRole\"\n\n NodeRole:\n Value:\n Fn::Join:\n - \"\"\n - - \"arn:\"\n - !Ref \"AWS::Partition\"\n - \":iam::\"\n - !Ref \"AWS::AccountId\"\n - \":role/\"\n - !Ref NodeRole\n Export:\n Name:\n Fn::Sub: \"${AWS::StackName}::NodeRole\"\n" }, { "path": "internal/deployers/eksapi/templates/nvidia_static_cluster_nodepool.yaml.template", "content": "apiVersion: karpenter.sh/v1\nkind: NodePool\nmetadata:\n labels:\n app.kubernetes.io/managed-by: eks\n name: nvidia\nspec:\n weight: 50\n template:\n spec:\n requirements:\n - key: kubernetes.io/arch\n operator: In\n values: [{{.Arch}}]\n - key: kubernetes.io/os\n operator: In\n values: [\"linux\"]\n - key: karpenter.sh/capacity-type\n operator: In\n values: [\"on-demand\"]\n - key: node.kubernetes.io/instance-type\n operator: In\n values: \n {{- range .InstanceTypes}}\n - \"{{.}}\"\n {{- end}}\n - key: eks.amazonaws.com/instance-gpu-count\n operator: Exists\n nodeClassRef:\n group: eks.amazonaws.com\n kind: NodeClass\n name: default\n expireAfter: 336h \n disruption:\n budgets:\n - nodes: 10%\n consolidationPolicy: WhenEmpty\n consolidateAfter: 600s\n" }, { "path": "internal/deployers/eksapi/templates/templates.go", "content": "package templates\n\nimport (\n\t_ \"embed\"\n\t\"text/template\"\n)\n\n//go:embed infra.yaml\nvar Infrastructure string\n\n//go:embed cloudwatch_agent_infra.yaml\nvar CloudWatchAgentRbac []byte\n\nvar (\n\t//go:embed unmanaged-nodegroup.yaml.template\n\tunmanagedNodegroupTemplate string\n\tUnmanagedNodegroup = template.Must(template.New(\"unmanagedNodegroup\").Parse(unmanagedNodegroupTemplate))\n)\n\n//go:embed cloudwatch-infra.yaml.template\nvar CloudWatchInfra string\n\ntype NetworkInterface struct {\n\tDescription *string\n\tNetworkCardIndex *int\n\tDeviceIndex *int\n\tInterfaceType *string\n\tGroups []string\n\tSubnetId *string\n\tDeleteOnTermination *bool\n}\n\ntype UnmanagedNodegroupTemplateData struct {\n\tNetworkInterfaces []NetworkInterface\n\tKubernetesVersion string\n\tInstanceTypes []string\n}\n\ntype BusyboxDeploymentTemplateData struct {\n\tNodes int\n}\n\ntype NvidiaStaticClusterNodepoolTemplateData struct {\n\tArch string\n\tInstanceTypes []string\n}\n\nvar (\n\t//go:embed userdata_bootstrap.sh.mimepart.template\n\tuserDataBootstrapShTemplate string\n\tUserDataBootstrapSh = template.Must(template.New(\"userDataBootstrapSh\").Parse(userDataBootstrapShTemplate))\n\n\t//go:embed userdata_nodeadm.yaml.mimepart.template\n\tuserDataNodeadmTemplate string\n\tUserDataNodeadm = template.Must(template.New(\"userDataNodeadm\").Parse(userDataNodeadmTemplate))\n\n\t//go:embed userdata_bottlerocket.toml.template\n\tuserDataBottlerocketTemplate string\n\tUserDataBottlerocket = template.Must(template.New(\"userDataBottlerocket\").Parse(userDataBottlerocketTemplate))\n\n\t//go:embed busybox_deployment.yaml.template\n\tbusyboxDeploymentTemplate string\n\tBusyboxDeployment = template.Must(template.New(\"busyboxDeployment\").Parse(busyboxDeploymentTemplate))\n\n\t//go:embed nvidia_static_cluster_nodepool.yaml.template\n\tnvidiaStaticClusterNodepoolTemplate string\n\tNvidiaStaticClusterNodepool = template.Must(template.New(\"nvidiaStaticClusterNodepool\").Parse(nvidiaStaticClusterNodepoolTemplate))\n)\n\ntype UserDataTemplateData struct {\n\tName string\n\tCertificateAuthority string\n\tCIDR string\n\tClusterDNSIP string\n\tAPIServerEndpoint string\n\tKubeletFeatureGates map[string]bool\n\tNodeadmFeatureGates map[string]bool\n}\n\nvar (\n\t//go:embed auth_map_role.yaml.template\n\tauthMapRoleTemplate string\n\tAuthMapRole = template.Must(template.New(\"authMapRole\").Parse(authMapRoleTemplate))\n)\n\ntype AuthMapRoleTemplateData struct {\n\tNodeNameStrategy string\n\tRolearn string\n}\n" }, { "path": "internal/deployers/eksapi/templates/templates_test.go", "content": "package templates\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n)\n\nfunc Test_UnmanagedNodegroup(t *testing.T) {\n\tbuf := bytes.Buffer{}\n\terr := UnmanagedNodegroup.Execute(&buf, UnmanagedNodegroupTemplateData{\n\t\tKubernetesVersion: \"1.28\",\n\t\tInstanceTypes: []string{\n\t\t\t\"t2.medium\",\n\t\t\t\"t2.large\",\n\t\t\t\"t2.xlarge\",\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n}\n" }, { "path": "internal/deployers/eksapi/templates/unmanaged-nodegroup.yaml.template", "content": "---\nAWSTemplateFormatVersion: '2010-09-09'\nDescription: 'kubetest2-eksapi unmanaged nodegroup'\n\nParameters:\n ResourceId:\n Description: Unique identifier for this kubetest2-eksapi execution.\n Type: String\n\n VpcId:\n Type: AWS::EC2::VPC::Id\n\n SubnetIds:\n Type: List\n\n SecurityGroup:\n Type: AWS::EC2::SecurityGroup::Id\n\n AMIId:\n Type: String\n Description: Specify AMI id for the node instances.\n\n NodeDiskSize:\n Type: Number\n Description: Node disk size in gigabytes.\n Default: 100\n\n NodeCount:\n Type: Number\n\n ClusterName:\n Type: String\n\n NodeRoleName:\n Description: The IAM role name of worker nodes.\n Type: String\n\n UserData:\n Type: String\n\n VolumeMountPath:\n Type: String\n\n CapacityReservationId:\n Type: String\n Description: Capacity reservation id for the unmanaged nodegroup\n\n UserDataIsMIMEPart:\n Description: \"User data should be embedded as a part of a multi-part MIME document\"\n Default: true\n Type: String\n AllowedValues: [true, false]\n\nConditions:\n IsCapacityReservationIdSet: !Not [!Equals [!Ref CapacityReservationId, \"\"]]\n IsUserDataMIMEPart: !Equals [true, !Ref UserDataIsMIMEPart]\n\nResources:\n EFASecurityGroupIngress:\n Type: \"AWS::EC2::SecurityGroupIngress\"\n Properties:\n Description: Allow node to communicate with each other\n FromPort: 0\n ToPort: 65535\n GroupId: !Ref SecurityGroup\n IpProtocol: \"-1\"\n SourceSecurityGroupId: !Ref SecurityGroup\n\n EFASecurityGroupEgress:\n Type: \"AWS::EC2::SecurityGroupEgress\"\n Properties:\n Description: Allow the efa worker nodes outbound communication\n DestinationSecurityGroupId: !Ref SecurityGroup\n FromPort: 0\n ToPort: 65536\n GroupId: !Ref SecurityGroup\n IpProtocol: \"-1\"\n \n EFASecurityGroupEgressAllIpv4:\n Type: \"AWS::EC2::SecurityGroupEgress\"\n Properties:\n Description: Allow the efa worker nodes outbound communication\n FromPort: 0\n ToPort: 65536\n CidrIp: \"0.0.0.0/0\"\n GroupId: !Ref SecurityGroup\n IpProtocol: \"-1\"\n\n EFASecurityGroupEgressAllIpv6:\n Type: \"AWS::EC2::SecurityGroupEgress\"\n Properties:\n Description: Allow the efa worker nodes outbound communication\n FromPort: 0\n ToPort: 65536\n CidrIpv6: \"::/0\"\n GroupId: !Ref SecurityGroup\n IpProtocol: \"-1\"\n\n NodeInstanceProfile:\n Type: AWS::IAM::InstanceProfile\n Properties:\n Path: \"/\"\n Roles:\n - !Ref NodeRoleName\n\n NodeLaunchTemplate:\n Type: AWS::EC2::LaunchTemplate\n Properties:\n LaunchTemplateName: !Ref ResourceId\n LaunchTemplateData:\n BlockDeviceMappings:\n - DeviceName: !Ref VolumeMountPath\n Ebs:\n DeleteOnTermination: true\n VolumeSize: !Ref NodeDiskSize\n VolumeType: gp2\n CapacityReservationSpecification:\n Fn::If:\n - IsCapacityReservationIdSet\n - CapacityReservationTarget:\n CapacityReservationId: !Ref CapacityReservationId\n - !Ref AWS::NoValue\n IamInstanceProfile:\n Arn: !GetAtt NodeInstanceProfile.Arn\n ImageId: !Ref AMIId\n InstanceType: \"{{index .InstanceTypes 0}}\"\n MetadataOptions: \n HttpTokens: required\n {{ if .NetworkInterfaces -}}\n NetworkInterfaces:\n {{- range .NetworkInterfaces}}\n - NetworkCardIndex: {{ .NetworkCardIndex }}{{ if .DeviceIndex }} {{/* network card index cannot be empty */}}\n DeviceIndex: {{ .DeviceIndex }}{{ end }}{{ if .InterfaceType }}\n InterfaceType: {{ .InterfaceType }}{{ end }}{{ if .Groups }}\n Groups: {{ .Groups }}{{ end }}{{ if .SubnetId }}\n SubnetId: {{ .SubnetId }}{{ end }}{{ if .DeleteOnTermination }}\n DeleteOnTermination: {{ .DeleteOnTermination }}{{ end }}{{ if .Description }}\n Description: {{ .Description}}{{ end -}}\n {{- end}}\n {{ end -}}\n UserData:\n Fn::Base64:\n Fn::If:\n - IsUserDataMIMEPart\n - Fn::Sub: |\n Content-Type: multipart/mixed; boundary=\"BOUNDARY\"\n MIME-Version: 1.0\n\n --BOUNDARY\n ${UserData}\n\n --BOUNDARY\n Content-Type: text/x-shellscript; charset=\"us-ascii\"\n MIME-Version: 1.0\n\n #!/usr/bin/env bash\n /opt/aws/bin/cfn-signal \\\n --stack ${AWS::StackName} \\\n --resource NodeGroup \\\n --region ${AWS::Region}\n\n --BOUNDARY--\n - Fn::Sub: |\n ${UserData}\n\n NodeGroup:\n Type: AWS::AutoScaling::AutoScalingGroup\n UpdatePolicy:\n AutoScalingRollingUpdate:\n WaitOnResourceSignals: true\n PauseTime: PT15M\n Properties:\n AutoScalingGroupName: !Ref ResourceId\n MixedInstancesPolicy:\n LaunchTemplate:\n LaunchTemplateSpecification:\n LaunchTemplateId: !Ref NodeLaunchTemplate\n Version: !GetAtt NodeLaunchTemplate.LatestVersionNumber\n Overrides:\n {{- range .InstanceTypes}}\n - InstanceType: \"{{.}}\"\n {{- end}}\n DesiredCapacity: !Ref NodeCount\n MinSize: !Ref NodeCount\n MaxSize: !Ref NodeCount\n VPCZoneIdentifier: !Ref SubnetIds\n Tags:\n - Key: Name\n Value: !Sub \"${ClusterName}-Node\"\n PropagateAtLaunch: true\n # necessary for kubelet's legacy, in-tree cloud provider\n - Key: !Sub \"kubernetes.io/cluster/${ClusterName}\"\n Value: owned\n PropagateAtLaunch: true" }, { "path": "internal/deployers/eksapi/templates/userdata_bootstrap.sh.mimepart.template", "content": "Content-Type: text/x-shellscript; charset=\"us-ascii\"\nMIME-Version: 1.0\n\n#!/usr/bin/env bash\n/etc/eks/bootstrap.sh {{.Name}} \\\n --b64-cluster-ca {{.CertificateAuthority}} \\\n --apiserver-endpoint {{.APIServerEndpoint}}\n" }, { "path": "internal/deployers/eksapi/templates/userdata_bottlerocket.toml.template", "content": "[settings.kubernetes]\n\"cluster-name\" = \"{{.Name}}\"\n\"api-server\" = \"{{.APIServerEndpoint}}\"\n\"cluster-certificate\" = \"{{.CertificateAuthority}}\"\n{{- if .ClusterDNSIP}}\n\"cluster-dns-ip\" = \"{{.ClusterDNSIP}}\"\n{{- end}}\ndevice-ownership-from-security-context = true\n\n[settings.host-containers.admin]\n\"enabled\" = true\n" }, { "path": "internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template", "content": "Content-Type: application/node.eks.aws\nMIME-Version: 1.0\n\n---\napiVersion: node.eks.aws/v1alpha1\nkind: NodeConfig\nspec:\n{{- if .NodeadmFeatureGates}}\n featureGates:\n {{- range $gate, $value := .NodeadmFeatureGates }}\n {{$gate}}: {{$value}}\n {{- end }}\n{{- end }}\n cluster:\n name: {{.Name}}\n apiServerEndpoint: {{.APIServerEndpoint}}\n certificateAuthority: {{.CertificateAuthority}}\n cidr: {{.CIDR}}\n{{- if .KubeletFeatureGates}}\n kubelet:\n config:\n featureGates:\n {{- range $gate, $value := .KubeletFeatureGates }}\n {{$gate}}: {{$value}}\n {{- end }}\n{{- end }}\n" }, { "path": "internal/deployers/eksapi/userdata.go", "content": "package eksapi\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"net\"\n\t\"strconv\"\n\t\"strings\"\n\t\"text/template\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates\"\n)\n\nconst (\n\tUserDataBootstrapSh = \"bootstrap.sh\"\n\tUserDataNodeadm = \"nodeadm\"\n\tUserDataBottlerocket = \"bottlerocket\"\n)\n\nfunc generateUserData(cluster *Cluster, opts *deployerOptions) (string, bool, error) {\n\tuserDataIsMimePart := true\n\tvar t *template.Template\n\tswitch opts.UserDataFormat {\n\tcase UserDataBootstrapSh:\n\t\tt = templates.UserDataBootstrapSh\n\tcase UserDataNodeadm:\n\t\t// TODO: replace the YAML template with proper usage of the nodeadm API go types\n\t\tt = templates.UserDataNodeadm\n\tcase UserDataBottlerocket:\n\t\tt = templates.UserDataBottlerocket\n\t\tuserDataIsMimePart = false\n\tdefault:\n\t\treturn \"\", false, fmt.Errorf(\"unknown user data format: '%s'\", opts.UserDataFormat)\n\t}\n\n\tkubeletFeatureGates := map[string]bool{}\n\t// DRA is in beta for 1.33, and so needs to be explicitly enabled.\n\tif opts.KubernetesVersion == \"1.33\" {\n\t\tkubeletFeatureGates[\"DynamicResourceAllocation\"] = true\n\t}\n\n\tnodeadmFeatureGates, err := extractFeatureGates(opts.NodeadmFeatureGates)\n\tif err != nil {\n\t\treturn \"\", false, err\n\t}\n\n\tvar dnsIP string\n\tif opts.SetClusterDNSIP {\n\t\tdnsIP, err = deriveClusterDNSIP(cluster.cidr)\n\t\tif err != nil {\n\t\t\treturn \"\", false, err\n\t\t}\n\t}\n\n\tvar buf bytes.Buffer\n\tif err := t.Execute(&buf, templates.UserDataTemplateData{\n\t\tAPIServerEndpoint: cluster.endpoint,\n\t\tCertificateAuthority: cluster.certificateAuthorityData,\n\t\tCIDR: cluster.cidr,\n\t\tClusterDNSIP: dnsIP,\n\t\tName: cluster.name,\n\t\tKubeletFeatureGates: kubeletFeatureGates,\n\t\tNodeadmFeatureGates: nodeadmFeatureGates,\n\t}); err != nil {\n\t\treturn \"\", false, err\n\t}\n\treturn buf.String(), userDataIsMimePart, nil\n}\n\nfunc deriveClusterDNSIP(cidr string) (string, error) {\n\t_, ipNet, err := net.ParseCIDR(cidr)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"invalid CIDR: %v\", err)\n\t}\n\tip := ipNet.IP\n\tip[len(ip)-1] += 10\n\treturn ip.String(), nil\n}\n\nfunc extractFeatureGates(featureGatePairs []string) (map[string]bool, error) {\n\tfeatureGateMap := make(map[string]bool)\n\tfor _, keyValuePair := range featureGatePairs {\n\t\tcomponents := strings.Split(keyValuePair, \"=\")\n\t\tif len(components) != 2 {\n\t\t\treturn featureGateMap, fmt.Errorf(\"expected key=value pairs but %s has %d components\", keyValuePair, len(components))\n\t\t}\n\t\tboolValue, err := strconv.ParseBool(components[1])\n\t\tif err != nil {\n\t\t\treturn featureGateMap, fmt.Errorf(\"expected bool value in %s: %v\", keyValuePair, err)\n\t\t}\n\t\tfeatureGateMap[components[0]] = boolValue\n\t}\n\treturn featureGateMap, nil\n}\n" }, { "path": "internal/deployers/eksapi/userdata_test.go", "content": "package eksapi\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar cluster = Cluster{\n\tname: \"cluster\",\n\tendpoint: \"https://example.com\",\n\tcertificateAuthorityData: \"certificateAuthority\",\n\tcidr: \"10.100.0.0/16\",\n}\n\nconst bootstrapShUserData = `Content-Type: text/x-shellscript; charset=\"us-ascii\"\nMIME-Version: 1.0\n\n#!/usr/bin/env bash\n/etc/eks/bootstrap.sh cluster \\\n --b64-cluster-ca certificateAuthority \\\n --apiserver-endpoint https://example.com\n`\n\nconst nodeadmUserData = `Content-Type: application/node.eks.aws\nMIME-Version: 1.0\n\n---\napiVersion: node.eks.aws/v1alpha1\nkind: NodeConfig\nspec:\n cluster:\n name: cluster\n apiServerEndpoint: https://example.com\n certificateAuthority: certificateAuthority\n cidr: 10.100.0.0/16\n`\n\nconst nodeadmUserDataKubeletDRA = `Content-Type: application/node.eks.aws\nMIME-Version: 1.0\n\n---\napiVersion: node.eks.aws/v1alpha1\nkind: NodeConfig\nspec:\n cluster:\n name: cluster\n apiServerEndpoint: https://example.com\n certificateAuthority: certificateAuthority\n cidr: 10.100.0.0/16\n kubelet:\n config:\n featureGates:\n DynamicResourceAllocation: true\n`\n\nconst nodeadmUserDataFeatureGate = `Content-Type: application/node.eks.aws\nMIME-Version: 1.0\n\n---\napiVersion: node.eks.aws/v1alpha1\nkind: NodeConfig\nspec:\n featureGates:\n foo: true\n cluster:\n name: cluster\n apiServerEndpoint: https://example.com\n certificateAuthority: certificateAuthority\n cidr: 10.100.0.0/16\n`\n\nconst bottlerocketUserData = `[settings.kubernetes]\n\"cluster-name\" = \"cluster\"\n\"api-server\" = \"https://example.com\"\n\"cluster-certificate\" = \"certificateAuthority\"\ndevice-ownership-from-security-context = true\n\n[settings.host-containers.admin]\n\"enabled\" = true\n`\n\nconst bottlerocketUserDataWithDNS = `[settings.kubernetes]\n\"cluster-name\" = \"cluster\"\n\"api-server\" = \"https://example.com\"\n\"cluster-certificate\" = \"certificateAuthority\"\n\"cluster-dns-ip\" = \"10.100.0.10\"\ndevice-ownership-from-security-context = true\n\n[settings.host-containers.admin]\n\"enabled\" = true\n`\n\nfunc Test_generateUserData(t *testing.T) {\n\tcases := []struct {\n\t\tformat string\n\t\texpected string\n\t\texpectedIsMimePart bool\n\t\tkubernetesVersion string\n\t\tNodeadmFeatureGates []string\n\t\tsetClusterDNSIP bool\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tformat: \"bootstrap.sh\",\n\t\t\texpected: bootstrapShUserData,\n\t\t\texpectedIsMimePart: true,\n\t\t},\n\t\t{\n\t\t\tformat: \"nodeadm\",\n\t\t\texpected: nodeadmUserData,\n\t\t\texpectedIsMimePart: true,\n\t\t},\n\t\t{\n\t\t\tformat: \"bottlerocket\",\n\t\t\texpected: bottlerocketUserData,\n\t\t\texpectedIsMimePart: false,\n\t\t},\n\t\t{\n\t\t\tformat: \"bottlerocket\",\n\t\t\texpected: bottlerocketUserDataWithDNS,\n\t\t\texpectedIsMimePart: false,\n\t\t\tsetClusterDNSIP: true,\n\t\t},\n\t\t{\n\t\t\tformat: \"nodeadm\",\n\t\t\texpected: nodeadmUserDataKubeletDRA,\n\t\t\tkubernetesVersion: \"1.33\",\n\t\t\texpectedIsMimePart: true,\n\t\t},\n\t\t{\n\t\t\tformat: \"nodeadm\",\n\t\t\texpected: nodeadmUserDataFeatureGate,\n\t\t\tkubernetesVersion: \"1.30\",\n\t\t\tNodeadmFeatureGates: []string{\"foo=true\"},\n\t\t\texpectedIsMimePart: true,\n\t\t},\n\t}\n\tfor _, c := range cases {\n\t\tt.Run(c.format, func(t *testing.T) {\n\t\t\tdeployerOpts := &deployerOptions{\n\t\t\t\tKubernetesVersion: c.kubernetesVersion,\n\t\t\t\tNodeadmFeatureGates: c.NodeadmFeatureGates,\n\t\t\t\tSetClusterDNSIP: c.setClusterDNSIP,\n\t\t\t\tUserDataFormat: c.format,\n\t\t\t}\n\t\t\tactual, isMimePart, err := generateUserData(&cluster, deployerOpts)\n\t\t\tif err != nil {\n\t\t\t\tt.Log(err)\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tassert.Equal(t, c.expected, actual)\n\t\t\tassert.Equal(t, c.expectedIsMimePart, isMimePart)\n\t\t})\n\t}\n}\n\nfunc Test_extractFeatureGates(t *testing.T) {\n\ttestCases := []struct {\n\t\tinput []string\n\t\texpected map[string]bool\n\t\texpectErr bool\n\t}{\n\t\t{\n\t\t\tinput: []string{\"foo=true\", \"bar=false\"},\n\t\t\texpected: map[string]bool{\n\t\t\t\t\"foo\": true,\n\t\t\t\t\"bar\": false,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinput: []string{\"foo:true\"},\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput: []string{\"foo=bar\"},\n\t\t\texpectErr: true,\n\t\t},\n\t}\n\tfor _, testCase := range testCases {\n\t\toutput, err := extractFeatureGates(testCase.input)\n\t\tif testCase.expectErr {\n\t\t\tassert.Error(t, err)\n\t\t} else {\n\t\t\tassert.NoError(t, err)\n\t\t\tassert.Equal(t, testCase.expected, output)\n\t\t}\n\t}\n}\n\nfunc Test_deriveClusterDNSIP(t *testing.T) {\n\ttestCases := []struct {\n\t\tcidr string\n\t\texpected string\n\t\texpectErr bool\n\t}{\n\t\t{cidr: \"192.0.2.0/24\", expected: \"192.0.2.10\"},\n\t\t{cidr: \"198.51.100.0/24\", expected: \"198.51.100.10\"},\n\t\t{cidr: \"2001:db8:1234::/108\", expected: \"2001:db8:1234::a\"},\n\t\t{cidr: \"invalid\", expectErr: true},\n\t}\n\tfor _, tc := range testCases {\n\t\tresult, err := deriveClusterDNSIP(tc.cidr)\n\t\tif tc.expectErr {\n\t\t\tassert.Error(t, err)\n\t\t} else {\n\t\t\tassert.NoError(t, err)\n\t\t\tassert.Equal(t, tc.expected, result)\n\t\t}\n\t}\n}\n" }, { "path": "internal/deployers/eksapi/vpccni.go", "content": "package eksapi\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/types\"\n)\n\nconst vpcCNIDaemonSetPatch = `{\n\t\"spec\": {\n\t\t\"template\": {\n\t\t\t\"spec\": {\n\t\t\t\t\"containers\": [\n\t\t\t\t\t{\n\t\t\t\t\t\t\"name\": \"aws-node\",\n\t\t\t\t\t\t\"env\": [\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\"name\": \"ENABLE_PREFIX_DELEGATION\",\n\t\t\t\t\t\t\t\t\"value\": \"true\"\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\"name\": \"MINIMUM_IP_TARGET\",\n\t\t\t\t\t\t\t\t\"value\": \"80\"\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\"name\": \"WARM_IP_TARGET\",\n\t\t\t\t\t\t\t\t\"value\": \"10\"\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t]\n\t\t\t\t\t}\n\t\t\t\t]\n\t\t\t}\n\t\t}\n\t}\n}`\n\n// tuneVPCCNI applies configuration to the VPC CNI DaemonSet that helps prevent test flakiness\nfunc (k *k8sClient) tuneVPCCNI() error {\n\tvar patch bytes.Buffer\n\tif err := json.Compact(&patch, []byte(vpcCNIDaemonSetPatch)); err != nil {\n\t\treturn err\n\t}\n\t_, err := k.clientset.AppsV1().DaemonSets(\"kube-system\").Patch(context.TODO(), \"aws-node\", types.StrategicMergePatchType, patch.Bytes(), metav1.PatchOptions{})\n\treturn err\n}\n" }, { "path": "internal/deployers/eksapi/vpccni_test.go", "content": "package eksapi\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n)\n\nfunc Test_validVPCCNIDaemonSetPatch(t *testing.T) {\n\tvar j json.RawMessage\n\tif err := json.Unmarshal([]byte(vpcCNIDaemonSetPatch), &j); err != nil {\n\t\tt.Error(err)\n\t}\n}\n" }, { "path": "internal/deployers/eksctl/build.go", "content": "package eksctl\n\n// Build is a no-op\nfunc (d *deployer) Build() error {\n\treturn nil\n}\n" }, { "path": "internal/deployers/eksctl/cluster_config.go", "content": "package eksctl\n\nimport (\n\t\"fmt\"\n\t\"log/slog\"\n\n\teksctl_api \"github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5\"\n\t\"sigs.k8s.io/yaml\"\n)\n\n// CreateClusterConfig constructs an eksctl_api.ClusterConfig object based on UpOptions.\n// This function replaces the string-based template rendering.\nfunc (d *deployer) CreateClusterConfig() (*eksctl_api.ClusterConfig, error) {\n\td.initClusterName()\n\n\tcfg := eksctl_api.NewClusterConfig()\n\t// Metadata\n\tcfg.Metadata.Name = d.clusterName\n\tcfg.Metadata.Region = d.Region\n\tcfg.Metadata.Version = d.KubernetesVersion\n\t// IAM\n\tcfg.IAM.WithOIDC = &d.WithOIDC\n\n\tamiFamily := d.AMIFamily\n\tif amiFamily == \"\" {\n\t\tamiFamily = eksctl_api.NodeImageFamilyAmazonLinux2\n\t}\n\tnodeGroupName := d.NodegroupName\n\tif nodeGroupName == \"\" {\n\t\tnodeGroupName = \"ng-1\"\n\t}\n\t// Create node group or managed node group (MNG)\n\tif d.UseUnmanagedNodegroup {\n\t\tng := cfg.NewNodeGroup()\n\t\t// TODO: update this when we add support for SSH.\n\t\tng.SSH = nil\n\t\tng.AMIFamily = amiFamily\n\t\tng.Name = nodeGroupName\n\t\tif len(d.InstanceTypes) > 0 {\n\t\t\tng.InstanceType = d.InstanceTypes[0]\n\t\t}\n\t\tif d.Nodes >= 0 {\n\t\t\tng.MinSize = &d.Nodes\n\t\t\tng.MaxSize = &d.Nodes\n\t\t\tng.DesiredCapacity = &d.Nodes\n\t\t}\n\t\tif d.VolumeSize >= 0 {\n\t\t\tng.VolumeSize = &d.VolumeSize\n\t\t}\n\t\tng.PrivateNetworking = d.PrivateNetworking\n\t\tng.EFAEnabled = &d.EFAEnabled\n\t\tif len(d.AvailabilityZones) > 0 {\n\t\t\tng.AvailabilityZones = d.AvailabilityZones\n\t\t}\n\t\tif d.AMI != \"\" && amiFamily == eksctl_api.NodeImageFamilyAmazonLinux2 {\n\t\t\tbootstrapCommand := fmt.Sprintf(`#!/bin/bash\nsource /var/lib/cloud/scripts/eksctl/bootstrap.helper.sh\n/etc/eks/bootstrap.sh %s --kubelet-extra-args \"--node-labels=${NODE_LABELS}\"`, d.clusterName)\n\t\t\tng.OverrideBootstrapCommand = &bootstrapCommand\n\t\t}\n\t} else {\n\t\t// Create managed node group\n\t\tmng := eksctl_api.NewManagedNodeGroup()\n\t\tcfg.ManagedNodeGroups = append(cfg.ManagedNodeGroups, mng)\n\t\t// TODO: update this when we add support for SSH.\n\t\tmng.SSH = nil\n\t\tmng.AMIFamily = amiFamily\n\t\tmng.Name = nodeGroupName\n\t\tmng.InstanceTypes = d.InstanceTypes\n\t\tif d.Nodes >= 0 {\n\t\t\tmng.MinSize = &d.Nodes\n\t\t\tmng.MaxSize = &d.Nodes\n\t\t\tmng.DesiredCapacity = &d.Nodes\n\t\t}\n\t\tif d.VolumeSize >= 0 {\n\t\t\tmng.VolumeSize = &d.VolumeSize\n\t\t}\n\t\tmng.PrivateNetworking = d.PrivateNetworking\n\t\tmng.EFAEnabled = &d.EFAEnabled\n\t\tif len(d.AvailabilityZones) > 0 {\n\t\t\tmng.AvailabilityZones = d.AvailabilityZones\n\t\t}\n\t\tif d.AMI != \"\" && amiFamily == eksctl_api.NodeImageFamilyAmazonLinux2 {\n\t\t\tbootstrapCommand := fmt.Sprintf(`#!/bin/bash\nsource /var/lib/cloud/scripts/eksctl/bootstrap.helper.sh\n/etc/eks/bootstrap.sh %s --kubelet-extra-args \"--node-labels=${NODE_LABELS}\"`, d.clusterName)\n\t\t\tmng.OverrideBootstrapCommand = &bootstrapCommand\n\t\t} else if d.AMI != \"\" && amiFamily == eksctl_api.NodeImageFamilyBottlerocket {\n\t\t\tmng.AMI = d.AMI\n\t\t}\n\t}\n\treturn cfg, nil\n}\n\ntype clusterConfigTemplateParams struct {\n\tUpOptions\n\tClusterName string\n\tRegion string\n}\n\nfunc (d *deployer) RenderClusterConfig() ([]byte, error) {\n\n\tcfg, err := d.CreateClusterConfig()\n\tif err != nil {\n\t\tslog.Error(\"failed to create ClusterConfig\", \"error\", err)\n\t}\n\tslog.Info(\"rendering cluster config yaml\", \"config\", cfg)\n\treturn yaml.Marshal(cfg)\n}\n" }, { "path": "internal/deployers/eksctl/deployer.go", "content": "package eksctl\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/aws/aws-k8s-tester/internal\"\n\t\"github.com/aws/aws-k8s-tester/internal/awssdk\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks\"\n\t\"github.com/spf13/pflag\"\n\t\"github.com/urfave/sflags/gen/gpflag\"\n\t\"sigs.k8s.io/kubetest2/pkg/types\"\n\t\"sigs.k8s.io/yaml\"\n)\n\n// DeployerName is the name of the deployer\nconst DeployerName = \"eksctl\"\n\ntype deployer struct {\n\t// generic parts\n\tcommonOptions types.Options\n\t*UpOptions\n\tawsConfig aws.Config\n\teksClient *eks.Client\n\tKubeconfigPath string `flag:\"kubeconfig\" desc:\"Path to kubeconfig\"`\n\t// ClusterName is the effective cluster name (from flag or RunID)\n\tclusterName string\n}\n\n// NewDeployer implements deployer.New for EKS using eksctl\nfunc NewDeployer(opts types.Options) (types.Deployer, *pflag.FlagSet) {\n\t// create a deployer object and set fields that are not flag controlled\n\tawsConfig := awssdk.NewConfig()\n\td := &deployer{\n\t\tcommonOptions: opts,\n\t\tawsConfig: awsConfig,\n\t\teksClient: eks.NewFromConfig(awsConfig),\n\t}\n\t// register flags and return\n\treturn d, bindFlags(d)\n}\n\nfunc (d *deployer) DumpClusterLogs() error {\n\treturn nil\n}\n\nfunc (d *deployer) Kubeconfig() (string, error) {\n\tif d.KubeconfigPath != \"\" {\n\t\treturn d.KubeconfigPath, nil\n\t}\n\treturn filepath.Join(d.commonOptions.RunDir(), \"kubeconfig\"), nil\n}\n\nfunc (d *deployer) Version() string {\n\treturn internal.Version\n}\n\n// bindFlags is a helper used to create & bind a flagset to the deployer\nfunc bindFlags(d *deployer) *pflag.FlagSet {\n\tflags, err := gpflag.Parse(d)\n\tif err != nil {\n\t\tslog.Error(\"unable to bind flags for deployer\")\n\t\tos.Exit(1)\n\t}\n\tflags.AddGoFlagSet(flag.CommandLine)\n\treturn flags\n}\n\n// initClusterName sets the effective cluster name with this precedence:\n// 1. config file\n// 2. --cluster-name flag\n// 3. RunID of the kubetest\nfunc (d *deployer) initClusterName() {\n\t// First priority: config file if provided\n\tif d.UpOptions.ConfigFile != \"\" {\n\t\tclusterName, err := d.parseClusterNameFromConfig(d.UpOptions.ConfigFile)\n\t\tif err == nil {\n\t\t\td.clusterName = clusterName\n\t\t\tslog.Debug(\"using cluster name from config file\", \"clusterName\", d.clusterName)\n\t\t\treturn\n\t\t}\n\t\tslog.Warn(\"failed to extract cluster name from config file\", \"error\", err)\n\t\t// Continue with other methods if parsing fails\n\t}\n\n\tif d.UpOptions.ClusterName != \"\" {\n\t\td.clusterName = d.UpOptions.ClusterName\n\t\tslog.Debug(\"using cluster name from flag\", \"clusterName\", d.clusterName)\n\t} else {\n\t\td.clusterName = d.commonOptions.RunID()\n\t\tslog.Debug(\"using RunID for cluster name\", \"clusterName\", d.clusterName)\n\t}\n}\n\n// parseClusterNameFromConfig extracts the cluster name from an eksctl config file\nfunc (d *deployer) parseClusterNameFromConfig(configFilePath string) (string, error) {\n\tconfigData, err := os.ReadFile(configFilePath)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to read config file: %v\", err)\n\t}\n\n\t// Simple YAML parsing to extract the cluster name\n\tvar configMap map[string]interface{}\n\tif err := yaml.Unmarshal(configData, &configMap); err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to parse config file YAML: %v\", err)\n\t}\n\n\t// Extract metadata section\n\tmetadata, ok := configMap[\"metadata\"].(map[string]interface{})\n\tif !ok {\n\t\treturn \"\", fmt.Errorf(\"metadata section missing in config file\")\n\t}\n\n\t// Extract name field\n\tname, ok := metadata[\"name\"].(string)\n\tif !ok || name == \"\" {\n\t\treturn \"\", fmt.Errorf(\"cluster name not found in config file metadata\")\n\t}\n\n\treturn name, nil\n}\n\n// assert that deployer implements types.DeployerWithKubeconfig\nvar _ types.DeployerWithKubeconfig = &deployer{}\n" }, { "path": "internal/deployers/eksctl/down.go", "content": "package eksctl\n\nimport (\n\t\"fmt\"\n\t\"log/slog\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/util\"\n)\n\nfunc (d *deployer) Down() error {\n\td.initClusterName()\n\n\tvar err error\n\n\tif d.DeployTarget == \"nodegroup\" {\n\t\tslog.Info(\"deleting nodegroup\", \"nodegroupName\", d.NodegroupName, \"clusterName\", d.clusterName)\n\t\terr = util.ExecuteCommand(\"eksctl\", \"delete\", \"nodegroup\", \"--cluster\", d.clusterName, \"--name\", d.NodegroupName, \"--drain=false\", \"--wait\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete nodegroup: %v\", err)\n\t\t}\n\t\tslog.Info(\"successfully deleted nodegroup\", \"nodegroupName\", d.NodegroupName, \"clusterName\", d.clusterName)\n\t} else if d.DeployTarget == \"cluster\" {\n\t\tslog.Info(\"deleting cluster\", \"clusterName\", d.clusterName)\n\t\terr = util.ExecuteCommand(\"eksctl\", \"delete\", \"cluster\", \"--name\", d.clusterName, \"--wait\", \"--disable-nodegroup-eviction\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete cluster: %v\", err)\n\t\t}\n\t\tslog.Info(\"successfully deleted cluster\", \"clusterName\", d.clusterName)\n\t} else {\n\t\treturn fmt.Errorf(\"Unsupported deploy target: %s, supported options: `cluster`, `nodegroup`.\", d.DeployTarget)\n\t}\n\treturn nil\n}\n" }, { "path": "internal/deployers/eksctl/up.go", "content": "package eksctl\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/util\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks\"\n\tekstypes \"github.com/aws/aws-sdk-go-v2/service/eks/types\"\n)\n\ntype UpOptions struct {\n\tRegion string `flag:\"region\" desc:\"AWS region for EKS cluster\"`\n\tKubernetesVersion string `flag:\"kubernetes-version\" desc:\"cluster Kubernetes version\"`\n\tNodes int `flag:\"nodes\" desc:\"number of nodes to launch in cluster\"`\n\tAMI string `flag:\"ami\" desc:\"Node AMI\"`\n\tInstanceTypes []string `flag:\"instance-types\" desc:\"Node instance types\"`\n\tConfigFile string `flag:\"config-file\" desc:\"Path to eksctl config file (if provided, other flags are ignored)\"`\n\tAvailabilityZones []string `flag:\"availability-zones\" desc:\"Node availability zones\"`\n\tAMIFamily string `flag:\"ami-family\" desc:\"AMI family to use (AmazonLinux2023, Bottlerocket)\"`\n\tEFAEnabled bool `flag:\"efa-enabled\" desc:\"Enable Elastic Fabric Adapter for the nodegroup\"`\n\tVolumeSize int `flag:\"volume-size\" desc:\"Size of the node root volume in GB\"`\n\tPrivateNetworking bool `flag:\"private-networking\" desc:\"Use private networking for nodes\"`\n\tWithOIDC bool `flag:\"with-oidc\" desc:\"Enable OIDC provider for IAM roles for service accounts\"`\n\tDeployTarget string `flag:\"deploy-target\" desc:\"The target to deploy, supported values: cluster | nodegroup (defaults to 'cluster'). It is a thin wrapper to eksctl create subcommand with limited supported values.\"`\n\tClusterName string `flag:\"cluster-name\" desc:\"Name of the EKS cluster (defaults to RunID if not specified)\"`\n\tUseUnmanagedNodegroup bool `flag:\"unmanaged-nodegroup\" desc:\"Use unmanaged nodegroup instead of managed nodegroup\"`\n\tNodegroupName string `flag:\"nodegroup-name\" desc:\"Name of the nodegroup (defaults to 'ng-1')\"`\n}\n\nfunc (d *deployer) verifyUpFlags() error {\n\tsupportedDeployTargets := []string{\"cluster\", \"nodegroup\"}\n\t// Skip validation if using a config file\n\tif d.ConfigFile != \"\" {\n\t\tslog.Info(\"using config file, skipping command-line flag validation\", \"configFile\", d.ConfigFile)\n\t\treturn nil\n\t}\n\n\tif d.KubernetesVersion == \"\" {\n\t\tslog.Info(\"--kubernetes-version is empty, attempting to detect it...\")\n\t\tdetectedVersion, err := detectKubernetesVersion()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"unable to detect --kubernetes-version, flag cannot be empty\")\n\t\t}\n\t\tslog.Info(\"detected kubernetes version\", \"version\", detectedVersion)\n\t\td.KubernetesVersion = detectedVersion\n\t}\n\tif d.Nodes < 0 {\n\t\treturn fmt.Errorf(\"number of nodes must be greater than zero\")\n\t}\n\tif d.Nodes == 0 {\n\t\td.Nodes = 4\n\t\tslog.Debug(\"using default number of nodes\", \"nodes\", d.Nodes)\n\t}\n\n\t// Validate instance types for unmanaged nodegroups\n\tif d.UseUnmanagedNodegroup {\n\t\tif len(d.InstanceTypes) > 1 {\n\t\t\treturn fmt.Errorf(\"Unmanaged nodegroups only support a single instance type. Using the first one: %s\", d.InstanceTypes[0])\n\t\t} else if len(d.InstanceTypes) == 0 {\n\t\t\t// If no instance type specified, use a default\n\t\t\td.InstanceTypes = []string{\"m5.xlarge\"}\n\t\t\tslog.Info(\"no instance type specified for unmanaged nodegroup, using default\", \"instanceType\", d.InstanceTypes[0])\n\t\t}\n\t}\n\n\tif d.DeployTarget != \"\" && !slices.Contains(supportedDeployTargets, d.DeployTarget) {\n\t\treturn fmt.Errorf(\"Unsupported deploy target: %s, supported options: `cluster`, `nodegroup`.\", d.DeployTarget)\n\t} else if d.DeployTarget == \"\" {\n\t\t\t// If no deploy target specified, use \"cluster\" as default\n\t\t\td.DeployTarget = \"cluster\"\n\t\t\tslog.Info(\"no deploy target specified, using default\", \"deployTarget\", d.DeployTarget)\n\t}\n\n\treturn nil\n}\n\nfunc (d *deployer) Up() error {\n\td.initClusterName()\n\n\tif err := d.verifyUpFlags(); err != nil {\n\t\treturn fmt.Errorf(\"up flags are invalid: %v\", err)\n\t}\n\n\tif d.UseUnmanagedNodegroup {\n\t\tslog.Info(\"using unmanaged nodegroup\", \"clusterName\", d.clusterName)\n\t} else {\n\t\tslog.Info(\"using managed nodegroup\", \"clusterName\", d.clusterName)\n\t}\n\n\tvar args []string\n\n\tif d.ConfigFile != \"\" {\n\t\t// If config file is provided, use it\n\t\targs = d.renderEksctlArgs(d.ConfigFile)\n\t} else {\n\t\t// Use rendered cluster config\n\t\tclusterConfig, err := d.RenderClusterConfig()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tslog.Info(\"rendered cluster config\", \"config\", string(clusterConfig))\n\n\t\tclusterConfigFile, err := os.CreateTemp(\"\", \"kubetest2-eksctl-cluster-config\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer clusterConfigFile.Close()\n\n\t\t_, err = clusterConfigFile.Write(clusterConfig)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\targs = d.renderEksctlArgs(clusterConfigFile.Name())\n\t}\n\n\terr := util.ExecuteCommand(\"eksctl\", args...)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create cluster: %v\", err)\n\t}\n\n\t// Write kubeconfig to the rundir\n\tkubeConfigPath, err := d.Kubeconfig()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error determining kubeconfig path: %v\", err)\n\t}\n\n\t// Create directory if it doesn't exist\n\terr = os.MkdirAll(filepath.Dir(kubeConfigPath), 0755)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error creating directory for kubeconfig: %v\", err)\n\t}\n\n\tslog.Info(\"writing kubeconfig\", \"path\", kubeConfigPath)\n\twriteKubeconfigArgs := []string{\n\t\t\"utils\",\n\t\t\"write-kubeconfig\",\n\t\t\"--cluster\", d.clusterName,\n\t\t\"--region\", d.Region,\n\t\t\"--kubeconfig\", kubeConfigPath,\n\t}\n\n\terr = util.ExecuteCommand(\"eksctl\", writeKubeconfigArgs...)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to write kubeconfig: %v\", err)\n\t}\n\n\tslog.Info(\"successfully wrote kubeconfig\", \"path\", kubeConfigPath)\n\td.KubeconfigPath = kubeConfigPath\n\treturn nil\n}\n\nfunc (d *deployer) renderEksctlArgs(configFilePath string) []string {\n\treturn []string{\n\t\t\"create\",\n\t\td.DeployTarget,\n\t\t\"--config-file\", configFilePath,\n\t}\n}\n\nfunc (d *deployer) IsUp() (up bool, err error) {\n\td.initClusterName()\n\n\tresult, err := d.eksClient.DescribeCluster(context.TODO(), &eks.DescribeClusterInput{\n\t\tName: aws.String(d.clusterName),\n\t})\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tswitch result.Cluster.Status {\n\tcase ekstypes.ClusterStatusActive:\n\t\treturn true, nil\n\tcase ekstypes.ClusterStatusCreating:\n\t\treturn false, nil\n\tdefault:\n\t\treturn false, fmt.Errorf(\"cluster status is: %v\", result.Cluster.Status)\n\t}\n}\n\nfunc detectKubernetesVersion() (string, error) {\n\tdetectedVersion, err := util.DetectKubernetesVersion()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tminorVersion, err := util.ParseMinorVersion(detectedVersion)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn minorVersion, nil\n}\n" }, { "path": "internal/e2e/client.go", "content": "package e2e\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"html/template\"\n\t\"io\"\n\t\"os\"\n\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/meta\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured\"\n\t\"k8s.io/apimachinery/pkg/runtime\"\n\t\"k8s.io/apimachinery/pkg/runtime/schema\"\n\t\"k8s.io/cli-runtime/pkg/resource\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n\t\"k8s.io/client-go/restmapper\"\n\t\"sigs.k8s.io/e2e-framework/klient/decoder\"\n\t\"sigs.k8s.io/e2e-framework/klient/k8s\"\n)\n\n// ApplyFiles creates Kubernetes objects contained in manifest file(s), in a manner similar to `kubectl apply -f`\n// Multiple objects may be in each manifest file.\n// The manifest files are processed in order.\nfunc ApplyFiles(restConfig *rest.Config, manifestFiles ...string) error {\n\tfor _, manifestFile := range manifestFiles {\n\t\tif f, err := os.Open(manifestFile); err != nil {\n\t\t\treturn err\n\t\t} else if err := applyManifests(restConfig, f); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// ApplyManifests creates Kubernetes objects contained in manifests, in a manner similar to `kubectl apply -f`\n// Multiple objects may be in the manifest data.\nfunc ApplyManifests(restConfig *rest.Config, manifests ...[]byte) error {\n\treturn applyManifests(restConfig, bytesSlicesToReaders(manifests...)...)\n}\n\nfunc applyManifests(restConfig *rest.Config, manifests ...io.Reader) error {\n\tfor _, manifest := range manifests {\n\t\tif objs, err := decoder.DecodeAll(context.TODO(), manifest); err != nil {\n\t\t\treturn err\n\t\t} else if err := processObjects(restConfig, objs, func(client *resource.Helper, obj k8s.Object) error {\n\t\t\tnamespace, err := meta.NewAccessor().Namespace(obj)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif namespace == \"\" {\n\t\t\t\tnamespace = \"default\"\n\t\t\t}\n\t\t\t_, err = client.Create(namespace, false, obj)\n\t\t\treturn err\n\t\t}); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// DeleteFiles deletes Kubernetes objects contained in manifest file(s), in a manner similar to `kubectl delete -f`\n// Multiple objects may be in each manifest file.\nfunc DeleteFiles(restConfig *rest.Config, manifestFiles ...string) error {\n\tfor _, manifestFile := range manifestFiles {\n\t\tif f, err := os.Open(manifestFile); err != nil {\n\t\t\treturn err\n\t\t} else if err := deleteManifests(restConfig, f); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// DeleteManifests deletes Kubernetes objects contained in manifest(s), in a manner similar to `kubectl delete -f`\n// Multiple objects may be in each manifest.\nfunc DeleteManifests(restConfig *rest.Config, manifests ...[]byte) error {\n\treturn deleteManifests(restConfig, bytesSlicesToReaders(manifests...)...)\n}\n\nfunc deleteManifests(restConfig *rest.Config, manifests ...io.Reader) error {\n\tfor _, manifest := range manifests {\n\t\tif objs, err := decoder.DecodeAll(context.TODO(), manifest); err != nil {\n\t\t\treturn err\n\t\t} else if err := processObjects(restConfig, objs, func(client *resource.Helper, obj k8s.Object) error {\n\t\t\tname, err := meta.NewAccessor().Name(obj)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tnamespace, err := meta.NewAccessor().Namespace(obj)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif namespace == \"\" {\n\t\t\t\tnamespace = \"default\"\n\t\t\t}\n\t\t\tdeletePolicy := metav1.DeletePropagationBackground\n\t\t\t_, err = client.DeleteWithOptions(namespace, name, &metav1.DeleteOptions{\n\t\t\t\tPropagationPolicy: &deletePolicy,\n\t\t\t})\n\t\t\treturn err\n\t\t}); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// RenderManifests renders manifests with the supplied data\nfunc RenderManifests(file []byte, templateData interface{}) ([]byte, error) {\n\ttpl, err := template.New(\"Manifest\").Parse(string(file))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbuf := bytes.Buffer{}\n\terr = tpl.Execute(&buf, templateData)\n\treturn buf.Bytes(), err\n}\n\n// GetJobLogs get logs from MPIJob\nfunc GetJobLogs(restConfig *rest.Config, job k8s.Object) (string, error) {\n\tctx := context.Background()\n\tclientset, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tvar jobLabel string\n\tswitch job.(type) {\n\tcase *unstructured.Unstructured: // assume this is an MPIJob\n\t\tjobLabel = fmt.Sprintf(\"job-name=%s-launcher\", job.GetName())\n\tcase *batchv1.Job:\n\t\tjobLabel = fmt.Sprintf(\"job-name=%s\", job.GetName())\n\tdefault:\n\t\treturn \"\", fmt.Errorf(\"unsupported job type %T\", job)\n\t}\n\tpods, err := clientset.CoreV1().Pods(job.GetNamespace()).List(ctx, metav1.ListOptions{LabelSelector: jobLabel})\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif len(pods.Items) == 0 {\n\t\treturn \"\", fmt.Errorf(\"no pods found for job %s\", job.GetName())\n\t}\n\tlog := clientset.CoreV1().Pods(job.GetNamespace()).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{})\n\tpodLogs, err := log.Stream(ctx)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer podLogs.Close()\n\tbuf := new(bytes.Buffer)\n\t_, err = io.Copy(buf, podLogs)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tstr := buf.String()\n\treturn str, nil\n}\n\nfunc bytesSlicesToReaders(byteSlices ...[]byte) []io.Reader {\n\tvar readers []io.Reader\n\tfor _, b := range byteSlices {\n\t\treaders = append(readers, bytes.NewReader(b))\n\t}\n\treturn readers\n}\n\n// processObjects applies a processFunc to each object, supplying it a dynamically-typed client appropriate for the object\nfunc processObjects(restConfig *rest.Config, objs []k8s.Object, processFunc func(client *resource.Helper, obj k8s.Object) error) error {\n\tclientset, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn err\n\t}\n\tgroupResources, err := restmapper.GetAPIGroupResources(clientset.Discovery())\n\tif err != nil {\n\t\treturn err\n\t}\n\trm := restmapper.NewDiscoveryRESTMapper(groupResources)\n\tfor _, obj := range objs {\n\t\tclient, err := newResourceHelper(restConfig, rm, obj)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tprocessFunc(client, obj)\n\t}\n\treturn nil\n}\n\nfunc newResourceHelper(restConfig *rest.Config, rm meta.RESTMapper, obj runtime.Object) (*resource.Helper, error) {\n\tgvk := obj.GetObjectKind().GroupVersionKind()\n\tgk := schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}\n\tmapping, err := rm.RESTMapping(gk, gvk.Version)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tgv := mapping.GroupVersionKind.GroupVersion()\n\trestConfig.ContentConfig = resource.UnstructuredPlusDefaultContentConfig()\n\trestConfig.GroupVersion = &gv\n\tif len(gv.Group) == 0 {\n\t\trestConfig.APIPath = \"/api\"\n\t} else {\n\t\trestConfig.APIPath = \"/apis\"\n\t}\n\trestClient, err := rest.RESTClientFor(restConfig)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn resource.NewHelper(restClient, mapping), nil\n}\n" }, { "path": "internal/e2e/conditions.go", "content": "package e2e\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tapimachinerywait \"k8s.io/apimachinery/pkg/util/wait\"\n\n\t\"sigs.k8s.io/e2e-framework/klient/k8s\"\n\t\"sigs.k8s.io/e2e-framework/klient/k8s/resources\"\n)\n\ntype ConditionExtension struct {\n\tresources *resources.Resources\n}\n\nfunc NewConditionExtension(r *resources.Resources) *ConditionExtension {\n\treturn &ConditionExtension{resources: r}\n}\n\n// ResourceMatch is a helper function used to check if the resource under question has met a pre-defined state. This can\n// be leveraged for checking fields on a resource that may not be immediately present upon creation.\nfunc (c *ConditionExtension) ResourceMatch(obj k8s.Object, matchFetcher func(object k8s.Object) bool) apimachinerywait.ConditionWithContextFunc {\n\treturn func(ctx context.Context) (done bool, err error) {\n\t\tif err := c.resources.Get(ctx, obj.GetName(), obj.GetNamespace(), obj); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\treturn matchFetcher(obj), nil\n\t}\n}\n\nfunc (c *ConditionExtension) PodRunning(pod k8s.Object) apimachinerywait.ConditionWithContextFunc {\n\treturn func(ctx context.Context) (done bool, err error) {\n\t\tif err := c.resources.Get(ctx, pod.GetName(), pod.GetNamespace(), pod); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tstatus := pod.(*v1.Pod).Status\n\t\tswitch status.Phase {\n\t\tcase v1.PodRunning:\n\t\t\treturn true, nil\n\t\tcase v1.PodPending:\n\t\t\treturn false, nil\n\t\tdefault:\n\t\t\treturn false, fmt.Errorf(\"pod cannot transition to running from current status: %s\", status.Phase)\n\t\t}\n\t}\n}\n\nfunc (c *ConditionExtension) PodSucceeded(pod k8s.Object) apimachinerywait.ConditionWithContextFunc {\n\treturn func(ctx context.Context) (done bool, err error) {\n\t\tif err := c.resources.Get(ctx, pod.GetName(), pod.GetNamespace(), pod); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tstatus := pod.(*v1.Pod).Status\n\t\tif status.Phase == v1.PodSucceeded {\n\t\t\treturn true, nil\n\t\t} else if status.Phase == v1.PodFailed {\n\t\t\treturn false, fmt.Errorf(\"Pod in Failed status\")\n\t\t}\n\t\treturn false, nil\n\t}\n}\n\nfunc (c *ConditionExtension) DaemonSetReady(daemonset k8s.Object) apimachinerywait.ConditionWithContextFunc {\n\treturn func(ctx context.Context) (done bool, err error) {\n\t\tif err := c.resources.Get(ctx, daemonset.GetName(), daemonset.GetNamespace(), daemonset); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tstatus := daemonset.(*appsv1.DaemonSet).Status\n\t\tif status.NumberReady == status.DesiredNumberScheduled && status.NumberUnavailable == 0 {\n\t\t\tdone = true\n\t\t}\n\t\treturn\n\t}\n}\n\nfunc (c *ConditionExtension) JobSucceeded(job k8s.Object) apimachinerywait.ConditionWithContextFunc {\n\treturn func(ctx context.Context) (done bool, err error) {\n\t\tif err := c.resources.Get(ctx, job.GetName(), job.GetNamespace(), job); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tbatchJob := job.(*batchv1.Job)\n\t\tstatus := batchJob.Status\n\t\tspec := batchJob.Spec\n\t\tfor _, condition := range status.Conditions {\n\t\t\tif condition.Type == batchv1.JobFailed && condition.Status == v1.ConditionTrue {\n\t\t\t\treturn false, fmt.Errorf(\"job failed\")\n\t\t\t}\n\t\t}\n\t\tif status.Succeeded != *spec.Completions {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn true, nil\n\t}\n}\n\nfunc (c *ConditionExtension) AllNodesHaveNonZeroResourceCapacity(resourceLabel string) apimachinerywait.ConditionWithContextFunc {\n\treturn func(ctx context.Context) (done bool, err error) {\n\t\tnodeList := &v1.NodeList{}\n\t\tif err := c.resources.List(ctx, nodeList); err != nil {\n\t\t\treturn false, fmt.Errorf(\"failed to list nodes: %w\", err)\n\t\t}\n\t\tif len(nodeList.Items) == 0 {\n\t\t\treturn false, fmt.Errorf(\"no nodes found in the cluster\")\n\t\t}\n\t\tfor _, node := range nodeList.Items {\n\t\t\tresource, ok := node.Status.Capacity[v1.ResourceName(resourceLabel)]\n\t\t\tif !ok {\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t\tif resource.Value() <= 0 {\n\t\t\t\treturn false, nil\n\t\t\t}\n\t\t}\n\t\treturn true, nil\n\t}\n}\n" }, { "path": "internal/e2e/doc.go", "content": "// Package frameworkext contains extensions to sigs.k8s.io/e2e-framework\npackage e2e\n" }, { "path": "internal/e2e/ec2.go", "content": "package e2e\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/awssdk\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ec2\"\n\tec2types \"github.com/aws/aws-sdk-go-v2/service/ec2/types\"\n)\n\ntype EC2Client interface {\n\tDescribeInstanceType(instanceType string) (ec2types.InstanceTypeInfo, error)\n}\n\ntype ec2Client struct {\n\tclient *ec2.Client\n}\n\nfunc NewEC2Client() *ec2Client {\n\treturn &ec2Client{\n\t\tclient: ec2.NewFromConfig(awssdk.NewConfig()),\n\t}\n}\n\nfunc (c *ec2Client) DescribeInstanceTopology(instanceIDs []string) ([]ec2types.InstanceTopology, error) {\n\tvar instanceTopologies []ec2types.InstanceTopology\n\tpaginator := ec2.NewDescribeInstanceTopologyPaginator(c.client, &ec2.DescribeInstanceTopologyInput{\n\t\tInstanceIds: instanceIDs,\n\t})\n\tfor paginator.HasMorePages() {\n\t\tinstanceTopologyOuput, err := paginator.NextPage(context.TODO())\n\t\tif err != nil {\n\t\t\treturn []ec2types.InstanceTopology{}, err\n\t\t}\n\t\tinstanceTopologies = append(instanceTopologies, instanceTopologyOuput.Instances...)\n\t}\n\treturn instanceTopologies, nil\n}\n\nfunc (c *ec2Client) DescribeInstanceType(instanceType string) (ec2types.InstanceTypeInfo, error) {\n\tdescribeResponse, err := c.client.DescribeInstanceTypes(context.TODO(), &ec2.DescribeInstanceTypesInput{\n\t\tInstanceTypes: []ec2types.InstanceType{ec2types.InstanceType(instanceType)},\n\t})\n\tif err != nil {\n\t\treturn ec2types.InstanceTypeInfo{}, fmt.Errorf(\"failed to describe instance type: %s: %v\", instanceType, err)\n\t} else {\n\t\treturn describeResponse.InstanceTypes[0], nil\n\t}\n}\n" }, { "path": "internal/e2e/health.go", "content": "package e2e\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n)\n\n// KubeletIsResponsive returns true if the kubelet /healthz endpoint responds with a 200 status code, and propagates\n// any non-connection specific errors\nfunc KubeletIsResponsive(ctx context.Context, cfg *rest.Config, nodeName string) (bool, error) {\n\tclient, err := kubernetes.NewForConfig(cfg)\n\tif err != nil {\n\t\treturn false, fmt.Errorf(\"failed to initialize client set: %v\", err)\n\t}\n\n\tnodeHealthResponse := client.CoreV1().RESTClient().Get().Resource(\"nodes\").\n\t\tName(nodeName).SubResource(\"proxy\").Suffix(\"/healthz\").\n\t\tDo(ctx)\n\n\tif nodeHealthResponse.Error() != nil {\n\t\terrMsg := nodeHealthResponse.Error().Error()\n\t\t// TODO: match errors against types, e.g. syscall.ECONNREFUSED instead, the k8s client doesn't\n\t\t// currently properly wrap the underlying error to allow this though\n\t\tif strings.Contains(errMsg, \"connection refused\") ||\n\t\t\tstrings.Contains(errMsg, \"connection reset by peer\") ||\n\t\t\tstrings.Contains(errMsg, \"http2: client connection lost\") {\n\t\t\t// these errors indicate reachability to the node in general but an unstable connection to kubelet\n\t\t\treturn false, nil\n\t\t}\n\n\t\t// propagate other errors, e.g. i/o timeout, that may result from things unrelated to kubelet health,\n\t\t// e.g. security group rules on the instance restricting traffic from the CP\n\t\treturn false, fmt.Errorf(\"could not reach /healthz endpoint for node %s: %w\", nodeName, nodeHealthResponse.Error())\n\t}\n\n\tvar statusCode int\n\tnodeHealthResponse.StatusCode(&statusCode)\n\treturn statusCode == 200, nil\n}\n" }, { "path": "internal/e2e/logs.go", "content": "package e2e\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"testing\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n)\n\n// PrintDaemonSetPodLogs retrieves logs from each container in each pod of a DaemonSet.\n// namespace & labelSelector identify the DaemonSet's pods (e.g. \"default\", \"app=containerd-check\").\nfunc PrintDaemonSetPodLogs(\n\tt *testing.T,\n\tctx context.Context,\n\trestConfig *rest.Config,\n\tnamespace string,\n\tlabelSelector string,\n) {\n\tclientset, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\tt.Logf(\"failed to create typed clientset: %v\", err)\n\t\treturn\n\t}\n\n\tpods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{\n\t\tLabelSelector: labelSelector,\n\t})\n\tif err != nil {\n\t\tt.Logf(\"failed to list pods: %v\", err)\n\t\treturn\n\t}\n\tif len(pods.Items) == 0 {\n\t\tt.Logf(\"No pods found for DaemonSet with label %q in namespace %q.\", labelSelector, namespace)\n\t\treturn\n\t}\n\n\tfor _, pod := range pods.Items {\n\t\tt.Logf(\"Pod %s status: %s\", pod.Name, pod.Status.Phase)\n\t\tfor _, container := range pod.Spec.Containers {\n\t\t\tlogs, logErr := ReadPodLogs(ctx, restConfig, pod.Namespace, pod.Name, container.Name)\n\t\t\tif logErr != nil {\n\t\t\t\tt.Logf(\"Failed reading logs from %s/%s: %v\", pod.Name, container.Name, logErr)\n\t\t\t} else {\n\t\t\t\tt.Logf(\"=== Logs from %s/%s ===\\n%s\", pod.Name, container.Name, logs)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// ReadPodLogs streams logs for a specific container in a pod.\nfunc ReadPodLogs(\n\tctx context.Context,\n\trestConfig *rest.Config,\n\tnamespace, podName, containerName string,\n) (string, error) {\n\tclientset, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to create typed clientset: %w\", err)\n\t}\n\treq := clientset.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{\n\t\tContainer: containerName,\n\t})\n\tstream, err := req.Stream(ctx)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"failed to open log stream for %s/%s: %w\", podName, containerName, err)\n\t}\n\tdefer stream.Close()\n\n\tdata, err := io.ReadAll(stream)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"error reading logs: %w\", err)\n\t}\n\treturn string(data), nil\n}\n" }, { "path": "internal/e2e/mpijobs/conditions.go", "content": "package mpijobs\n\nimport (\n\t\"fmt\"\n\n\t\"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured\"\n\t\"sigs.k8s.io/e2e-framework/klient/k8s\"\n)\n\n// MPIJobSucceeded returns true if the specified k8s.Object is an unstructured.Unstructured\n// with .status.conditions[\"Succeeded\"] = \"True\"\nfunc MPIJobSucceeded(obj k8s.Object) bool {\n\tu := obj.(*unstructured.Unstructured)\n\tconditions, found, err := unstructured.NestedSlice(u.Object, \"status\", \"conditions\")\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"MPIJob does not match expected schema: %v\", err))\n\t}\n\tif !found {\n\t\treturn false\n\t}\n\tfor _, condition := range conditions {\n\t\tc := condition.(map[string]interface{})\n\t\tcType, found, err := unstructured.NestedString(c, \"type\")\n\t\tif err != nil {\n\t\t\tpanic(fmt.Errorf(\"MPIJob does not match expected schema: %v\", err))\n\t\t}\n\t\tif !found {\n\t\t\tcontinue\n\t\t}\n\t\tif cType == \"Succeeded\" {\n\t\t\tcStatus, found, err := unstructured.NestedString(c, \"status\")\n\t\t\tif err != nil {\n\t\t\t\tpanic(fmt.Errorf(\"MPIJob does not match expected schema: %v\", err))\n\t\t\t}\n\t\t\tif !found {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn cStatus == \"True\"\n\t\t}\n\t}\n\treturn false\n}\n" }, { "path": "internal/e2e/mpijobs/conditions_test.go", "content": "package mpijobs\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured\"\n)\n\nfunc Test_MPIJobSucceeded(t *testing.T) {\n\tu := unstructured.Unstructured{\n\t\tObject: map[string]interface{}{\n\t\t\t\"status\": map[string]interface{}{\n\t\t\t\t\"conditions\": []interface{}{\n\t\t\t\t\tmap[string]interface{}{\n\t\t\t\t\t\t\"type\": \"Succeeded\",\n\t\t\t\t\t\t\"status\": \"True\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tassert.True(t, MPIJobSucceeded(&u))\n\n\tu = unstructured.Unstructured{\n\t\tObject: map[string]interface{}{\n\t\t\t\"status\": map[string]interface{}{\n\t\t\t\t\"conditions\": []interface{}{\n\t\t\t\t\tmap[string]interface{}{\n\t\t\t\t\t\t\"type\": \"Succeeded\",\n\t\t\t\t\t\t\"status\": \"False\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tassert.False(t, MPIJobSucceeded(&u))\n}\n" }, { "path": "internal/e2e/mpijobs/types.go", "content": "package mpijobs\n\nimport (\n\t\"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured\"\n\t\"k8s.io/apimachinery/pkg/runtime/schema\"\n)\n\nvar MPIJobGVK = schema.GroupVersionKind{\n\tGroup: \"kubeflow.org\",\n\tVersion: \"v2beta1\",\n\tKind: \"MPIJob\",\n}\n\nfunc NewUnstructured(name, namespace string) *unstructured.Unstructured {\n\tu := unstructured.Unstructured{}\n\tu.SetGroupVersionKind(MPIJobGVK)\n\tu.SetName(name)\n\tu.SetNamespace(namespace)\n\treturn &u\n}\n" }, { "path": "internal/e2e/resources.go", "content": "package e2e\n\nimport (\n\t\"fmt\"\n\n\tv1 \"k8s.io/api/core/v1\"\n)\n\nfunc GetNonZeroResourceCapacity(node *v1.Node, resourceName string) (int, error) {\n\tcapacity, ok := node.Status.Capacity[v1.ResourceName(resourceName)]\n\tif !ok {\n\t\treturn 0, fmt.Errorf(\"node %q has no resource %q\", node.Name, resourceName)\n\t}\n\tif capacity.Value() == 0 {\n\t\treturn 0, fmt.Errorf(\"node %q has zero capacity for resource %q\", node.Name, resourceName)\n\t}\n\treturn int(capacity.Value()), nil\n}\n" }, { "path": "internal/metrics/cloudwatch.go", "content": "package metrics\n\nimport (\n\t\"context\"\n\t\"log/slog\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudwatch\"\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types\"\n\t\"github.com/aws/aws-sdk-go/aws\"\n)\n\n// NewCloudWatchRegistry creates a new metric registry that will emit values using the specified cloudwatch client\nfunc NewCloudWatchRegistry(cw *cloudwatch.Client) MetricRegistry {\n\treturn &cloudwatchRegistry{\n\t\tcw: cw,\n\t\tlock: &sync.Mutex{},\n\t\tdataByNamespace: make(map[string][]*cloudwatchMetricDatum),\n\t}\n}\n\ntype cloudwatchRegistry struct {\n\tcw *cloudwatch.Client\n\tlock *sync.Mutex\n\tdataByNamespace map[string][]*cloudwatchMetricDatum\n}\n\ntype cloudwatchMetricDatum struct {\n\tspec *MetricSpec\n\tvalue float64\n\tdimensions map[string]string\n\ttimestamp time.Time\n}\n\nfunc (r *cloudwatchRegistry) Record(spec *MetricSpec, value float64, dimensions map[string]string) {\n\tr.lock.Lock()\n\tdefer r.lock.Unlock()\n\tr.dataByNamespace[spec.Namespace] = append(r.dataByNamespace[spec.Namespace], &cloudwatchMetricDatum{\n\t\tspec: spec,\n\t\tvalue: value,\n\t\tdimensions: dimensions,\n\t\ttimestamp: time.Now(),\n\t})\n}\n\nfunc (r *cloudwatchRegistry) Emit() error {\n\tr.lock.Lock()\n\tdefer r.lock.Unlock()\n\tfor namespace, data := range r.dataByNamespace {\n\t\tfor i := 0; i < len(data); {\n\t\t\tvar metricData []types.MetricDatum\n\t\t\t// we can emit up to 1000 values per PutMetricData\n\t\t\tfor j := 0; j < len(data) && j < 1000; j++ {\n\t\t\t\tdatum := data[i]\n\t\t\t\tvar dimensions []types.Dimension\n\t\t\t\tfor key, val := range datum.dimensions {\n\t\t\t\t\tdimensions = append(dimensions, types.Dimension{\n\t\t\t\t\t\tName: aws.String(key),\n\t\t\t\t\t\tValue: aws.String(val),\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t\tmetricData = append(metricData, types.MetricDatum{\n\t\t\t\t\tMetricName: aws.String(datum.spec.Metric),\n\t\t\t\t\tValue: aws.Float64(datum.value),\n\t\t\t\t\tDimensions: dimensions,\n\t\t\t\t\tTimestamp: &datum.timestamp,\n\t\t\t\t})\n\t\t\t\ti++\n\t\t\t}\n\t\t\t_, err := r.cw.PutMetricData(context.TODO(), &cloudwatch.PutMetricDataInput{\n\t\t\t\tNamespace: aws.String(namespace),\n\t\t\t\tMetricData: metricData,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tslog.Info(\"emitted metrics\", \"count\", len(data), \"namespace\", namespace)\n\t}\n\tr.dataByNamespace = make(map[string][]*cloudwatchMetricDatum)\n\treturn nil\n}\n\nfunc (r *cloudwatchRegistry) GetRegistered() int {\n\tr.lock.Lock()\n\tdefer r.lock.Unlock()\n\tregistered := 0\n\tfor _, data := range r.dataByNamespace {\n\t\tregistered += len(data)\n\t}\n\treturn registered\n}\n" }, { "path": "internal/metrics/noop.go", "content": "package metrics\n\nfunc NewNoopMetricRegistry() MetricRegistry {\n\treturn &noopRegistry{}\n}\n\ntype noopRegistry struct{}\n\nfunc (r *noopRegistry) Record(spec *MetricSpec, value float64, dimensions map[string]string) {}\n\nfunc (r *noopRegistry) Emit() error {\n\treturn nil\n}\n" }, { "path": "internal/metrics/registry.go", "content": "package metrics\n\nimport (\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types\"\n)\n\ntype MetricRegistry interface {\n\t// Record adds a new metric value to the registry\n\tRecord(spec *MetricSpec, value float64, dimensions map[string]string)\n\t// Emit sends all registered metric values to cloudwatch, emptying the registry\n\tEmit() error\n}\n\ntype MetricSpec struct {\n\tNamespace string\n\tMetric string\n\tUnit types.StandardUnit\n}\n" }, { "path": "internal/testers/ginkgov1/LICENSE.original", "content": "THIS IS A COPY OF THE ORIGINAL LICENSE FOR `kubetest2` AT COMMIT `d7fcb799ce84ceda66c8b9b1ec8eefcbe226f293`.\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "internal/testers/ginkgov1/README.md", "content": "This tester supports ginkgo 1.x versions, which were used for Kubernetes versions prior to 1.25.\n\n---\n\nThis is a fork of the `ginkgo` tester: https://github.com/kubernetes-sigs/kubetest2/tree/master/pkg/testers/ginkgo\n\nThe fork originated at commit `d7fcb799ce84ceda66c8b9b1ec8eefcbe226f293`.\n\nA copy of the original license is provided in the file named `LICENSE.original`.\n" }, { "path": "internal/testers/ginkgov1/ginkgo.go", "content": "// This file has been modified in the following ways:\n// 1. The `ginkgo` package has been renamed to `ginkgov1`.\n// 2. The `--timeout` flag has been removed.\n// 3. The `--flake-attempts` flag has been implemented for ginkgo 1.x versions.\n\n/*\nCopyright 2019 The Kubernetes Authors.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage ginkgov1\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\tstdexec \"os/exec\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/kballard/go-shellquote\"\n\t\"github.com/urfave/sflags/gen/gpflag\"\n\t\"log/slog\"\n\n\t\"sigs.k8s.io/kubetest2/pkg/artifacts\"\n\t\"sigs.k8s.io/kubetest2/pkg/build\"\n\t\"sigs.k8s.io/kubetest2/pkg/exec\"\n\t\"sigs.k8s.io/kubetest2/pkg/testers\"\n)\n\nvar GitTag string\n\ntype Tester struct {\n\tFlakeAttempts int `desc:\"Make up to this many attempts to run each spec.\"`\n\tGinkgoArgs string `desc:\"Additional arguments supported by the ginkgo binary.\"`\n\tParallel int `desc:\"Run this many tests in parallel at once.\"`\n\tSkipRegex string `desc:\"Regular expression of jobs to skip.\"`\n\tFocusRegex string `desc:\"Regular expression of jobs to focus on.\"`\n\tTestPackageVersion string `desc:\"The ginkgo tester uses a test package made during the kubernetes build. The tester downloads this test package from one of the release tars published to the Release bucket. Defaults to latest. visit https://kubernetes.io/releases/ to find release names. Example: v1.20.0-alpha.0\"`\n\tTestPackageBucket string `desc:\"The bucket which release tars will be downloaded from to acquire the test package. Defaults to the main kubernetes project bucket.\"`\n\tTestPackageDir string `desc:\"The directory in the bucket which represents the type of release. Default to the release directory.\"`\n\tTestPackageMarker string `desc:\"The version marker in the directory containing the package version to download when unspecified. Defaults to latest.txt.\"`\n\tTestArgs string `desc:\"Additional arguments supported by the e2e test framework (https://godoc.org/k8s.io/kubernetes/test/e2e/framework#TestContextType).\"`\n\tUseBuiltBinaries bool `desc:\"Look for binaries in _rundir/$KUBETEST2_RUN_DIR instead of extracting from tars downloaded from GCS.\"`\n\tUseBinariesFromPath bool `desc:\"Look for binaries in the $PATH instead of extracting from tars downloaded from GCS.\"`\n\tEnv []string `desc:\"List of env variables to pass to ginkgo libraries\"`\n\n\tkubeconfigPath string\n\trunDir string\n\n\t// These paths are set up by AcquireTestPackage()\n\te2eTestPath string\n\tginkgoPath string\n\tkubectlPath string\n}\n\n// Test runs the test\nfunc (t *Tester) Test() error {\n\tif err := testers.WriteVersionToMetadata(GitTag, \"\"); err != nil {\n\t\treturn err\n\t}\n\n\tif err := t.pretestSetup(); err != nil {\n\t\treturn err\n\t}\n\n\te2eTestArgs := []string{\n\t\t\"--kubeconfig=\" + t.kubeconfigPath,\n\t\t\"--kubectl-path=\" + t.kubectlPath,\n\t\t\"--ginkgo.skip=\" + t.SkipRegex,\n\t\t\"--ginkgo.focus=\" + t.FocusRegex,\n\t\t\"--report-dir=\" + artifacts.BaseDir(),\n\t}\n\n\t// some ginkgo flags and behaviors are not backwards compatible\n\tswitch v := t.ginkgoMajorVersion(); v {\n\tcase \"1\":\n\t\te2eTestArgs = append(e2eTestArgs,\n\t\t\t\"--ginkgo.flakeAttempts=\"+strconv.Itoa(t.FlakeAttempts),\n\t\t)\n\tcase \"2\":\n\t\te2eTestArgs = append(e2eTestArgs,\n\t\t\t\"--ginkgo.flake-attempts=\"+strconv.Itoa(t.FlakeAttempts),\n\t\t)\n\tdefault:\n\t\treturn fmt.Errorf(\"unsupported ginkgo version: %s\", v)\n\t}\n\n\textraE2EArgs, err := shellquote.Split(t.TestArgs)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error parsing --test-args: %v\", err)\n\t}\n\te2eTestArgs = append(e2eTestArgs, extraE2EArgs...)\n\n\textraGingkoArgs, err := shellquote.Split(t.GinkgoArgs)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error parsing --gingko-args: %v\", err)\n\t}\n\n\tginkgoArgs := append(extraGingkoArgs,\n\t\t\"--nodes=\"+strconv.Itoa(t.Parallel),\n\t\tt.e2eTestPath,\n\t\t\"--\")\n\tginkgoArgs = append(ginkgoArgs, e2eTestArgs...)\n\n\tslog.Info(\"running ginkgo test\", \"path\", t.ginkgoPath, \"args\", ginkgoArgs)\n\tcmd := exec.Command(t.ginkgoPath, ginkgoArgs...)\n\tcmd.SetEnv(t.Env...)\n\texec.InheritOutput(cmd)\n\treturn cmd.Run()\n}\n\nfunc (t *Tester) pretestSetup() error {\n\tif config := os.Getenv(\"KUBECONFIG\"); config != \"\" {\n\t\t// The ginkgo tester errors out if the kubeconfig provided\n\t\t// is not an absolute path, likely because ginkgo changes its\n\t\t// working directory while executing. To get around this problem\n\t\t// we can manually edit the provided KUBECONFIG to ensure a\n\t\t// successful run.\n\t\tif !filepath.IsAbs(config) {\n\t\t\tnewKubeconfig, err := filepath.Abs(config)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to convert kubeconfig to absolute path: %s\", err)\n\t\t\t}\n\t\t\tslog.Info(\"ginkgo tester received non-absolute KUBECONFIG path, updating\", \"path\", newKubeconfig)\n\t\t\tconfig = newKubeconfig\n\t\t}\n\n\t\tt.kubeconfigPath = config\n\t} else {\n\t\thome, err := os.UserHomeDir()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to find home directory: %v\", err)\n\t\t}\n\t\tt.kubeconfigPath = filepath.Join(home, \".kube\", \"config\")\n\t}\n\tslog.Info(\"using kubeconfig\", \"path\", t.kubeconfigPath)\n\n\tif t.UseBuiltBinaries {\n\t\treturn t.validateLocalBinaries()\n\t}\n\tif t.UseBinariesFromPath {\n\t\treturn t.validateBinariesFromPath()\n\t}\n\n\tif err := t.AcquireTestPackage(); err != nil {\n\t\treturn fmt.Errorf(\"failed to get ginkgo test package from published releases: %s\", err)\n\t}\n\n\treturn nil\n}\n\nfunc (t *Tester) validateLocalBinaries() error {\n\tslog.Debug(\"checking existing test binaries...\")\n\tfor _, binary := range build.CommonTestBinaries {\n\t\tpath := filepath.Join(t.runDir, binary)\n\t\tif _, err := os.Stat(path); err != nil {\n\t\t\tlogPath := path\n\t\t\tif abspath, err := filepath.Abs(path); err != nil {\n\t\t\t\tslog.Warn(\"failed to convert path to absolute path\", \"path\", path, \"error\", err)\n\t\t\t} else {\n\t\t\t\tlogPath = abspath\n\t\t\t}\n\t\t\treturn fmt.Errorf(\"failed to validate pre-built binary %s (checked at %q): %w\", binary, logPath, err)\n\t\t}\n\t\tslog.Debug(\"found existing binary\", \"binary\", binary, \"path\", path)\n\t}\n\tt.e2eTestPath = filepath.Join(t.runDir, \"e2e.test\")\n\tt.ginkgoPath = filepath.Join(t.runDir, \"ginkgo\")\n\tt.kubectlPath = filepath.Join(t.runDir, \"kubectl\")\n\treturn nil\n}\n\nfunc (t *Tester) validateBinariesFromPath() error {\n\tslog.Debug(\"checking for test binaries on PATH...\")\n\tfor _, binary := range build.CommonTestBinaries {\n\t\tpath, err := stdexec.LookPath(binary)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to validate binary %s from PATH: %w\", binary, err)\n\t\t}\n\t\tslog.Debug(\"found existing binary\", \"binary\", binary, \"path\", path)\n\t\tswitch binary {\n\t\tcase \"e2e.test\":\n\t\t\tt.e2eTestPath = path\n\t\tcase \"ginkgo\":\n\t\t\tt.ginkgoPath = path\n\t\tcase \"kubectl\":\n\t\t\tt.kubectlPath = path\n\t\t}\n\t}\n\treturn nil\n}\n\n// ginkgoMajorVersion returns the ginkgo major version\n// empty if not found\nfunc (t *Tester) ginkgoMajorVersion() string {\n\tslog.Debug(\"checking ginkgo version...\")\n\tcmd := exec.Command(t.ginkgoPath, \"version\")\n\tlines, err := exec.OutputLines(cmd)\n\tif err != nil || len(lines) != 1 {\n\t\treturn \"\"\n\t}\n\t// the output is in the format\n\t// Ginkgo Version 1.14.0\n\t// Ginkgo Version 2.1.4\n\tparts := strings.Split(lines[0], \" \")\n\tif len(parts) != 3 {\n\t\treturn \"\"\n\t}\n\tvers := strings.Split(parts[2], \".\")\n\tif len(vers) != 3 {\n\t\treturn \"\"\n\t}\n\treturn vers[0]\n}\n\nfunc (t *Tester) Execute() error {\n\tfs, err := gpflag.Parse(t)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to initialize tester: %v\", err)\n\t}\n\n\tfs.AddGoFlagSet(flag.CommandLine)\n\n\thelp := fs.BoolP(\"help\", \"h\", false, \"\")\n\n\tif err := fs.Parse(os.Args); err != nil {\n\t\treturn fmt.Errorf(\"failed to parse flags: %v\", err)\n\t}\n\n\tif *help {\n\t\tfs.SetOutput(os.Stdout)\n\t\tfs.PrintDefaults()\n\t\treturn nil\n\t}\n\n\tif err := t.initKubetest2Info(); err != nil {\n\t\treturn err\n\t}\n\treturn t.Test()\n}\n\n// initializes relevant information from the well defined kubetest2 environment variables.\nfunc (t *Tester) initKubetest2Info() error {\n\tif t.UseBuiltBinaries && t.UseBinariesFromPath {\n\t\treturn fmt.Errorf(\"--use-built-binaries and --use-binaries-from-path are mutually exclusive\")\n\t}\n\tif dir, ok := os.LookupEnv(\"KUBETEST2_RUN_DIR\"); ok {\n\t\tt.runDir = dir\n\t\treturn nil\n\t}\n\t// ginkgo/e2e.test/kubectl can be found in rundir when they are built\n\tif t.UseBuiltBinaries {\n\t\tt.runDir = artifacts.RunDir()\n\t\treturn nil\n\t}\n\t// default to current working directory if for some reason the env is not set\n\tdir, err := os.Getwd()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to set run dir: %v\", err)\n\t}\n\tt.runDir = dir\n\treturn nil\n}\n\nfunc (t *Tester) SetRunDir(dir string) {\n\tt.runDir = dir\n}\n\nfunc NewDefaultTester() *Tester {\n\treturn &Tester{\n\t\tFlakeAttempts: 1,\n\t\tParallel: 1,\n\t\tTestPackageBucket: \"kubernetes-release\",\n\t\tTestPackageDir: \"release\",\n\t\tTestPackageMarker: \"latest.txt\",\n\t\tEnv: nil,\n\t}\n}\n\nfunc Main() {\n\tt := NewDefaultTester()\n\tif err := t.Execute(); err != nil {\n\t\tslog.Error(\"failed to run ginkgo tester\", \"error\", err)\n\t\tos.Exit(1)\n\t}\n}\n" }, { "path": "internal/testers/ginkgov1/kubectl/kubectl.go", "content": "/*\nCopyright 2019 The Kubernetes Authors.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage kubectl\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\n\t\"sigs.k8s.io/kubetest2/pkg/exec\"\n)\n\nconst (\n\tkubectl = \"kubectl\"\n)\n\n// APIServerURL obtains the URL of the k8s master from kubectl\nfunc APIServerURL() (string, error) {\n\tkubecontext, err := execAndResult(kubectl, \"config\", \"view\", \"-o\", \"jsonpath=\\\"{.current-context}\\\"\")\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"Could not get kube context: %v\", err)\n\t}\n\n\tclustername, err := execAndResult(kubectl, \"config\", \"view\", \"-o\",\n\t\tfmt.Sprintf(\"jsonpath=\\\"{.contexts[?(@.name == %s)].context.cluster}\\\"\", kubecontext))\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"Could not get cluster name: %v\", err)\n\t}\n\n\tapiServerURL, err := execAndResult(kubectl, \"config\", \"view\", \"-o\",\n\t\tfmt.Sprintf(\"jsonpath={.clusters[?(@.name == %s)].cluster.server}\", clustername))\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn apiServerURL, nil\n}\n\n// execAndResult runs command with args and returns the entire output (or error)\nfunc execAndResult(command string, args ...string) (string, error) {\n\tcmd := exec.Command(command, args...)\n\tcmd.SetStderr(os.Stderr)\n\tbytes, err := exec.Output(cmd)\n\treturn string(bytes), err\n}\n" }, { "path": "internal/testers/ginkgov1/package.go", "content": "// This file has been modified in the following ways:\n// 1. The `ginkgo` package has been renamed to `ginkgov1`.\n\n/*\nCopyright 2019 The Kubernetes Authors.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n*/\n\npackage ginkgov1\n\nimport (\n\t\"archive/tar\"\n\t\"compress/gzip\"\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"log/slog\"\n\t\"sigs.k8s.io/kubetest2/pkg/artifacts\"\n\t\"sigs.k8s.io/kubetest2/pkg/exec\"\n)\n\n// AcquireTestPackage obtains three test binaries and places them in $KUBETEST2_RUN_DIR.\n// The first is \"ginkgo\", the actual ginkgo executable.\n// The second is \"e2e.test\", which contains kubernetes e2e test cases.\n// The third is \"kubectl\".\nfunc (t *Tester) AcquireTestPackage() error {\n\t// first, get the name of the latest release (e.g. v1.20.0-alpha.0)\n\tif t.TestPackageVersion == \"\" {\n\t\tcmd := exec.Command(\n\t\t\t\"gsutil\",\n\t\t\t\"cat\",\n\t\t\tfmt.Sprintf(\"gs://%s/%s/%s\", t.TestPackageBucket, t.TestPackageDir, t.TestPackageMarker),\n\t\t)\n\t\tlines, err := exec.OutputLines(cmd)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to get latest release name: %s\", err)\n\t\t}\n\t\tif len(lines) == 0 {\n\t\t\treturn fmt.Errorf(\"getting latest release name had no output\")\n\t\t}\n\t\tt.TestPackageVersion = lines[0]\n\n\t\tslog.Info(\"test package version not specified, using default\", \"marker\", t.TestPackageMarker, \"version\", t.TestPackageVersion)\n\t}\n\n\treleaseTar := fmt.Sprintf(\"kubernetes-test-%s-%s.tar.gz\", runtime.GOOS, runtime.GOARCH)\n\n\tdownloadDir, err := os.UserCacheDir()\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to get user cache directory: %v\", err)\n\t}\n\n\tdownloadPath := filepath.Join(downloadDir, releaseTar)\n\n\tif err := t.ensureReleaseTar(downloadPath, releaseTar); err != nil {\n\t\treturn err\n\t}\n\tif err := t.extractBinaries(downloadPath); err != nil {\n\t\treturn err\n\t}\n\n\tt.kubectlPath = filepath.Join(artifacts.RunDir(), \"kubectl\")\n\treturn t.ensureKubectl(t.kubectlPath)\n}\n\nfunc (t *Tester) extractBinaries(downloadPath string) error {\n\t// ensure the artifacts dir\n\tif err := os.MkdirAll(artifacts.BaseDir(), os.ModePerm); err != nil {\n\t\treturn err\n\t}\n\t// ensure the rundir\n\tif err := os.MkdirAll(artifacts.RunDir(), os.ModePerm); err != nil {\n\t\treturn err\n\t}\n\n\t// Extract files from the test package\n\tf, err := os.Open(downloadPath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to open downloaded tar at %s: %s\", downloadPath, err)\n\t}\n\tdefer f.Close()\n\tgzf, err := gzip.NewReader(f)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create gzip reader: %s\", err)\n\t}\n\tdefer gzf.Close()\n\n\ttarReader := tar.NewReader(gzf)\n\n\t// Map of paths in archive to destination paths\n\tt.e2eTestPath = filepath.Join(artifacts.RunDir(), \"e2e.test\")\n\tt.ginkgoPath = filepath.Join(artifacts.RunDir(), \"ginkgo\")\n\textract := map[string]string{\n\t\t\"kubernetes/test/bin/e2e.test\": t.e2eTestPath,\n\t\t\"kubernetes/test/bin/ginkgo\": t.ginkgoPath,\n\t}\n\textracted := map[string]bool{}\n\n\tfor {\n\t\tif len(extracted) == len(extract) {\n\t\t\tbreak\n\t\t}\n\n\t\theader, err := tarReader.Next()\n\t\tif err == io.EOF {\n\t\t\tbreak\n\t\t}\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error during tar read: %s\", err)\n\t\t}\n\n\t\tif dest := extract[header.Name]; dest != \"\" {\n\t\t\toutFile, err := os.Create(dest)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"error creating file at %s: %s\", dest, err)\n\t\t\t}\n\t\t\tdefer outFile.Close()\n\n\t\t\tif err := outFile.Chmod(0700); err != nil {\n\t\t\t\treturn fmt.Errorf(\"failed to make %s executable: %s\", dest, err)\n\t\t\t}\n\n\t\t\tif _, err := io.Copy(outFile, tarReader); err != nil {\n\t\t\t\treturn fmt.Errorf(\"error reading data from tar with header name %s: %s\", header.Name, err)\n\t\t\t}\n\n\t\t\textracted[header.Name] = true\n\t\t}\n\t}\n\tfor k := range extract {\n\t\tif !extracted[k] {\n\t\t\treturn fmt.Errorf(\"failed to find %s in %s\", k, downloadPath)\n\t\t}\n\t}\n\treturn nil\n}\n\n// ensureKubectl checks if the kubectl exists and verifies the hashes\n// else downloads it from GCS\nfunc (t *Tester) ensureKubectl(downloadPath string) error {\n\n\tkubectlPathInGCS := fmt.Sprintf(\n\t\t\"gs://%s/%s/%s/bin/%s/%s/kubectl\",\n\t\tt.TestPackageBucket,\n\t\tt.TestPackageDir,\n\t\tt.TestPackageVersion,\n\t\truntime.GOOS,\n\t\truntime.GOARCH,\n\t)\n\tif _, err := os.Stat(downloadPath); err == nil {\n\t\tslog.Info(\"found existing kubectl\", \"path\", downloadPath)\n\t\terr := t.compareSHA(downloadPath, kubectlPathInGCS)\n\t\tif err == nil {\n\t\t\tslog.Info(\"validated hash for existing kubectl\", \"path\", downloadPath)\n\t\t\treturn nil\n\t\t}\n\t\tslog.Warn(\"hash validation failed\", \"error\", err)\n\t}\n\n\tcmd := exec.Command(\"gsutil\", \"cp\", kubectlPathInGCS, downloadPath)\n\texec.InheritOutput(cmd)\n\tif err := cmd.Run(); err != nil {\n\t\treturn fmt.Errorf(\"failed to download kubectl for release %s: %s\", t.TestPackageVersion, err)\n\t}\n\tif err := os.Chmod(downloadPath, 0700); err != nil {\n\t\treturn fmt.Errorf(\"failed to make %s executable: %s\", downloadPath, err)\n\t}\n\treturn nil\n}\n\n// ensureReleaseTar checks if the kubernetes test tarball already exists\n// and verifies the hashes\n// else downloads it from GCS\nfunc (t *Tester) ensureReleaseTar(downloadPath, releaseTar string) error {\n\n\treleaseTarPathInGCS := fmt.Sprintf(\n\t\t\"gs://%s/%s/%s/%s\",\n\t\tt.TestPackageBucket,\n\t\tt.TestPackageDir,\n\t\tt.TestPackageVersion,\n\t\treleaseTar,\n\t)\n\n\tif _, err := os.Stat(downloadPath); err == nil {\n\t\tslog.Info(\"found existing tar\", \"path\", downloadPath)\n\t\terr := t.compareSHA(downloadPath, releaseTarPathInGCS)\n\t\tif err == nil {\n\t\t\tslog.Info(\"validated hash for existing tar\", \"path\", downloadPath)\n\t\t\treturn nil\n\t\t}\n\t\tslog.Warn(\"hash validation failed\", \"error\", err)\n\t}\n\n\tcmd := exec.Command(\"gsutil\", \"cp\",\n\t\treleaseTarPathInGCS,\n\t\tdownloadPath,\n\t)\n\texec.InheritOutput(cmd)\n\tif err := cmd.Run(); err != nil {\n\t\treturn fmt.Errorf(\"failed to download release tar %s for release %s: %s\", releaseTar, t.TestPackageVersion, err)\n\t}\n\treturn nil\n}\n\nfunc (t *Tester) compareSHA(downloadPath string, gcsFilePath string) error {\n\tcmd := exec.Command(\"gsutil\", \"cat\",\n\t\tfmt.Sprintf(\"%s.sha256\", gcsFilePath),\n\t)\n\texpectedSHABytes, err := exec.Output(cmd)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to get sha256 for file %s for release %s: %s\", gcsFilePath, t.TestPackageVersion, err)\n\t}\n\texpectedSHA := strings.TrimSuffix(string(expectedSHABytes), \"\\n\")\n\tactualSHA, err := sha256sum(downloadPath)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to compute sha256 for %q: %v\", downloadPath, err)\n\t}\n\tif actualSHA != expectedSHA {\n\t\treturn fmt.Errorf(\"sha256 does not match\")\n\t}\n\treturn nil\n}\n\nfunc sha256sum(path string) (string, error) {\n\tf, err := os.Open(path)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer f.Close()\n\n\th := sha256.New()\n\tif _, err := io.Copy(h, f); err != nil {\n\t\treturn \"\", err\n\t}\n\treturn hex.EncodeToString(h.Sum(nil)), nil\n}\n" }, { "path": "internal/testers/multi/cmd.go", "content": "package multi\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"log/slog\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-k8s-tester/internal\"\n\t\"github.com/urfave/sflags/gen/gpflag\"\n\t\"sigs.k8s.io/kubetest2/pkg/app/shim\"\n\t\"sigs.k8s.io/kubetest2/pkg/artifacts\"\n\t\"sigs.k8s.io/kubetest2/pkg/process\"\n\t\"sigs.k8s.io/kubetest2/pkg/testers\"\n)\n\nconst TesterName = \"multi\"\n\nconst usage = `kubetest2 --test=multi -- [MultiTesterDriverArgs] -- [TesterName] [TesterArgs] -- ...\n\n MultiTesterDriverArgs: arguments passed to the multi-tester driver\n\n TesterName: the name of the tester to run\n TesterArgs: arguments passed to tester\n\n Each tester clause is separated by \"--\".\n`\n\nfunc Main() {\n\tif err := execute(); err != nil {\n\t\tslog.Error(\"failed to run multi tester\", \"error\", err)\n\t\tos.Exit(1)\n\t}\n}\n\ntype multiTesterDriver struct {\n\targv []string\n}\n\ntype tester struct {\n\tname string\n\tpath string\n\targs []string\n}\n\nfunc execute() error {\n\tdriverArgs, testerClauses := splitArguments(os.Args)\n\tdriver := multiTesterDriver{\n\t\targv: driverArgs,\n\t}\n\tfs, err := gpflag.Parse(&driver)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to initialize tester: %v\", err)\n\t}\n\n\tfs.Usage = func() {\n\t\tfmt.Print(usage)\n\t}\n\n\tif len(testerClauses) == 0 {\n\t\tfs.Usage()\n\t\treturn nil\n\t}\n\n\t// gracefully handle -h or --help if it is the only argument\n\thelp := fs.BoolP(\"help\", \"h\", false, \"\")\n\n\tfailFast := fs.Bool(\"fail-fast\", false, \"Exit immediately if any tester fails\")\n\n\t// we don't care about errors, only if -h / --help was set\n\terr = fs.Parse(driver.argv)\n\tif err != nil {\n\t\tfs.Usage()\n\t\treturn err\n\t}\n\n\tif *help {\n\t\tfs.Usage()\n\t\treturn nil\n\t}\n\n\tif err := testers.WriteVersionToMetadata(internal.Version, \"\"); err != nil {\n\t\treturn err\n\t}\n\n\tif testers, err := prepareTesters(testerClauses); err != nil {\n\t\treturn err\n\t} else {\n\t\treturn test(testers, *failFast)\n\t}\n}\n\nfunc test(testers []tester, failFast bool) error {\n\tmetadataPath := filepath.Join(artifacts.BaseDir(), \"metadata.json\")\n\tbackupMetdataPath := metadataPath + \".bak\"\n\tif err := os.Rename(metadataPath, backupMetdataPath); err != nil {\n\t\tslog.Error(\"failed to backup driver metadata\", \"error\", err)\n\t}\n\tvar testerErrs []error\n\tfor _, tester := range testers {\n\t\tif err := tester.run(); err != nil {\n\t\t\tslog.Error(\"tester failed\", \"tester\", tester, \"error\", err)\n\t\t\ttesterErrs = append(testerErrs, fmt.Errorf(\"%+v: %v\", tester, err))\n\t\t\tif failFast {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\t// reset the metadata.json file\n\t\t// testers will try to set the tester-version key and cause conflicts\n\t\tif err := os.Remove(metadataPath); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete tester metadata: %v\", err)\n\t\t}\n\t}\n\tif err := os.Rename(backupMetdataPath, metadataPath); err != nil {\n\t\treturn fmt.Errorf(\"failed to restore driver metadata: %v\", err)\n\t}\n\tif len(testerErrs) > 0 {\n\t\treturn errors.Join(testerErrs...)\n\t}\n\treturn nil\n}\n\n// splitArguments splits arguments into driver arguments and tester clauses, separated by \"--\".\nfunc splitArguments(argv []string) ([]string, [][]string) {\n\tvar clauses [][]string\n\tvar last int\n\tfor i, arg := range argv {\n\t\tif arg == \"--\" {\n\t\t\tclauses = append(clauses, argv[last:i])\n\t\t\tlast = i + 1\n\t\t}\n\t}\n\tclauses = append(clauses, argv[last:])\n\treturn clauses[0], clauses[1:]\n}\n\nfunc prepareTesters(testerClauses [][]string) ([]tester, error) {\n\tvar testers []tester\n\tfor _, clause := range testerClauses {\n\t\ttesterName := clause[0]\n\t\tif testerName == TesterName {\n\t\t\treturn nil, fmt.Errorf(\"nesting isn't possible with the %s tester\", TesterName)\n\t\t}\n\t\tpath, err := shim.FindTester(testerName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ttester := tester{\n\t\t\tname: testerName,\n\t\t\tpath: path,\n\t\t\targs: expandEnv(clause[1:]),\n\t\t}\n\t\ttesters = append(testers, tester)\n\t}\n\treturn testers, nil\n}\n\nfunc expandEnv(args []string) []string {\n\texpandedArgs := make([]string, len(args))\n\tfor i, arg := range args {\n\t\t// best effort handle literal dollar for backward compatibility\n\t\t// this is not an all-purpose shell special character handler\n\t\tif strings.Contains(arg, `\\$`) {\n\t\t\texpandedArgs[i] = strings.ReplaceAll(arg, `\\$`, `$`)\n\t\t} else {\n\t\t\texpandedArgs[i] = os.ExpandEnv(arg)\n\t\t}\n\t}\n\treturn expandedArgs\n}\n\nfunc (t *tester) run() error {\n\tslog.Info(\"running tester\", \"tester\", t)\n\treturn process.ExecJUnit(t.path, t.args, os.Environ())\n}\n" }, { "path": "internal/util/cloudformation.go", "content": "package util\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/service/cloudformation\"\n\ttypes \"github.com/aws/aws-sdk-go-v2/service/cloudformation/types\"\n)\n\n// TODO: implement AWS client wrappers, and incorporate this into the cfn:CreateStack call\nfunc WrapCFNStackFailure(ctx context.Context, cfnClient *cloudformation.Client, createStackErr error, stackName string) error {\n\tif createStackErr == nil {\n\t\treturn nil\n\t}\n\tresourceByFailureMode := make(map[string][]string)\n\teventsPaginator := cloudformation.NewDescribeStackEventsPaginator(cfnClient, &cloudformation.DescribeStackEventsInput{\n\t\tStackName: &stackName,\n\t})\n\tfor eventsPaginator.HasMorePages() {\n\t\tpage, err := eventsPaginator.NextPage(ctx)\n\t\tif err != nil {\n\t\t\treturn createStackErr\n\t\t}\n\t\tfor _, event := range page.StackEvents {\n\t\t\tif event.ResourceStatus == types.ResourceStatusCreateFailed {\n\t\t\t\tif _, ok := resourceByFailureMode[aws.ToString(event.ResourceStatusReason)]; !ok {\n\t\t\t\t\tresourceByFailureMode[aws.ToString(event.ResourceStatusReason)] = []string{}\n\t\t\t\t}\n\t\t\t\tresourceByFailureMode[aws.ToString(event.ResourceStatusReason)] = append(resourceByFailureMode[aws.ToString(event.ResourceStatusReason)], aws.ToString(event.LogicalResourceId))\n\t\t\t}\n\t\t}\n\t}\n\tnonCancellationFailure := len(resourceByFailureMode) > 1\n\tvar enhancedDetails []string\n\tfor reason, resources := range resourceByFailureMode {\n\t\tif nonCancellationFailure && reason == \"Resource creation cancelled\" {\n\t\t\t// Ignore resource cancellation errors if there's another failure reported, those failures\n\t\t\t// would just be a consequence of that failure. If all the failures are resource cancellation,\n\t\t\t// then there was likely a user initiated delete of the whole stack based on a timeout\n\t\t\t// waiting for one of the resources to create\n\t\t\tcontinue\n\t\t}\n\t\tenhancedDetails = append(enhancedDetails, fmt.Sprintf(\"%s: %s\", strings.Join(resources, \",\"), reason))\n\t}\n\treturn fmt.Errorf(\"%w: %s\", createStackErr, strings.Join(enhancedDetails, \"--\"))\n}\n" }, { "path": "internal/util/exec.go", "content": "package util\n\nimport (\n\t\"os\"\n\t\"os/exec\"\n)\n\nfunc ExecuteCommand(name string, args ...string) error {\n\tcommand := exec.Command(name, args...)\n\tcommand.Stdout = os.Stdout\n\tcommand.Stderr = os.Stderr\n\treturn command.Run()\n}\n" }, { "path": "internal/util/http.go", "content": "package util\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/aws/smithy-go/middleware\"\n\tsmithyhttp \"github.com/aws/smithy-go/transport/http\"\n)\n\nconst httpHeaderBoundary = \": \"\n\n// NewHTTPHeaderAPIOptions returns a slice of middleware options that adds the\n// specified HTTP headers to an API request.\n// Each header should be of the format `Header-Key: Header-Value`, in the same manner\n// as headers are passed with `curl`-s `-H` flag.\nfunc NewHTTPHeaderAPIOptions(headers []string) ([]func(*middleware.Stack) error, error) {\n\tvar opts []func(*middleware.Stack) error\n\tfor _, header := range headers {\n\t\tboundary := strings.Index(header, httpHeaderBoundary)\n\t\tif boundary == -1 {\n\t\t\treturn nil, fmt.Errorf(\"malformed HTTP header: '%s'\", header)\n\t\t}\n\t\tkey := header[:boundary]\n\t\tval := header[boundary+len(httpHeaderBoundary):]\n\t\topts = append(opts, smithyhttp.AddHeaderValue(key, val))\n\t}\n\treturn opts, nil\n}\n" }, { "path": "internal/util/http_test.go", "content": "package util\n\nimport (\n\t\"testing\"\n)\n\nfunc Test_NewHTTPHeaderAPIOptions(t *testing.T) {\n\ttestCases := []struct {\n\t\tname string\n\t\theaders []string\n\t\texpectError bool\n\t}{\n\t\t{\n\t\t\tname: \"empty\",\n\t\t\theaders: []string{},\n\t\t},\n\t\t{\n\t\t\tname: \"single valid header\",\n\t\t\theaders: []string{\"Content-Type: application/json\"},\n\t\t},\n\t\t{\n\t\t\tname: \"multiple valid headers\",\n\t\t\theaders: []string{\"Content-Type: application/json\", \"Accept: application/json\"},\n\t\t},\n\t\t{\n\t\t\tname: \"invalid header\",\n\t\t\theaders: []string{\"Invalid header\"},\n\t\t\texpectError: true,\n\t\t},\n\t}\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\t_, err := NewHTTPHeaderAPIOptions(tc.headers)\n\t\t\tif err != nil && !tc.expectError {\n\t\t\t\tt.Errorf(\"unexpected error: %v\", err)\n\t\t\t}\n\t\t\tif err == nil && tc.expectError {\n\t\t\t\tt.Error(\"expected error but got none\")\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "internal/util/lang.go", "content": "package util\n\nfunc Must[T any](t T, err error) T {\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn t\n}\n" }, { "path": "internal/util/path.go", "content": "package util\n\nimport (\n\t\"errors\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"syscall\"\n)\n\nvar ErrFileNotFoundInPath = errors.New(\"file not found in $PATH\")\n\n// LookPath finds a file on the PATH.\n// It uses a similar process to exec.LookPath, but can find regular files.\nfunc LookPath(file string) (string, error) {\n\tpath := os.Getenv(\"PATH\")\n\tfor _, dir := range filepath.SplitList(path) {\n\t\tif dir == \"\" {\n\t\t\t// Unix shell semantics: path element \"\" means \".\"\n\t\t\tdir = \".\"\n\t\t}\n\t\tpath := filepath.Join(dir, file)\n\t\tif err := checkFile(path); err == nil {\n\t\t\treturn path, nil\n\t\t}\n\t}\n\treturn \"\", ErrFileNotFoundInPath\n}\n\nfunc checkFile(file string) error {\n\td, err := os.Stat(file)\n\tif err != nil {\n\t\treturn err\n\t}\n\tm := d.Mode()\n\tif m.IsDir() {\n\t\treturn syscall.EISDIR\n\t}\n\treturn nil\n}\n" }, { "path": "internal/util/version.go", "content": "package util\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n)\n\nconst KubernetesVersionFile = \"kubernetes-version.txt\"\n\nfunc DetectKubernetesVersion() (string, error) {\n\tversionFile, err := LookPath(KubernetesVersionFile)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tbytes, err := os.ReadFile(versionFile)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\t// \"v1.2.3\"\n\tversionTag := string(bytes)\n\treturn strings.ReplaceAll(versionTag, \"v\", \"\"), nil\n}\n\nfunc ParseMinorVersion(semanticVersion string) (string, error) {\n\tparts := strings.Split(semanticVersion, \".\")\n\tif len(parts) < 2 {\n\t\treturn \"\", fmt.Errorf(\"malformed semantic version: '%s'\", semanticVersion)\n\t}\n\treturn strings.Join(parts[:2], \".\"), nil\n}\n" }, { "path": "internal/version.go", "content": "package internal\n\nvar Version string\n" }, { "path": "test/cases/disruptive/graceful_reboot_test.go", "content": "//go:build e2e\n\npackage disruptive\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/awssdk\"\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\n\t\"github.com/aws/aws-sdk-go-v2/service/ec2\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nfunc getSleepPodTemplate(name string) corev1.Pod {\n\treturn corev1.Pod{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: name,\n\t\t\tNamespace: \"default\",\n\t\t},\n\t\tSpec: corev1.PodSpec{\n\t\t\tContainers: []corev1.Container{\n\t\t\t\t{\n\t\t\t\t\tName: name,\n\t\t\t\t\tImage: \"public.ecr.aws/amazonlinux/amazonlinux:2023\",\n\t\t\t\t\tCommand: []string{\"sleep\", \"infinity\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tRestartPolicy: corev1.RestartPolicyNever,\n\t\t},\n\t}\n}\n\nfunc TestGracefulReboot(t *testing.T) {\n\tterminationCanaryPodName := fmt.Sprintf(\"termination-canary-%d\", time.Now().Unix())\n\tcanaryPod := getSleepPodTemplate(terminationCanaryPodName)\n\tbootIndicatorPodName := fmt.Sprintf(\"boot-detection-%d\", time.Now().Unix())\n\tbootIndicatorPod := getSleepPodTemplate(bootIndicatorPodName)\n\n\tfeat := features.New(\"graceful-reboot\").\n\t\tWithLabel(\"suite\", \"disruptive\").\n\t\tAssess(\"Node gracefully reboots\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\t// Create an initial pod to allow the default scheduler to do the work of identifying a healthy node.\n\t\t\t// Starting with a healthy node is essential to the test, as the only expectation is for the node to\n\t\t\t// return to its same initial state after the reboot.\n\t\t\tif err := cfg.Client().Resources().Create(ctx, &canaryPod); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to create heartbeat pod: %v\", err)\n\t\t\t}\n\n\t\t\tif err := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).PodRunning(&canaryPod),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(5*time.Minute),\n\t\t\t); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to wait for pod %s to go into running status: %v\", terminationCanaryPodName, err)\n\t\t\t}\n\n\t\t\tvar targetNode corev1.Node\n\t\t\tif err := cfg.Client().Resources().Get(ctx, canaryPod.Spec.NodeName, \"\", &targetNode); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to get node %s: %v\", canaryPod.Spec.NodeName, err)\n\t\t\t}\n\n\t\t\tt.Logf(\"Pod %s is running on node %s\", terminationCanaryPodName, targetNode.Name)\n\n\t\t\t// Do an initial check of the /healthz endpoint reachability to ensure we can rely on it later.\n\t\t\t// This might fail even if the node is healthy if, for example, the node's security group rules\n\t\t\t// do not allow ingress traffic from the control plane.\n\t\t\t// Retry for up to 1 minute to handle transient TLS errors during cert rotation.\n\t\t\tvar kubeletResponsive bool\n\t\t\tvar err error\n\t\t\thealthCheckCtx, healthCheckCancel := context.WithTimeout(ctx, 5*time.Minute)\n\t\t\tdefer healthCheckCancel()\n\t\t\tfor {\n\t\t\t\tkubeletResponsive, err = fwext.KubeletIsResponsive(healthCheckCtx, cfg.Client().RESTConfig(), targetNode.Name)\n\t\t\t\tif err == nil && kubeletResponsive {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tselect {\n\t\t\t\tcase <-healthCheckCtx.Done():\n\t\t\t\t\tt.Fatalf(\"Node %s is not responding to initial /healthz checks: %v\", targetNode.Name, err)\n\t\t\t\tcase <-time.After(5 * time.Second):\n\t\t\t\t\tt.Logf(\"Retrying /healthz check for node %s (last error: %v, responsive: %v)\", targetNode.Name, err, kubeletResponsive)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tproviderIDParts := strings.Split(targetNode.Spec.ProviderID, \"/\")\n\t\t\tinstanceID := providerIDParts[len(providerIDParts)-1]\n\t\t\tt.Logf(\"Rebooting underlying instance %s for node %s...\", instanceID, targetNode.Name)\n\n\t\t\tec2Client := ec2.NewFromConfig(awssdk.NewConfig())\n\t\t\tif _, err := ec2Client.RebootInstances(ctx, &ec2.RebootInstancesInput{\n\t\t\t\tInstanceIds: []string{instanceID},\n\t\t\t}); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to reboot instance %s: %v\", instanceID, err)\n\t\t\t}\n\n\t\t\tt.Logf(\"Successfully triggered reboot of instance %s, waiting for kubelet to become unresponsive...\", instanceID)\n\n\t\t\tkubeletShutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)\n\t\t\tdefer cancel()\n\n\t\t\t// Use kubelet health probes as the signal for instance shutdown. Since the health endpoint\n\t\t\t// could previously be reached, a refused connection implies kubelet was killed.\n\t\t\tfor kubeletResponsive {\n\t\t\t\tselect {\n\t\t\t\tcase <-kubeletShutdownCtx.Done():\n\t\t\t\t\tt.Fatalf(\"Failed to wait for kubelet to become unresponsive: %v\", ctx.Err())\n\t\t\t\tcase <-time.Tick(1 * time.Second):\n\t\t\t\t\tif kubeletResponsive, err = fwext.KubeletIsResponsive(ctx, cfg.Client().RESTConfig(), targetNode.Name); err != nil {\n\t\t\t\t\t\tt.Fatalf(\"Unpexected error while monitoring kubelet on node %s: %v\", targetNode.Name, err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tt.Logf(\"Node %s has become unresponsive, waiting for the node to become schedulable again...\", targetNode.Name)\n\n\t\t\t// Create a second pod, we will rely on this pod starting to run as an indication of a healthy state.\n\t\t\t// Since kubelet was killed at this point, we know the reboot must complete and kubelet must start\n\t\t\t// again for this pod to start running.\n\t\t\tbootIndicatorPod.Spec.NodeSelector = map[string]string{\n\t\t\t\t\"kubernetes.io/hostname\": targetNode.Name,\n\t\t\t}\n\t\t\tif err := cfg.Client().Resources().Create(ctx, &bootIndicatorPod); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to create boot indicator pod: %v\", err)\n\t\t\t}\n\n\t\t\tif err := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).PodRunning(&bootIndicatorPod),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(10*time.Minute), // TODO: bring down this value after collecting some more data\n\t\t\t); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to wait for pod to go into running status %s: %v\", bootIndicatorPodName, err)\n\t\t\t}\n\n\t\t\tt.Logf(\"Node %s became ready and schedulable within %v!\", targetNode.Name, time.Since(bootIndicatorPod.CreationTimestamp.Time))\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tif err := cfg.Client().Resources().Delete(ctx, &canaryPod); err != nil {\n\t\t\t\tt.Logf(\"Failed to delete pod %s: %v\", terminationCanaryPodName, err)\n\t\t\t}\n\n\t\t\tif err := cfg.Client().Resources().Delete(ctx, &bootIndicatorPod); err != nil {\n\t\t\t\tt.Logf(\"Failed to delete pod %s: %v\", bootIndicatorPodName, err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, feat)\n}\n" }, { "path": "test/cases/disruptive/graceful_shutdown_test.go", "content": "//go:build e2e\n\npackage disruptive\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"regexp\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/awssdk\"\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-sdk-go-v2/service/ec2\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/utils/pointer\"\n\n\t\"sigs.k8s.io/e2e-framework/klient/k8s\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\n// getPodLogs retrieves logs from a pod using kubernetes clientset\nfunc getPodLogs(ctx context.Context, cfg *envconf.Config, podName, namespace string) (string, error) {\n\tclient, err := kubernetes.NewForConfig(cfg.Client().RESTConfig())\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treq := client.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{})\n\tlogs, err := req.Stream(ctx)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer logs.Close()\n\n\tvar result strings.Builder\n\t_, err = io.Copy(&result, logs)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn result.String(), nil\n}\n\n// checkLogPattern checks if a log pattern exists in the pod logs\nfunc checkLogPattern(ctx context.Context, cfg *envconf.Config, podName, namespace, pattern string) (bool, error) {\n\tlogs, err := getPodLogs(ctx, cfg, podName, namespace)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tmatched, err := regexp.MatchString(pattern, logs)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn matched, nil\n}\n\n// countLogMatches counts how many times a pattern appears in the logs\nfunc countLogMatches(ctx context.Context, cfg *envconf.Config, podName, namespace, pattern string) (int, error) {\n\tlogs, err := getPodLogs(ctx, cfg, podName, namespace)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tre, err := regexp.Compile(pattern)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tmatches := re.FindAllString(logs, -1)\n\treturn len(matches), nil\n}\n\nfunc TestKubeletGracefulShutdown(t *testing.T) {\n\tfeat := features.New(\"kubelet-graceful-shutdown\").\n\t\tWithLabel(\"suite\", \"disruptive\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog.Println(\"[Setup] Setting up Kubelet Graceful Shutdown test...\")\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Kubelet gracefully shuts down pods during node termination\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\t// Create heartbeat pod that will log its status\n\t\t\tpod := &corev1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tName: fmt.Sprintf(\"graceful-shutdown-test-%d\", time.Now().Unix()),\n\t\t\t\t\tNamespace: \"default\",\n\t\t\t\t\tLabels: map[string]string{\n\t\t\t\t\t\t\"app\": \"graceful-shutdown-test\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tSpec: corev1.PodSpec{\n\t\t\t\t\tContainers: []corev1.Container{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName: \"heartbeat-container\",\n\t\t\t\t\t\t\tImage: \"public.ecr.aws/amazonlinux/amazonlinux:2023\",\n\t\t\t\t\t\t\tCommand: []string{\"/usr/bin/bash\", \"-c\"},\n\t\t\t\t\t\t\tArgs: []string{`\n\t\t\t\t\t\t\t\tset -x\n\t\t\t\t\t\t\t\techo \"[GRACEFUL-TEST] Starting graceful shutdown test pod...\"\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\tfunction handle_sigterm() {\n\t\t\t\t\t\t\t\t\techo \"[GRACEFUL-TEST] $(date): SIGTERM-RECEIVED - starting graceful shutdown period\"\n\t\t\t\t\t\t\t\t\t# Continue heartbeating until we are SIGKILL-d\n\t\t\t\t\t\t\t\t\tstart_time=$(date +%s)\n\t\t\t\t\t\t\t\t\twhile true; do\n\t\t\t\t\t\t\t\t\t\tcurrent_time=$(date +%s)\n\t\t\t\t\t\t\t\t\t\telapsed=$((current_time - start_time))\n\t\t\t\t\t\t\t\t\t\techo \"[GRACEFUL-TEST] $(date): HEARTBEAT-AFTER-SIGTERM elapsed=${elapsed}s\"\n\t\t\t\t\t\t\t\t\t\tsleep 1\n\t\t\t\t\t\t\t\t\tdone\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\ttrap handle_sigterm TERM\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t# Initial heartbeat to show pod is running\n\t\t\t\t\t\t\t\techo \"[GRACEFUL-TEST] $(date): POD-STARTED - pod started successfully\"\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t# Keep running and heartbeating until terminated\n\t\t\t\t\t\t\t\tcounter=0\n\t\t\t\t\t\t\t\twhile true; do\n\t\t\t\t\t\t\t\t\techo \"[GRACEFUL-TEST] $(date): NORMAL-HEARTBEAT counter=$counter\"\n\t\t\t\t\t\t\t\t\tcounter=$((counter + 1))\n\t\t\t\t\t\t\t\t\tsleep 10\n\t\t\t\t\t\t\t\tdone\n\t\t\t\t\t\t\t`},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tRestartPolicy: corev1.RestartPolicyNever,\n\t\t\t\t\tTerminationGracePeriodSeconds: pointer.Int64(150), // 2.5 minutes to allow for graceful shutdown testing\n\t\t\t\t},\n\t\t\t}\n\n\t\t\tif err := cfg.Client().Resources().Create(ctx, pod); err != nil {\n\t\t\t\tt.Fatalf(\"[Assess] Failed to create heartbeat pod: %v\", err)\n\t\t\t}\n\t\t\tlog.Printf(\"[Assess] Created heartbeat pod: %s\", pod.Name)\n\n\t\t\t// Store pod name in context for cleanup\n\t\t\tctx = context.WithValue(ctx, \"podName\", pod.Name)\n\n\t\t\tlog.Printf(\"[Assess] Waiting for pod %s to start running...\", pod.Name)\n\t\t\terr := wait.For(\n\t\t\t\te2e.NewConditionExtension(cfg.Client().Resources()).ResourceMatch(pod, func(object k8s.Object) bool {\n\t\t\t\t\tpod := object.(*corev1.Pod)\n\t\t\t\t\treturn pod.Status.Phase == corev1.PodRunning\n\t\t\t\t}),\n\t\t\t\twait.WithTimeout(2*time.Minute),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"[Assess] Pod did not start running: %v\", err)\n\t\t\t}\n\n\t\t\t// Wait a bit for initial heartbeats\n\t\t\tlog.Printf(\"[Assess] Waiting for initial heartbeats...\")\n\t\t\ttime.Sleep(30 * time.Second)\n\n\t\t\t// Verify pod started successfully by checking logs\n\t\t\tpodStarted, err := checkLogPattern(ctx, cfg, pod.Name, pod.Namespace, `POD-STARTED`)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"[Assess] Failed to check pod logs: %v\", err)\n\t\t\t}\n\t\t\tif !podStarted {\n\t\t\t\tt.Fatalf(\"[Assess] Pod did not log successful startup\")\n\t\t\t}\n\t\t\tlog.Printf(\"[Assess] ✓ Pod startup confirmed via logs\")\n\n\t\t\t// Get the node the pod is running on\n\t\t\tif err := cfg.Client().Resources().Get(ctx, pod.Name, pod.Namespace, pod); err != nil {\n\t\t\t\tt.Fatalf(\"[Assess] Failed to get pod details: %v\", err)\n\t\t\t}\n\n\t\t\tnodeName := pod.Spec.NodeName\n\t\t\tif nodeName == \"\" {\n\t\t\t\tt.Fatalf(\"[Assess] Pod is not scheduled to any node\")\n\t\t\t}\n\t\t\tlog.Printf(\"[Assess] Pod is running on node: %s\", nodeName)\n\n\t\t\t// Get the EC2 instance ID for this node\n\t\t\tvar node corev1.Node\n\t\t\tif err := cfg.Client().Resources().Get(ctx, nodeName, \"\", &node); err != nil {\n\t\t\t\tt.Fatalf(\"[Assess] Failed to get node %s: %v\", nodeName, err)\n\t\t\t}\n\t\t\tproviderID := node.Spec.ProviderID\n\t\t\tif providerID == \"\" {\n\t\t\t\tt.Fatalf(\"[Assess] Node %s has no providerID\", nodeName)\n\t\t\t}\n\t\t\tparts := strings.Split(providerID, \"/\")\n\t\t\tif len(parts) < 2 {\n\t\t\t\tt.Fatalf(\"[Assess] Invalid providerID format: %s\", providerID)\n\t\t\t}\n\t\t\tinstanceID := parts[len(parts)-1]\n\t\t\tlog.Printf(\"[Assess] Node %s corresponds to EC2 instance: %s\", nodeName, instanceID)\n\n\t\t\t// Terminate the EC2 instance\n\t\t\tlog.Printf(\"[Assess] Terminating EC2 instance %s to test graceful shutdown...\", instanceID)\n\t\t\tec2Client := ec2.NewFromConfig(awssdk.NewConfig())\n\t\t\t_, err = ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{\n\t\t\t\tInstanceIds: []string{instanceID},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"[Assess] Failed to terminate EC2 instance %s: %v\", instanceID, err)\n\t\t\t}\n\t\t\tlog.Printf(\"[Assess] Successfully initiated termination of instance %s\", instanceID)\n\n\t\t\t// Wait and monitor the graceful shutdown process via logs\n\t\t\tlog.Printf(\"[Assess] Monitoring graceful shutdown process for 3 minutes...\")\n\n\t\t\t// Wait for SIGTERM to be received (should happen within 60 seconds)\n\t\t\tsigtermReceived := false\n\t\t\tfor i := 0; i < 30; i++ {\n\t\t\t\treceived, err := checkLogPattern(ctx, cfg, pod.Name, pod.Namespace, `SIGTERM-RECEIVED`)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Printf(\"[Assess] Warning: Failed to check logs: %v\", err)\n\t\t\t\t} else if received {\n\t\t\t\t\tsigtermReceived = true\n\t\t\t\t\tlog.Printf(\"[Assess] ✓ SIGTERM received by pod (detected after %d seconds)\", i*2)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\ttime.Sleep(2 * time.Second)\n\t\t\t}\n\n\t\t\tif !sigtermReceived {\n\t\t\t\tt.Fatalf(\"[Assess] Pod did not receive SIGTERM within 60 seconds of instance termination\")\n\t\t\t}\n\n\t\t\t// Monitor heartbeats for the next 2+ minutes to verify graceful shutdown behavior\n\t\t\tlog.Printf(\"[Assess] Verifying pod continues running during graceful shutdown period...\")\n\t\t\tgracefulShutdownStart := time.Now()\n\n\t\t\tvar heartbeatsAfterSigterm int\n\t\t\tfor time.Since(gracefulShutdownStart) < 2*time.Minute { // Monitor for 2 minutes\n\t\t\t\t// Count heartbeats after SIGTERM\n\t\t\t\tmatches, err := countLogMatches(ctx, cfg, pod.Name, pod.Namespace, `HEARTBEAT-AFTER-SIGTERM`)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Printf(\"[Assess] Warning: Failed to count heartbeats: %v\", err)\n\t\t\t\t} else if matches > 0 {\n\t\t\t\t\tlog.Printf(\"[Assess] ✓ Pod still running after SIGTERM (%d heartbeats logged)\", matches)\n\t\t\t\t\theartbeatsAfterSigterm = matches\n\t\t\t\t}\n\n\t\t\t\ttime.Sleep(1 * time.Second)\n\t\t\t}\n\n\t\t\t// Verify we got heartbeats during the graceful shutdown period\n\t\t\t// These happen once a second, so we should observe at least 110 of them for a 2 minute grace period\n\t\t\tif heartbeatsAfterSigterm < 110 {\n\t\t\t\tt.Fatalf(\"[Assess] Expected at least 110 heartbeats during graceful shutdown, got %d\", heartbeatsAfterSigterm)\n\t\t\t}\n\n\t\t\tlog.Printf(\"[Assess] ✓ Pod continued running and heartbeating for graceful shutdown period\")\n\t\t\tlog.Printf(\"[Assess] ✓ Total heartbeats after SIGTERM: %d\", heartbeatsAfterSigterm)\n\n\t\t\t// Check for graceful exit\n\t\t\tgracefulExit, err := checkLogPattern(ctx, cfg, pod.Name, pod.Namespace, `GRACEFUL-EXIT`)\n\t\t\tif err != nil {\n\t\t\t\tlog.Printf(\"[Assess] Warning: Failed to check for graceful exit: %v\", err)\n\t\t\t} else if gracefulExit {\n\t\t\t\tlog.Printf(\"[Assess] ✓ Pod logged graceful exit\")\n\t\t\t}\n\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tpodName, ok := ctx.Value(\"podName\").(string)\n\t\t\tif !ok {\n\t\t\t\tlog.Printf(\"[Teardown] No pod name in context, nothing to clean up\")\n\t\t\t\treturn ctx\n\t\t\t}\n\n\t\t\tlog.Printf(\"[Teardown] Cleaning up test pod %s...\", podName)\n\n\t\t\t// Get final logs for debugging if needed\n\t\t\tlogs, err := getPodLogs(ctx, cfg, podName, \"default\")\n\t\t\tif err != nil {\n\t\t\t\tlog.Printf(\"[Teardown] Warning: Failed to get final logs: %v\", err)\n\t\t\t} else {\n\t\t\t\tlog.Printf(\"[Teardown] Final pod logs:\\n%s\", logs)\n\t\t\t}\n\n\t\t\t// Delete the pod (it may already be terminated)\n\t\t\tpod := &corev1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tName: podName,\n\t\t\t\t\tNamespace: \"default\",\n\t\t\t\t},\n\t\t\t}\n\t\t\tif err := cfg.Client().Resources().Delete(ctx, pod); err != nil {\n\t\t\t\tlog.Printf(\"[Teardown] Warning: Failed to delete pod %s: %v\", podName, err)\n\t\t\t} else {\n\t\t\t\tlog.Printf(\"[Teardown] Successfully cleaned up pod %s\", podName)\n\t\t\t}\n\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, feat)\n}\n" }, { "path": "test/cases/disruptive/main_test.go", "content": "//go:build e2e\n\npackage disruptive\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"testing\"\n\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nvar (\n\ttestenv env.Environment\n)\n\nfunc TestMain(m *testing.M) {\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\ttestenv = env.NewWithConfig(cfg)\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = testenv.WithContext(ctx)\n\n\ttestenv.Setup(func(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\tlog.Println(\"Starting quick test suite...\")\n\t\treturn ctx, nil\n\t})\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/dra/dra_example_driver_test.go", "content": "//go:build e2e\n\npackage dra\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\n\t\"github.com/stretchr/testify/assert\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\trbacv1 \"k8s.io/api/rbac/v1\"\n\t\"k8s.io/api/resource/v1beta1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient/k8s\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\n// see: https://github.com/kubernetes-sigs/dra-example-driver\nfunc TestDraExampleDriver(t *testing.T) {\n\tdraDriverResources := draDriverResources()\n\tdeviceClass, resourceClaimTemplate, pod := testResources()\n\n\texampleDraDriver := features.New(\"dra-example-driver\").\n\t\tWithLabel(\"feature\", \"dra\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tfor _, obj := range draDriverResources {\n\t\t\t\tassert.NoError(t, cfg.Client().Resources().Create(ctx, obj))\n\t\t\t}\n\t\t\tassert.NoError(t, cfg.Client().Resources().Create(ctx, &deviceClass))\n\t\t\tassert.NoError(t, cfg.Client().Resources().Create(ctx, &resourceClaimTemplate))\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"device driver present\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tassert.NoError(t, cfg.Client().Resources().Create(ctx, &pod))\n\t\t\tdefer func() {\n\t\t\t\tassert.NoError(t, cfg.Client().Resources().Delete(ctx, &pod))\n\t\t\t\tassert.NoError(t, wait.For(conditions.New(cfg.Client().Resources()).ResourceDeleted(&pod),\n\t\t\t\t\twait.WithTimeout(time.Minute),\n\t\t\t\t\twait.WithContext(ctx),\n\t\t\t\t))\n\t\t\t}()\n\n\t\t\tassert.NoError(t, wait.For(conditions.New(cfg.Client().Resources()).PodRunning(&pod),\n\t\t\t\twait.WithTimeout(time.Minute),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t))\n\n\t\t\tpodLogs, err := e2e.ReadPodLogs(ctx, cfg.Client().RESTConfig(), pod.Namespace, pod.Name, pod.Spec.Containers[0].Name)\n\t\t\tif assert.NoErrorf(t, err, \"skipping error getting pod logs %q: %v\", pod.Name, err) {\n\t\t\t\tt.Logf(\"Logs for %q\\n%s\", pod.Name, podLogs)\n\t\t\t\tassert.Contains(t, podLogs, fmt.Sprintf(`DRA_RESOURCE_DRIVER_NAME=\"%s\"`, deviceClass.Name))\n\t\t\t}\n\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tassert.NoError(t, cfg.Client().Resources().Delete(ctx, &deviceClass))\n\t\t\tassert.NoError(t, cfg.Client().Resources().Delete(ctx, &resourceClaimTemplate))\n\t\t\tfor _, obj := range draDriverResources {\n\t\t\t\tassert.NoError(t, cfg.Client().Resources().Delete(ctx, obj))\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, exampleDraDriver)\n}\n\nfunc testResources() (v1beta1.DeviceClass, v1beta1.ResourceClaimTemplate, corev1.Pod) {\n\tdeviceClass := v1beta1.DeviceClass{\n\t\tTypeMeta: metav1.TypeMeta{\n\t\t\tAPIVersion: \"resource.k8s.io/v1beta1\",\n\t\t\tKind: \"DeviceClass\",\n\t\t},\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"gpu.example.com\",\n\t\t},\n\t\tSpec: v1beta1.DeviceClassSpec{\n\t\t\tSelectors: []v1beta1.DeviceSelector{\n\t\t\t\t{\n\t\t\t\t\tCEL: &v1beta1.CELDeviceSelector{\n\t\t\t\t\t\tExpression: \"device.driver == 'gpu.example.com'\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tdeviceRequest := v1beta1.DeviceRequest{\n\t\tName: \"gpu\",\n\t\tDeviceClassName: deviceClass.Name,\n\t}\n\n\tresourceClaimTemplate := v1beta1.ResourceClaimTemplate{\n\t\tTypeMeta: metav1.TypeMeta{\n\t\t\tAPIVersion: \"resource.k8s.io/v1beta1\",\n\t\t\tKind: \"ResourceClaimTemplate\",\n\t\t},\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"single-gpu\",\n\t\t\tNamespace: corev1.NamespaceDefault,\n\t\t},\n\t\tSpec: v1beta1.ResourceClaimTemplateSpec{\n\t\t\tSpec: v1beta1.ResourceClaimSpec{\n\t\t\t\tDevices: v1beta1.DeviceClaim{\n\t\t\t\t\tRequests: []v1beta1.DeviceRequest{deviceRequest},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tpod := corev1.Pod{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"pod0\",\n\t\t\tNamespace: corev1.NamespaceDefault,\n\t\t},\n\t\tSpec: corev1.PodSpec{\n\t\t\tContainers: []corev1.Container{\n\t\t\t\t{\n\t\t\t\t\tName: \"ctr0\",\n\t\t\t\t\tImage: \"public.ecr.aws/amazonlinux/amazonlinux:latest\",\n\t\t\t\t\tCommand: []string{\"bash\", \"-c\"},\n\t\t\t\t\tArgs: []string{\"export; trap 'exit 0' TERM; sleep infinity & wait\"},\n\t\t\t\t\tResources: corev1.ResourceRequirements{\n\t\t\t\t\t\tClaims: []corev1.ResourceClaim{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tName: deviceRequest.Name,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tResourceClaims: []corev1.PodResourceClaim{\n\t\t\t\t{\n\t\t\t\t\tName: deviceRequest.Name,\n\t\t\t\t\tResourceClaimTemplateName: &resourceClaimTemplate.Name,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\treturn deviceClass, resourceClaimTemplate, pod\n}\n\nfunc draDriverResources() []k8s.Object {\n\tserviceAccount := corev1.ServiceAccount{\n\t\tTypeMeta: metav1.TypeMeta{\n\t\t\tAPIVersion: \"rbac.authorization.k8s.io/v1\",\n\t\t\tKind: \"ServiceAccount\",\n\t\t},\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"dra-service-account\",\n\t\t\tNamespace: corev1.NamespaceDefault,\n\t\t},\n\t}\n\n\tclusterRole := rbacv1.ClusterRole{\n\t\tTypeMeta: metav1.TypeMeta{\n\t\t\tAPIVersion: \"rbac.authorization.k8s.io/v1\",\n\t\t\tKind: \"ClusterRole\",\n\t\t},\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"dra-example-driver-role\",\n\t\t\tNamespace: corev1.NamespaceDefault,\n\t\t},\n\t\tRules: []rbacv1.PolicyRule{\n\t\t\t{\n\t\t\t\tAPIGroups: []string{\"resource.k8s.io\"},\n\t\t\t\tResources: []string{\"resourceclaims\"},\n\t\t\t\tVerbs: []string{\"get\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tAPIGroups: []string{\"\"},\n\t\t\t\tResources: []string{\"nodes\"},\n\t\t\t\tVerbs: []string{\"get\"},\n\t\t\t},\n\t\t\t{\n\t\t\t\tAPIGroups: []string{\"resource.k8s.io\"},\n\t\t\t\tResources: []string{\"resourceslices\"},\n\t\t\t\tVerbs: []string{\"get\", \"list\", \"watch\", \"create\", \"update\", \"patch\", \"delete\"},\n\t\t\t},\n\t\t},\n\t}\n\n\tclusterRoleBinding := rbacv1.ClusterRoleBinding{\n\t\tTypeMeta: metav1.TypeMeta{\n\t\t\tAPIVersion: \"rbac.authorization.k8s.io/v1\",\n\t\t\tKind: \"ClusterRoleBinding\",\n\t\t},\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"dra-example-driver-role-binding\",\n\t\t\tNamespace: corev1.NamespaceDefault,\n\t\t},\n\t\tSubjects: []rbacv1.Subject{\n\t\t\t{\n\t\t\t\tKind: serviceAccount.Kind,\n\t\t\t\tName: serviceAccount.Name,\n\t\t\t\tNamespace: serviceAccount.Namespace,\n\t\t\t},\n\t\t},\n\t\tRoleRef: rbacv1.RoleRef{\n\t\t\tName: clusterRole.Name,\n\t\t\tKind: clusterRole.Kind,\n\t\t\tAPIGroup: \"rbac.authorization.k8s.io\",\n\t\t},\n\t}\n\n\tdriverDaemonset := appsv1.DaemonSet{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"dra-example-driver-kubeletplugin\",\n\t\t\tNamespace: corev1.NamespaceDefault,\n\t\t\tLabels: map[string]string{\n\t\t\t\t\"app.kubernetes.io/name\": \"dra-example-driver\",\n\t\t\t\t\"app.kubernetes.io/instance\": \"dra-example-driver\",\n\t\t\t\t\"app.kubernetes.io/component\": \"kubeletplugin\",\n\t\t\t},\n\t\t},\n\t\tSpec: appsv1.DaemonSetSpec{\n\t\t\tSelector: &metav1.LabelSelector{\n\t\t\t\tMatchLabels: map[string]string{\n\t\t\t\t\t\"app.kubernetes.io/name\": \"dra-example-driver\",\n\t\t\t\t\t\"app.kubernetes.io/instance\": \"dra-example-driver\",\n\t\t\t\t\t\"app.kubernetes.io/component\": \"kubeletplugin\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tUpdateStrategy: appsv1.DaemonSetUpdateStrategy{\n\t\t\t\tType: appsv1.RollingUpdateDaemonSetStrategyType,\n\t\t\t},\n\t\t\tTemplate: corev1.PodTemplateSpec{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tLabels: map[string]string{\n\t\t\t\t\t\t\"app.kubernetes.io/name\": \"dra-example-driver\",\n\t\t\t\t\t\t\"app.kubernetes.io/instance\": \"dra-example-driver\",\n\t\t\t\t\t\t\"app.kubernetes.io/component\": \"kubeletplugin\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tSpec: corev1.PodSpec{\n\t\t\t\t\tServiceAccountName: serviceAccount.Name,\n\t\t\t\t\tPriorityClassName: \"system-node-critical\",\n\t\t\t\t\tContainers: []corev1.Container{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName: \"plugin\",\n\t\t\t\t\t\t\tSecurityContext: &corev1.SecurityContext{Privileged: &[]bool{true}[0]},\n\t\t\t\t\t\t\tImage: \"registry.k8s.io/dra-example-driver/dra-example-driver:v0.1.0\",\n\t\t\t\t\t\t\tImagePullPolicy: corev1.PullIfNotPresent,\n\t\t\t\t\t\t\tCommand: []string{\"dra-example-kubeletplugin\"},\n\t\t\t\t\t\t\tEnv: []corev1.EnvVar{\n\t\t\t\t\t\t\t\t{Name: \"CDI_ROOT\", Value: \"/var/run/cdi\"},\n\t\t\t\t\t\t\t\t{Name: \"NODE_NAME\", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: \"spec.nodeName\"}}},\n\t\t\t\t\t\t\t\t{Name: \"NAMESPACE\", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: \"metadata.namespace\"}}},\n\t\t\t\t\t\t\t\t// NOTE: this is what arbitrarily decides the\n\t\t\t\t\t\t\t\t// number of GPUs being mocked on the node.\n\t\t\t\t\t\t\t\t{Name: \"NUM_DEVICES\", Value: \"8\"},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tVolumeMounts: []corev1.VolumeMount{\n\t\t\t\t\t\t\t\t{Name: \"plugins-registry\", MountPath: \"/var/lib/kubelet/plugins_registry\"},\n\t\t\t\t\t\t\t\t{Name: \"plugins\", MountPath: \"/var/lib/kubelet/plugins\"},\n\t\t\t\t\t\t\t\t{Name: \"cdi\", MountPath: \"/var/run/cdi\"},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tVolumes: []corev1.Volume{\n\t\t\t\t\t\t{Name: \"plugins-registry\", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: \"/var/lib/kubelet/plugins_registry\"}}},\n\t\t\t\t\t\t{Name: \"plugins\", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: \"/var/lib/kubelet/plugins\"}}},\n\t\t\t\t\t\t{Name: \"cdi\", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: \"/var/run/cdi\"}}},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\treturn []k8s.Object{\n\t\t&clusterRoleBinding,\n\t\t&clusterRole,\n\t\t&serviceAccount,\n\t\t&driverDaemonset,\n\t}\n}\n" }, { "path": "test/cases/dra/main_test.go", "content": "//go:build e2e\n\npackage dra\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"testing\"\n\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nvar (\n\ttestenv env.Environment\n)\n\nfunc TestMain(m *testing.M) {\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\ttestenv = env.NewWithConfig(cfg).WithContext(ctx)\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/efa/commons.go", "content": "//go:build e2e\n\npackage efa\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\nvar (\n\ttestenv env.Environment\n\tec2Client e2e.EC2Client\n\n\ttestImage *string\n\n\tpingPongSize *string\n\tpingPongIters *int\n\tpingPongDeadlineSeconds *int\n\n\tnodeType *string\n\texpectedEFADeviceCount *int\n\n\tverbose *bool\n)\n\nconst (\n\tEFA_RESOURCE_NAME = \"vpc.amazonaws.com/efa\"\n\tTEST_NAMESPACE_NAME = \"efa-tests\"\n)\n\nfunc getEfaCapacity(node corev1.Node) int {\n\tcapacity, ok := node.Status.Capacity[v1.ResourceName(EFA_RESOURCE_NAME)]\n\tif !ok {\n\t\treturn 0\n\t}\n\treturn int(capacity.Value())\n}\n\nfunc getEfaNodes(ctx context.Context, config *envconf.Config) ([]corev1.Node, error) {\n\tvar efaNodes []corev1.Node\n\tclientset, err := kubernetes.NewForConfig(config.Client().RESTConfig())\n\tif err != nil {\n\t\treturn []corev1.Node{}, fmt.Errorf(\"failed to create Kubernetes client: %w\", err)\n\t}\n\n\tnodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\tif err != nil {\n\t\treturn []corev1.Node{}, fmt.Errorf(\"failed to list nodes: %w\", err)\n\t}\n\n\tif len(nodes.Items) == 0 {\n\t\treturn []corev1.Node{}, fmt.Errorf(\"no nodes found in the cluster\")\n\t}\n\n\tfor _, node := range nodes.Items {\n\t\tinstanceType := node.Labels[\"node.kubernetes.io/instance-type\"]\n\n\t\tif aws.ToString(nodeType) != \"\" && instanceType != aws.ToString(nodeType) {\n\t\t\tlog.Printf(\"[INFO] Skipping node %s (type: %s), node is not of target type %s\", node.Name, instanceType, aws.ToString(nodeType))\n\t\t\tcontinue\n\t\t}\n\n\t\tnumEfaDevices, err := e2e.GetNonZeroResourceCapacity(&node, EFA_RESOURCE_NAME)\n\t\tif err != nil {\n\t\t\tlog.Printf(\"[INFO] Skipping node %s (type: %s): %v\", node.Name, instanceType, err)\n\t\t\tcontinue\n\t\t}\n\n\t\texpectedDeviceCount := aws.ToInt(expectedEFADeviceCount)\n\t\tif expectedDeviceCount < 0 {\n\t\t\tinstanceInfo, err := ec2Client.DescribeInstanceType(instanceType)\n\t\t\tif err != nil {\n\t\t\t\treturn []corev1.Node{}, err\n\t\t\t}\n\t\t\texpectedDeviceCount = int(aws.ToInt32(instanceInfo.NetworkInfo.EfaInfo.MaximumEfaInterfaces))\n\t\t}\n\n\t\tif expectedDeviceCount != numEfaDevices {\n\t\t\treturn []corev1.Node{}, fmt.Errorf(\"unexpected EFA device capacity on node %s: expected %d, got %d\", node.Name, expectedDeviceCount, numEfaDevices)\n\t\t}\n\n\t\tefaNodes = append(efaNodes, node)\n\t}\n\n\tif len(efaNodes) == 0 {\n\t\treturn []corev1.Node{}, fmt.Errorf(\"no nodes with EFA capacity found in the cluster\")\n\t}\n\n\treturn efaNodes, nil\n}\n" }, { "path": "test/cases/efa/main_test.go", "content": "//go:build e2e\n\npackage efa\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"flag\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nfunc getTestNamespace() *corev1.Namespace {\n\treturn &corev1.Namespace{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: TEST_NAMESPACE_NAME,\n\t\t},\n\t}\n}\n\nfunc deployEFAPlugin(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\terr := e2e.ApplyManifests(config.Client().RESTConfig(), manifests.EfaDevicePluginManifest)\n\tif err != nil {\n\t\treturn ctx, err\n\t}\n\tefaDS := appsv1.DaemonSet{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"aws-efa-k8s-device-plugin-daemonset\", Namespace: \"kube-system\"},\n\t}\n\terr = wait.For(e2e.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&efaDS),\n\t\twait.WithContext(ctx),\n\t\twait.WithTimeout(5*time.Minute),\n\t)\n\tif err != nil {\n\t\treturn ctx, err\n\t}\n\n\treturn ctx, nil\n}\n\nfunc TestMain(m *testing.M) {\n\ttestImage = flag.String(\"testImage\", \"\", \"container image to use for tests\")\n\tpingPongSize = flag.String(\"pingPongSize\", \"all\", \"sizes to use for ping pong\")\n\tpingPongIters = flag.Int(\"pingPongIters\", 10000, \"number of iterations to use for ping pong\")\n\tpingPongDeadlineSeconds = flag.Int(\"pingPongDeadlineSeconds\", 120, \"maximum run time for a ping pong attempt\")\n\tnodeType = flag.String(\"nodeType\", \"\", \"instance type to target for tests\")\n\texpectedEFADeviceCount = flag.Int(\"expectedEFADeviceCount\", -1, \"expected number of efa devices for the target nodes\")\n\tverbose = flag.Bool(\"verbose\", true, \"use verbose mode for tests\")\n\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\n\tif *testImage == \"\" {\n\t\tlog.Fatal(\"--testImage must be set, use https://github.com/aws/aws-k8s-tester/blob/main/test/efa/Dockerfile to build the image\")\n\t}\n\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttimedCtx, cancel := context.WithTimeout(ctx, 55*time.Minute)\n\tdefer cancel()\n\n\ttestenv = env.NewWithConfig(cfg)\n\ttestenv = testenv.WithContext(timedCtx)\n\n\tec2Client = e2e.NewEC2Client()\n\n\ttestenv.Setup(\n\t\tdeployEFAPlugin,\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t// Cooldown to let device plugin update node object with resources\n\t\t\tcase <-time.After(15 * time.Second):\n\t\t\t}\n\n\t\t\treturn ctx, cfg.Client().Resources().Create(ctx, getTestNamespace())\n\t\t},\n\t)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tcfg.Client().Resources().Delete(context.TODO(), getTestNamespace())\n\t\t\terr := e2e.DeleteManifests(cfg.Client().RESTConfig(), manifests.EfaDevicePluginManifest)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/efa/pingpong_test.go", "content": "//go:build e2e\n\npackage efa\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"log\"\n\t\"testing\"\n\t\"time\"\n\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/stretchr/testify/assert\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\nconst (\n\tPING_PONG_SERVICE_NAME = \"pingpong-service\"\n\tSERVER_POD_NAME = \"pingpong-server\"\n\tCLIENT_POD_NAME = \"pingpong-client\"\n\tPINGPONG_COMMAND = \"fi_pingpong\"\n)\n\nfunc getPingPongPodName(server bool) string {\n\tif server {\n\t\treturn SERVER_POD_NAME\n\t} else {\n\t\treturn CLIENT_POD_NAME\n\t}\n}\n\nfunc getPingPongArgs(server bool) (args []string) {\n\targs = []string{\"-S\", aws.ToString(pingPongSize), \"-I\", fmt.Sprint(aws.ToInt(pingPongIters)), \"-p\", \"efa\"}\n\tif aws.ToBool(verbose) {\n\t\targs = append(args, \"-v\")\n\t}\n\tif !server {\n\t\targs = append(args, fmt.Sprintf(\"%s.%s\", SERVER_POD_NAME, PING_PONG_SERVICE_NAME))\n\t}\n\treturn\n}\n\nfunc getPingPongResourceLabels(server bool) map[string]string {\n\treturn map[string]string{\n\t\t\"test-suite\": \"pingpong\",\n\t\t\"pingpong-server\": fmt.Sprint(server),\n\t}\n}\n\nfunc generatePingPongServiceManifest() corev1.Service {\n\treturn corev1.Service{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: PING_PONG_SERVICE_NAME,\n\t\t\tNamespace: TEST_NAMESPACE_NAME,\n\t\t},\n\t\tSpec: v1.ServiceSpec{\n\t\t\tSelector: getPingPongResourceLabels(true),\n\t\t\tClusterIP: \"None\",\n\t\t},\n\t}\n}\n\nfunc generatePingPongPodManifest(server bool, node corev1.Node) corev1.Pod {\n\tefaResourceQuantity := resource.MustParse(fmt.Sprint(getEfaCapacity(node)))\n\treturn corev1.Pod{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: getPingPongPodName(server),\n\t\t\tNamespace: TEST_NAMESPACE_NAME,\n\t\t\tLabels: getPingPongResourceLabels(server),\n\t\t},\n\t\tSpec: corev1.PodSpec{\n\t\t\tHostname: getPingPongPodName(server),\n\t\t\tSubdomain: PING_PONG_SERVICE_NAME,\n\t\t\tRestartPolicy: v1.RestartPolicyOnFailure,\n\t\t\t// TODO: centralize re-usable logic for pod spec formatting\n\t\t\tAffinity: &corev1.Affinity{\n\t\t\t\tNodeAffinity: &corev1.NodeAffinity{\n\t\t\t\t\tRequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{\n\t\t\t\t\t\tNodeSelectorTerms: []corev1.NodeSelectorTerm{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tMatchExpressions: []corev1.NodeSelectorRequirement{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tKey: \"kubernetes.io/hostname\",\n\t\t\t\t\t\t\t\t\t\tOperator: \"In\",\n\t\t\t\t\t\t\t\t\t\tValues: []string{\n\t\t\t\t\t\t\t\t\t\t\tnode.Name,\n\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tContainers: []corev1.Container{\n\t\t\t\t{\n\t\t\t\t\tName: \"pingpong\",\n\t\t\t\t\tImage: aws.ToString(testImage),\n\t\t\t\t\tCommand: []string{\"timeout\", fmt.Sprintf(\"%ds\", aws.ToInt(pingPongDeadlineSeconds)), PINGPONG_COMMAND},\n\t\t\t\t\tArgs: getPingPongArgs(server),\n\t\t\t\t\tResources: corev1.ResourceRequirements{\n\t\t\t\t\t\tRequests: corev1.ResourceList{\n\t\t\t\t\t\t\tEFA_RESOURCE_NAME: efaResourceQuantity,\n\t\t\t\t\t\t},\n\t\t\t\t\t\tLimits: corev1.ResourceList{\n\t\t\t\t\t\t\tEFA_RESOURCE_NAME: efaResourceQuantity,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc getPingPongPods(ctx context.Context, config *envconf.Config) (corev1.Pod, corev1.Pod, error) {\n\tefaNodes, err := getEfaNodes(ctx, config)\n\tif err != nil {\n\t\treturn corev1.Pod{}, corev1.Pod{}, err\n\t}\n\n\tif len(efaNodes) < 2 {\n\t\treturn corev1.Pod{}, corev1.Pod{}, fmt.Errorf(\"need at least 2 nodes with EFA capacity, got %d\", len(efaNodes))\n\t}\n\n\tserverNode := efaNodes[0]\n\tlog.Printf(\"[INFO] Using node %s (type: %s), as server\", serverNode.Name, serverNode.Labels[\"node.kubernetes.io/instance-type\"])\n\n\tclientNode := efaNodes[1]\n\tlog.Printf(\"[INFO] Using node %s (type: %s), as client\", clientNode.Name, clientNode.Labels[\"node.kubernetes.io/instance-type\"])\n\n\treturn generatePingPongPodManifest(true, serverNode), generatePingPongPodManifest(false, clientNode), nil\n}\n\nfunc TestPingPong(t *testing.T) {\n\tvar err error\n\tvar pingPongService corev1.Service\n\tvar client, server corev1.Pod\n\tpingpong := features.New(\"pingpong\").\n\t\tWithLabel(\"suite\", \"efa\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tpingPongService = generatePingPongServiceManifest()\n\t\t\tclient, server, err = getPingPongPods(ctx, cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tassert.NoError(t, cfg.Client().Resources().Create(ctx, &pingPongService))\n\t\t\tassert.NoError(t, cfg.Client().Resources().Create(ctx, &server))\n\t\t\tassert.NoError(t, cfg.Client().Resources().Create(ctx, &client))\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Pingpong between nodes succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tassert.NoError(t, wait.For(conditions.New(cfg.Client().Resources()).PodPhaseMatch(&server, v1.PodSucceeded),\n\t\t\t\twait.WithTimeout(15*time.Minute),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t))\n\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tserverPodLogs, err := e2e.ReadPodLogs(ctx, cfg.Client().RESTConfig(), server.Namespace, server.Name, server.Spec.Containers[0].Name)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"Could not get pods for server\")\n\t\t\t}\n\t\t\tt.Logf(\"Logs for server\\n%s\", serverPodLogs)\n\n\t\t\tassert.NoError(t, cfg.Client().Resources().Delete(ctx, &pingPongService))\n\t\t\tassert.NoError(t, cfg.Client().Resources().Delete(ctx, &server))\n\t\t\tassert.NoError(t, cfg.Client().Resources().Delete(ctx, &client))\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\ttestenv.Test(t, pingpong)\n}\n" }, { "path": "test/cases/efa/unit_test.go", "content": "//go:build e2e\n\npackage efa\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/stretchr/testify/assert\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nfunc generateUnitTestManifest(node corev1.Node, testIndex int) corev1.Pod {\n\tefaAllocatable := fmt.Sprint(getEfaCapacity(node))\n\tefaResourceQuantity := resource.MustParse(efaAllocatable)\n\treturn corev1.Pod{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: fmt.Sprintf(\"efa-unit-%d\", testIndex),\n\t\t\tNamespace: TEST_NAMESPACE_NAME,\n\t\t},\n\t\tSpec: corev1.PodSpec{\n\t\t\tRestartPolicy: v1.RestartPolicyOnFailure,\n\t\t\t// TODO: centralize re-usable logic for pod spec fkormatting\n\t\t\tAffinity: &corev1.Affinity{\n\t\t\t\tNodeAffinity: &corev1.NodeAffinity{\n\t\t\t\t\tRequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{\n\t\t\t\t\t\tNodeSelectorTerms: []corev1.NodeSelectorTerm{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tMatchExpressions: []corev1.NodeSelectorRequirement{\n\t\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t\tKey: \"kubernetes.io/hostname\",\n\t\t\t\t\t\t\t\t\t\tOperator: \"In\",\n\t\t\t\t\t\t\t\t\t\tValues: []string{\n\t\t\t\t\t\t\t\t\t\t\tnode.Name,\n\t\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tContainers: []corev1.Container{\n\t\t\t\t{\n\t\t\t\t\tName: \"unit-test\",\n\t\t\t\t\tImage: aws.ToString(testImage),\n\t\t\t\t\tCommand: []string{\"./scripts/unit-test.sh\"},\n\t\t\t\t\tEnv: []v1.EnvVar{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName: \"EXPECTED_EFA_DEVICE_COUNT\",\n\t\t\t\t\t\t\tValue: efaAllocatable,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName: \"EC2_INSTANCE_TYPE\",\n\t\t\t\t\t\t\tValue: node.Labels[\"node.kubernetes.io/instance-type\"],\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tResources: corev1.ResourceRequirements{\n\t\t\t\t\t\tRequests: corev1.ResourceList{\n\t\t\t\t\t\t\tEFA_RESOURCE_NAME: efaResourceQuantity,\n\t\t\t\t\t\t},\n\t\t\t\t\t\tLimits: corev1.ResourceList{\n\t\t\t\t\t\t\tEFA_RESOURCE_NAME: efaResourceQuantity,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc getUnitTestPodManifests(ctx context.Context, config *envconf.Config) ([]corev1.Pod, error) {\n\tvar podManifests []corev1.Pod\n\tefaNodes, err := getEfaNodes(ctx, config)\n\tif err != nil {\n\t\treturn []corev1.Pod{}, err\n\t}\n\n\tfor nodeIndex, node := range efaNodes {\n\t\tpodManifests = append(podManifests, generateUnitTestManifest(node, nodeIndex))\n\t}\n\n\treturn podManifests, err\n}\n\nfunc TestUnit(t *testing.T) {\n\tvar err error\n\tvar pods []corev1.Pod\n\tunit := features.New(\"unit\").\n\t\tWithLabel(\"suite\", \"efa\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tpods, err = getUnitTestPodManifests(ctx, cfg)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Failed to generate unit test manifests: %v\", err)\n\t\t\t}\n\n\t\t\tfor _, pod := range pods {\n\t\t\t\tassert.NoError(t, cfg.Client().Resources().Create(ctx, &pod))\n\t\t\t}\n\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Unit test succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tsuiteCtx, cancel := context.WithTimeout(ctx, 20*time.Minute)\n\t\t\tdefer cancel()\n\t\t\tfor _, pod := range pods {\n\t\t\t\tassert.NoError(t, wait.For(conditions.New(cfg.Client().Resources()).PodPhaseMatch(&pod, v1.PodSucceeded),\n\t\t\t\t\twait.WithContext(suiteCtx),\n\t\t\t\t))\n\t\t\t}\n\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tfor _, pod := range pods {\n\t\t\t\tpodLogs, err := e2e.ReadPodLogs(ctx, cfg.Client().RESTConfig(), pod.Namespace, pod.Name, pod.Spec.Containers[0].Name)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"Could not get logs for pod %q\", pod.Name)\n\t\t\t\t} else {\n\t\t\t\t\tt.Logf(\"Logs for pod %q\\n%s\", pod.Name, podLogs)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor _, pod := range pods {\n\t\t\t\tassert.NoError(t, cfg.Client().Resources().Delete(ctx, &pod))\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\ttestenv.Test(t, unit)\n}\n" }, { "path": "test/cases/fips/README.md", "content": "# FIPS TLS Compliance Test\n\nThis test validates that FIPS-enabled EKS nodes enforce FIPS-compliant TLS cipher suites when pulling container images.\n\n## What It Does\n\n1. Deploys two local container registries as DaemonSets on each node:\n - `registry-fips` (port 5000) — serves TLS using the node's default (FIPS-compliant) cipher suites\n - `registry-nonfips` (port 5001) — an nginx reverse proxy configured to only offer `ECDHE-RSA-CHACHA20-POLY1305`, a non-FIPS cipher\n2. Seeds both registries with a test image via `skopeo`\n3. Runs two test pods:\n - `test-pull-fips` — pulls from `localhost:5000` and expects success\n - `test-pull-nonfips` — pulls from `localhost:5001` and expects `ImagePullBackOff` (TLS handshake failure)\n\n## Prerequisites\n\n- An EKS cluster with FIPS-enabled nodes\n- TLS certificates available on each node at `/mnt/server-conf/certs/`:\n - `server.crt` — server certificate\n - `server.key` — private key\n- `kubeconfig` configured for the target cluster\n- Go 1.21+\n\n## Host Setup\n\n### Amazon Linux 2023\n\nFIPS mode must be enabled at launch time via the EKS AMI. Use a FIPS-enabled AL2023 AMI when creating the nodegroup:\n\n```bash\n# Create a FIPS-enabled nodegroup with eksctl\nkubetest2 eksctl \\\n --kubernetes-version=X.XX \\\n --ami-family=AmazonLinux2023 \\\n --up \\\n --down \\\n --test=exec \\\n -- \n```\n\nVerify FIPS is active on a node:\n```bash\n# SSH into a node and check\ncat /proc/sys/crypto/fips_enabled\n# Expected output: 1\n\n# Or check via sysctl\nsysctl crypto.fips_enabled\n# Expected output: crypto.fips_enabled = 1\n```\n\nGenerate the TLS certificates on each node:\n```bash\nsudo mkdir -p /mnt/server-conf/certs\nsudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \\\n -keyout /mnt/server-conf/certs/server.key \\\n -out /mnt/server-conf/certs/server.crt \\\n -subj \"/CN=localhost\" \\\n -addext \"subjectAltName=DNS:localhost,IP:127.0.0.1\"\n```\n\nAdd the certificate to the node's trust store so containerd trusts the local registries:\n```bash\nsudo cp /mnt/server-conf/certs/server.crt /etc/pki/ca-trust/source/anchors/\nsudo update-ca-trust\nsudo systemctl restart containerd\n```\n\nWithout this, containerd will reject the self-signed cert and both test pods would fail with `ImagePullBackOff`.\n\n### Bottlerocket\n\nBottlerocket is an immutable OS — you can't SSH in and run `openssl` directly. Certs must be provisioned via a bootstrap container that runs before kubelet starts.\n\n**1. Build the bootstrap container image**\n\nThe Dockerfile is minimal — it runs a user-data script at boot:\n\n```dockerfile\n\nFROM public.ecr.aws/docker/library/alpine:latest\nRUN apk add --no-cache openssl curl\nENTRYPOINT [\"/bin/sh\", \"/.bottlerocket/bootstrap-containers/gen-certs/user-data\"]\n```\n\nBuild and push to ECR:\n```bash\ndocker build -t .dkr.ecr..amazonaws.com/cert-bootstrap:v1 .\ndocker push .dkr.ecr..amazonaws.com/cert-bootstrap:v1\n```\n\n**2. Prepare the cert generation script**\n\nThe cert generation script generates a CA + server cert, writes them to the host at `/mnt/server-conf/certs/`, and registers the CA with Bottlerocket's trust store via `apiclient`:\n\n```bash\n#!/bin/sh\nset -xe\n\nWORK_DIR=$(mktemp -d)\nCERTS_DIR=/.bottlerocket/rootfs/mnt/server-conf/certs\nCSR_CONF=${WORK_DIR}/csr.conf\nCA_CRT=${WORK_DIR}/ca.crt\nCA_KEY=${WORK_DIR}/ca.key\n\nmkdir -p ${CERTS_DIR}\n\n# Generate CA\nopenssl genrsa -out ${CA_KEY} 2048\nopenssl req -x509 -new -nodes -key ${CA_KEY} \\\n -subj \"/CN=Bottlerocket Test CA/C=US/ST=WASHINGTON/L=Seattle/O=Bottlerocket\" \\\n -days 1825 -out ${CA_CRT}\n\n# Get instance metadata\nTOKEN=$(curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\")\nDOMAIN=$(curl -H \"X-aws-ec2-metadata-token: ${TOKEN}\" http://169.254.169.254/latest/meta-data/public-hostname)\nIP=$(curl -H \"X-aws-ec2-metadata-token: ${TOKEN}\" http://169.254.169.254/latest/meta-data/public-ipv4)\n\n# Generate CSR config with real values\ncat > ${CSR_CONF} <.dkr.ecr..amazonaws.com/cert-bootstrap:v1\"\nmode = \"once\"\nessential = true\nuser-data = \"\"\n```\n\n- `mode = \"once\"` — runs only on first boot\n- `essential = true` — node won't start if cert generation fails\n- The script runs before kubelet, so certs are ready when pods start\n\n**4. Launch with a FIPS Bottlerocket AMI**\n\n```bash\nkubetest2 eksctl \\\n --kubernetes-version=X.XX \\\n --ami-family=Bottlerocket \\\n --up \\\n --down \\\n --test=exec \\\n -- \n```\n\nVerify FIPS on Bottlerocket (via the admin container):\n```bash\ncat /proc/sys/crypto/fips_enabled\n# Expected output: 1\n```\n\n## Running the Test\n\n```bash\n# Run all FIPS test cases\ngo test -tags e2e -v ./test/cases/fips/ --kubeconfig=$HOME/.kube/config\n\n# Run a specific test case by label\ngo test -tags e2e -v ./test/cases/fips/ --kubeconfig=$HOME/.kube/config -labels=\"suite=fips\"\n```\n\nOr via `kubetest2`:\n```bash\nkubetest2 eksctl \\\n --kubernetes-version=X.XX \\\n --ami-family= \\\n --up \\\n --down \\\n --test=exec \\\n -- fips.test -v\n```\n\n## Test Cases\n\n| Test | Description | Expected Result |\n|------|-------------|-----------------|\n| `fips-tls-pull` | Pull image from FIPS-cipher registry (port 5000) | Pod succeeds |\n| `nonfips-tls-pull` | Pull image from non-FIPS-cipher registry (port 5001) | `ImagePullBackOff` — TLS handshake rejected |\n" }, { "path": "test/cases/fips/fips_test.go", "content": "//go:build e2e\n\npackage fips\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"io\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nconst (\n\tpullTimeout = 5 * time.Minute\n\trejectTimeout = 2 * time.Minute\n)\n\nvar (\n\t//go:embed manifests/registry-fips.yaml\n\tregistryFIPSManifest []byte\n\t//go:embed manifests/registry-nonfips.yaml\n\tregistryNonFIPSManifest []byte\n\n\t//go:embed manifests/test-pods.yaml\n\ttestPodsManifest []byte\n)\n\nfunc verifyNonfipsCipherRejection(ctx context.Context, t *testing.T, cfg *envconf.Config) {\n\tt.Helper()\n\tclientset, err := kubernetes.NewForConfig(cfg.Client().RESTConfig())\n\tif err != nil {\n\t\tt.Fatalf(\"could not create clientset for log verification: %v\", err)\n\t}\n\tlogCtx, logCancel := context.WithTimeout(ctx, logFetchTimeout)\n\tdefer logCancel()\n\tpods, err := clientset.CoreV1().Pods(\"default\").List(logCtx, metav1.ListOptions{\n\t\tLabelSelector: \"name=registry-nonfips\",\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"failed to list registry-nonfips pods: %v\", err)\n\t}\n\tif len(pods.Items) == 0 {\n\t\tt.Fatal(\"no registry-nonfips pods found for log verification\")\n\t}\n\tfor _, pod := range pods.Items {\n\t\treq := clientset.CoreV1().Pods(\"default\").GetLogs(pod.Name, &v1.PodLogOptions{\n\t\t\tContainer: \"nginx\",\n\t\t\tTailLines: int64Ptr(50),\n\t\t})\n\t\tstream, err := req.Stream(logCtx)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tbody, _ := io.ReadAll(stream)\n\t\tstream.Close()\n\t\tlogs := string(body)\n\t\tt.Logf(\"registry-nonfips nginx logs:\\n%s\", logs)\n\t\tif strings.Contains(logs, \"no shared cipher\") {\n\t\t\tt.Log(\"Verified: FIPS node rejected non-FIPS cipher suite (no shared cipher)\")\n\t\t\treturn\n\t\t}\n\t}\n\tt.Fatal(\"Expected 'no shared cipher' in registry-nonfips nginx logs but not found\")\n}\n\nfunc TestFIPSTLS(t *testing.T) {\n\tfipsPull := features.New(\"fips-tls-pull\").\n\t\tWithLabel(\"suite\", \"fips\").\n\t\tAssess(\"Pull from FIPS-cipher registry succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tpod := &v1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"test-pull-fips\", Namespace: \"default\"},\n\t\t\t}\n\t\t\terr := wait.For(\n\t\t\t\tconditions.New(cfg.Client().Resources()).PodPhaseMatch(pod, v1.PodSucceeded),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(pullTimeout),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"test-pull-fips pod did not succeed: %v\", err)\n\t\t\t}\n\t\t\tt.Log(\"FIPS TLS pull succeeded as expected\")\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tcfg.Client().Resources().Delete(ctx, &v1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"test-pull-fips\", Namespace: \"default\"},\n\t\t\t})\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\tnonfipsPull := features.New(\"nonfips-tls-pull\").\n\t\tWithLabel(\"suite\", \"fips\").\n\t\tAssess(\"Pull from non-FIPS-cipher registry fails on FIPS node\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tpod := &v1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"test-pull-nonfips\", Namespace: \"default\"},\n\t\t\t}\n\t\t\t// Poll for ImagePullBackOff/ErrImagePull — pod won't reach PodFailed phase\n\t\t\tdeadline := time.Now().Add(rejectTimeout)\n\t\t\tfor time.Now().Before(deadline) {\n\t\t\t\tselect {\n\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\tt.Fatalf(\"context cancelled while waiting for ImagePullBackOff: %v\", ctx.Err())\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t\terr := cfg.Client().Resources().Get(ctx, \"test-pull-nonfips\", \"default\", pod)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"failed to get test-pull-nonfips pod: %v\", err)\n\t\t\t\t}\n\t\t\t\t// #1: Log pod status during polling\n\t\t\t\tt.Logf(\"Polling test-pull-nonfips: Phase=%s\", pod.Status.Phase)\n\t\t\t\tfor _, cs := range pod.Status.ContainerStatuses {\n\t\t\t\t\tif cs.State.Waiting != nil {\n\t\t\t\t\t\tt.Logf(\" Container %s: Waiting (Reason=%s)\", cs.Name, cs.State.Waiting.Reason)\n\t\t\t\t\t} else if cs.State.Running != nil {\n\t\t\t\t\t\tt.Logf(\" Container %s: Running\", cs.Name)\n\t\t\t\t\t} else if cs.State.Terminated != nil {\n\t\t\t\t\t\tt.Logf(\" Container %s: Terminated (Reason=%s)\", cs.Name, cs.State.Terminated.Reason)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// #2: Detect unexpected success\n\t\t\t\tif pod.Status.Phase == v1.PodSucceeded {\n\t\t\t\t\tt.Fatal(\"test-pull-nonfips pod succeeded — expected ImagePullBackOff. Is this a FIPS node?\")\n\t\t\t\t}\n\t\t\t\tfor _, cs := range pod.Status.ContainerStatuses {\n\t\t\t\t\tif cs.State.Running != nil && cs.Ready {\n\t\t\t\t\t\tt.Fatal(\"test-pull-nonfips container is running — image pull succeeded. Is this a FIPS node?\")\n\t\t\t\t\t}\n\t\t\t\t\tif cs.State.Waiting != nil && (cs.State.Waiting.Reason == \"ImagePullBackOff\" || cs.State.Waiting.Reason == \"ErrImagePull\") {\n\t\t\t\t\t\tverifyNonfipsCipherRejection(ctx, t, cfg)\n\t\t\t\t\t\tt.Log(\"Non-FIPS TLS pull correctly rejected (ImagePullBackOff)\")\n\t\t\t\t\t\treturn ctx\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ttime.Sleep(pollInterval)\n\t\t\t}\n\t\t\tt.Fatal(\"test-pull-nonfips did not reach ImagePullBackOff within timeout\")\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tcfg.Client().Resources().Delete(ctx, &v1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"test-pull-nonfips\", Namespace: \"default\"},\n\t\t\t})\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, fipsPull, nonfipsPull)\n}\n" }, { "path": "test/cases/fips/main_test.go", "content": "//go:build e2e\n\npackage fips\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nconst (\n\tpollInterval = 5 * time.Second // polling interval for waitForSeed and status checks\n\tseedTimeout = 5 * time.Minute // apk install + skopeo copy can be slow on first pull\n\tdaemonSetTimeout = 2 * time.Minute // per DaemonSet; image pulls vary by network\n\tlogFetchTimeout = 30 * time.Second // timeout for fetching pod logs\n\t// Worst-case Setup: 2x daemonSetTimeout (4m) + 2x seedTimeout (6m) = ~10m\n)\n\nvar testenv env.Environment\n\nfunc int64Ptr(i int64) *int64 { return &i }\n\n\nfunc logDaemonSetDiagnostics(ctx context.Context, clientset *kubernetes.Clientset, dsName string) {\n\tlog.Printf(\"=== Diagnostics for DaemonSet %s ===\", dsName)\n\tpods, err := clientset.CoreV1().Pods(\"default\").List(ctx, metav1.ListOptions{\n\t\tLabelSelector: \"name=\" + dsName,\n\t})\n\tif err != nil {\n\t\tlog.Printf(\"Failed to list pods: %v\", err)\n\t\treturn\n\t}\n\tfor _, pod := range pods.Items {\n\t\tlog.Printf(\"Pod %s: Phase=%s\", pod.Name, pod.Status.Phase)\n\t\tfor _, cond := range pod.Status.Conditions {\n\t\t\tlog.Printf(\" Condition %s: %s (Reason: %s)\", cond.Type, cond.Status, cond.Reason)\n\t\t}\n\t\tfor _, cs := range pod.Status.ContainerStatuses {\n\t\t\tlog.Printf(\" Container %s: Ready=%v, RestartCount=%d\", cs.Name, cs.Ready, cs.RestartCount)\n\t\t\tif cs.State.Waiting != nil {\n\t\t\t\tlog.Printf(\" Waiting: %s - %s\", cs.State.Waiting.Reason, cs.State.Waiting.Message)\n\t\t\t}\n\t\t\tif cs.State.Terminated != nil {\n\t\t\t\tlog.Printf(\" Terminated: %s - %s\", cs.State.Terminated.Reason, cs.State.Terminated.Message)\n\t\t\t}\n\t\t\tif (cs.State.Waiting != nil && cs.State.Waiting.Reason == \"CrashLoopBackOff\") || cs.RestartCount > 0 {\n\t\t\t\treq := clientset.CoreV1().Pods(\"default\").GetLogs(pod.Name, &v1.PodLogOptions{\n\t\t\t\t\tContainer: cs.Name,\n\t\t\t\t\tTailLines: int64Ptr(20),\n\t\t\t\t})\n\t\t\t\tstream, err := req.Stream(ctx)\n\t\t\t\tif err == nil {\n\t\t\t\t\tbody, _ := io.ReadAll(stream)\n\t\t\t\t\tstream.Close()\n\t\t\t\t\tlog.Printf(\" Last logs:\\n%s\", string(body))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n\nfunc logNodeInfo(ctx context.Context, clientset *kubernetes.Clientset) {\n\tnodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\tif err != nil {\n\t\tlog.Printf(\"Warning: could not list nodes: %v\", err)\n\t\treturn\n\t}\n\tfor _, node := range nodes.Items {\n\t\tosImage := node.Status.NodeInfo.OSImage\n\t\tisFIPS := strings.Contains(strings.ToLower(osImage), \"fips\")\n\t\tlog.Printf(\"Node %s: OS=%s, FIPS=%v\", node.Name, osImage, isFIPS)\n\t}\n}\n\n// normally this will only take couple seconds.\nfunc waitForSeed(ctx context.Context, clientset *kubernetes.Clientset, dsName string) error {\n\tlog.Printf(\"Waiting for %s seed container to complete...\", dsName)\n\tdeadline := time.Now().Add(seedTimeout)\n\tvar lastLogs string\n\tfor time.Now().Before(deadline) {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn ctx.Err()\n\t\tdefault:\n\t\t}\n\t\tpods, err := clientset.CoreV1().Pods(\"default\").List(ctx, metav1.ListOptions{\n\t\t\tLabelSelector: \"name=\" + dsName,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif len(pods.Items) == 0 {\n\t\t\tlog.Printf(\"%s: no pods found yet, waiting...\", dsName)\n\t\t\ttime.Sleep(pollInterval)\n\t\t\tcontinue\n\t\t}\n\t\tallSeeded := true\n\t\tfor _, pod := range pods.Items {\n\t\t\treq := clientset.CoreV1().Pods(\"default\").GetLogs(pod.Name, &v1.PodLogOptions{\n\t\t\t\tContainer: \"seed-image\",\n\t\t\t})\n\t\t\tlogCtx, logCancel := context.WithTimeout(ctx, logFetchTimeout)\n\t\t\tstream, err := req.Stream(logCtx)\n\t\t\tif err != nil {\n\t\t\t\tlogCancel()\n\t\t\t\tlog.Printf(\"Failed to get logs for %s/%s: %v\", dsName, pod.Name, err)\n\t\t\t\tallSeeded = false\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tbody, _ := io.ReadAll(stream)\n\t\t\tstream.Close()\n\t\t\tlogCancel()\n\t\t\tlogs := string(body)\n\t\t\tif strings.Contains(logs, \"level=fatal\") {\n\t\t\t\treturn fmt.Errorf(\"%s seed failed: %s\", dsName, logs)\n\t\t\t}\n\t\t\tif !strings.Contains(logs, \"Image seeded successfully\") {\n\t\t\t\tallSeeded = false\n\t\t\t\tlastLogs = logs\n\t\t\t}\n\t\t}\n\t\tif allSeeded {\n\t\t\tlog.Printf(\"%s seed completed successfully on all %d pods\", dsName, len(pods.Items))\n\t\t\treturn nil\n\t\t}\n\t\tlog.Printf(\"%s seed still waiting... (got %d bytes of logs)\", dsName, len(lastLogs))\n\t\ttime.Sleep(pollInterval)\n\t}\n\t// Dump last logs on timeout\n\tif lastLogs != \"\" {\n\t\tlog.Printf(\"%s seed timeout - last logs:\\n%s\", dsName, lastLogs)\n\t}\n\treturn fmt.Errorf(\"%s seed did not complete within timeout\", dsName)\n}\n\nfunc TestMain(m *testing.M) {\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\ttestenv = env.NewWithConfig(cfg)\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = testenv.WithContext(ctx)\n\n\ttestenv.Setup(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tclientset, err := kubernetes.NewForConfig(config.Client().RESTConfig())\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to create Kubernetes client: %w\", err)\n\t\t\t}\n\t\t\tlogNodeInfo(ctx, clientset)\n\t\t\tif err := fwext.ApplyManifests(config.Client().RESTConfig(), registryFIPSManifest); err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to apply registry-fips manifest: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"registry-fips DaemonSet deployed\")\n\n\t\t\tif err := fwext.ApplyManifests(config.Client().RESTConfig(), registryNonFIPSManifest); err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to apply registry-nonfips manifest: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"registry-nonfips DaemonSet deployed\")\n\n\t\t\tfor _, name := range []string{\"registry-fips\", \"registry-nonfips\"} {\n\t\t\t\tds := appsv1.DaemonSet{\n\t\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: name, Namespace: \"default\"},\n\t\t\t\t}\n\t\t\t\tlog.Printf(\"Waiting for %s DaemonSet to be ready...\", name)\n\t\t\t\terr := wait.For(\n\t\t\t\t\tfwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&ds),\n\t\t\t\t\twait.WithContext(ctx),\n\t\t\t\t\twait.WithTimeout(daemonSetTimeout),\n\t\t\t\t)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlogDaemonSetDiagnostics(ctx, clientset, name)\n\t\t\t\t\treturn ctx, fmt.Errorf(\"%s DaemonSet not ready: %w\", name, err)\n\t\t\t\t}\n\t\t\t\tlog.Printf(\"%s DaemonSet is ready\", name)\n\t\t\t}\n\n\t\t\tfor _, dsName := range []string{\"registry-fips\", \"registry-nonfips\"} {\n\t\t\t\tif err := waitForSeed(ctx, clientset, dsName); err != nil {\n\t\t\t\t\treturn ctx, fmt.Errorf(\"seed verification failed for %s: %w\", dsName, err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif err := fwext.ApplyManifests(config.Client().RESTConfig(), testPodsManifest); err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to apply test-pods manifest: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"test pods deployed\")\n\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tfwext.DeleteManifests(config.Client().RESTConfig(), registryFIPSManifest)\n\t\t\tfwext.DeleteManifests(config.Client().RESTConfig(), registryNonFIPSManifest)\n\t\t\tfwext.DeleteManifests(config.Client().RESTConfig(), testPodsManifest)\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/fips/manifests/registry-fips.yaml", "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: registry-fips\nspec:\n selector:\n matchLabels:\n name: registry-fips\n template:\n metadata:\n labels:\n name: registry-fips\n spec:\n hostNetwork: true\n containers:\n - name: registry\n image: registry:2\n env:\n - name: REGISTRY_HTTP_ADDR\n value: \"0.0.0.0:5000\"\n - name: REGISTRY_HTTP_TLS_CERTIFICATE\n value: \"/certs/server.crt\"\n - name: REGISTRY_HTTP_TLS_KEY\n value: \"/certs/server.key\"\n volumeMounts:\n - name: certs\n mountPath: /certs\n - name: seed-image\n image: public.ecr.aws/docker/library/alpine:latest\n command:\n - /bin/sh\n - -c\n - |\n apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community skopeo\n sleep 5\n skopeo copy --dest-tls-verify=false docker://public.ecr.aws/docker/library/alpine:latest docker://127.0.0.1:5000/test:latest\n echo \"Image seeded successfully\"\n sleep infinity\n volumes:\n - name: certs\n hostPath:\n path: /mnt/server-conf/certs\n" }, { "path": "test/cases/fips/manifests/registry-nonfips.yaml", "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: nginx-nonfips-config\ndata:\n nginx.conf: |\n events {}\n http {\n error_log /dev/stderr debug;\n server {\n listen 5001 ssl;\n ssl_certificate /certs/server.crt;\n ssl_certificate_key /certs/server.key;\n ssl_protocols TLSv1.2;\n ssl_ciphers ECDHE-RSA-CHACHA20-POLY1305;\n location / {\n proxy_pass http://127.0.0.1:5002;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n }\n }\n }\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: registry-nonfips\nspec:\n selector:\n matchLabels:\n name: registry-nonfips\n template:\n metadata:\n labels:\n name: registry-nonfips\n spec:\n hostNetwork: true\n containers:\n - name: nginx\n image: public.ecr.aws/nginx/nginx:stable-alpine\n volumeMounts:\n - name: certs\n mountPath: /certs\n - name: nginx-config\n mountPath: /etc/nginx/nginx.conf\n subPath: nginx.conf\n - name: registry\n image: registry:2\n env:\n - name: REGISTRY_HTTP_ADDR\n value: \"127.0.0.1:5002\"\n - name: seed-image\n image: public.ecr.aws/docker/library/alpine:latest\n command:\n - /bin/sh\n - -c\n - |\n apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community skopeo\n sleep 5\n skopeo copy --dest-tls-verify=false docker://public.ecr.aws/docker/library/alpine:latest docker://127.0.0.1:5002/test:latest\n echo \"Image seeded successfully\"\n sleep infinity\n volumes:\n - name: certs\n hostPath:\n path: /mnt/server-conf/certs\n - name: nginx-config\n configMap:\n name: nginx-nonfips-config\n" }, { "path": "test/cases/fips/manifests/test-pods.yaml", "content": "apiVersion: v1\nkind: Pod\nmetadata:\n name: test-pull-fips\nspec:\n containers:\n - name: test\n image: localhost:5000/test:latest\n command: [\"echo\", \"FIPS cipher works\"]\n restartPolicy: Never\n---\napiVersion: v1\nkind: Pod\nmetadata:\n name: test-pull-nonfips\nspec:\n containers:\n - name: test\n image: localhost:5001/test:latest\n command: [\"echo\", \"should not reach here\"]\n restartPolicy: Never\n" }, { "path": "test/cases/netpol/main_test.go", "content": "//go:build e2e\n\npackage netpol\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"log\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks\"\n\t\"github.com/aws/aws-sdk-go-v2/service/eks/types\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/pkg/errors\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nvar (\n\ttestenv env.Environment\n\tclusterName string\n\tendPointUrl string\n\tkubernetesVersion string\n\taddonName string = \"vpc-cni\"\n)\n\nfunc TestMain(m *testing.M) {\n\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\n\tconfig, err := config.LoadDefaultConfig(context.TODO())\n\teksclient := eks.NewFromConfig(config)\n\ttestenv = env.NewWithConfig(cfg)\n\n\tflag.StringVar(&clusterName, \"cluster-name\", \"\", \"Name of the cluster\")\n\tflag.StringVar(&endPointUrl, \"endpoint-url\", \"\", \"Endpoint url to use\")\n\tflag.Parse()\n\n\tnamespaces := []string{\"a\", \"b\", \"c\"}\n\n\ttestenv.Setup(\n\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tclient, err := config.NewClient()\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\n\t\t\tservers := map[string]string{\n\t\t\t\t\"a\": \"a-server\",\n\t\t\t\t\"b\": \"b-server\",\n\t\t\t\t\"c\": \"c-server\",\n\t\t\t}\n\n\t\t\t// 1. Install Latest CNI version\n\t\t\tlog.Print(\"Install the latest VPC-CNI on the cluster\")\n\t\t\tkubernetesVersion, err = getClusterVersion(ctx, eksclient)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\n\t\t\terr = installLatestCNIVersion(ctx, config, eksclient)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\n\t\t\t// 2. Create three namespaces\n\t\t\tlog.Print(\"Creating the test namespaces\")\n\t\t\tfor _, ns := range namespaces {\n\t\t\t\terr = createNamespace(ns, client, ctx)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn ctx, errors.Wrapf(err, \"Failed to create namespace %s\", ns)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// 3. Create deployment and service\n\t\t\tlog.Print(\"Creating the test deployment and service\")\n\t\t\tfor ns, server := range servers {\n\t\t\t\terr = createServerAndService(ns, server, 1, client, ctx)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn ctx, errors.Wrapf(err, \"Failed to create deployment and service for %s\", server)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tclient, err := config.NewClient()\n\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\n\t\t\tlog.Print(\"Deleting the test namespaces\")\n\t\t\tfor _, ns := range namespaces {\n\t\t\t\tclient.Resources().Delete(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns, Namespace: ns}})\n\t\t\t}\n\n\t\t\tlog.Print(\"Installing the Default version of VPC-CNI on the cluster\")\n\t\t\terr = installDefaultCNIVersion(ctx, config, eksclient)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tos.Exit(testenv.Run(m))\n}\n\nfunc installDefaultCNIVersion(ctx context.Context, config *envconf.Config, eksclient *eks.Client) error {\n\n\t// Uninstall the currently install addon\n\tuninstallCNIAddon(ctx, config, eksclient)\n\n\t// Passing addonVersion empty installs the default version of addon\n\terr := installCNIAddon(ctx, config, eksclient, \"\", \"\")\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"Could not install the default addon version\")\n\t}\n\n\treturn nil\n}\n\nfunc installLatestCNIVersion(ctx context.Context, config *envconf.Config, eksclient *eks.Client) error {\n\n\tversion, err := getLatestCNIAddon(ctx, eksclient)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tconfigurationValues := \"{\\\"enableNetworkPolicy\\\": \\\"true\\\"}\"\n\terr = installCNIAddon(ctx, config, eksclient, version, configurationValues)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc uninstallCNIAddon(ctx context.Context, config *envconf.Config, eksclient *eks.Client) error {\n\n\tcniDS := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: \"aws-node\", Namespace: \"kube-system\"}}\n\n\t_, err := eksclient.DeleteAddon(ctx, &eks.DeleteAddonInput{\n\t\tAddonName: aws.String(addonName),\n\t\tClusterName: aws.String(clusterName),\n\t})\n\n\terr = wait.For(conditions.New(config.Client().Resources()).ResourceDeleted(cniDS), wait.WithTimeout(time.Minute*5))\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"Daemonset could not be deleted\")\n\t}\n\n\treturn nil\n}\n\nfunc getLatestCNIAddon(ctx context.Context, eksclient *eks.Client) (string, error) {\n\n\taddonVersions, err := eksclient.DescribeAddonVersions(ctx, &eks.DescribeAddonVersionsInput{\n\t\tAddonName: aws.String(addonName),\n\t\tKubernetesVersion: aws.String(kubernetesVersion),\n\t})\n\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif len(*&addonVersions.Addons) > 0 {\n\t\treturn *addonVersions.Addons[0].AddonVersions[0].AddonVersion, nil\n\t} else {\n\t\treturn \"\", errors.Errorf(\"Addon versions not available\")\n\t}\n}\n\nfunc installCNIAddon(ctx context.Context, config *envconf.Config, eksclient *eks.Client, addonVersion string, configurationValues string) error {\n\n\t// Delete old Daemonset if exists\n\tcniDS := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: \"aws-node\", Namespace: \"kube-system\"}}\n\tconfig.Client().Resources().Delete(ctx, cniDS)\n\n\t_, err := eksclient.CreateAddon(ctx, &eks.CreateAddonInput{\n\t\tAddonName: aws.String(addonName),\n\t\tClusterName: aws.String(clusterName),\n\t\tAddonVersion: aws.String(addonVersion),\n\t\tConfigurationValues: aws.String(configurationValues),\n\t\tResolveConflicts: types.ResolveConflictsOverwrite,\n\t})\n\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"Failed to create addon\")\n\t}\n\n\terr = wait.For(fwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(cniDS), wait.WithTimeout(time.Minute*5))\n\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"Daemonset failed to reach running state\")\n\t}\n\n\treturn nil\n}\n\nfunc getClusterVersion(ctx context.Context, eksclient *eks.Client) (string, error) {\n\n\tcluster, err := eksclient.DescribeCluster(ctx, &eks.DescribeClusterInput{\n\t\tName: aws.String(clusterName),\n\t})\n\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn *cluster.Cluster.Version, nil\n}\n\nfunc createNamespace(name string, client klient.Client, ctx context.Context) error {\n\n\tns := &v1.Namespace{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: name,\n\t\t\tNamespace: name,\n\t\t\tLabels: map[string]string{\"ns\": name},\n\t\t},\n\t}\n\n\tif err := client.Resources().Create(ctx, ns); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc createServerAndService(namespace string, name string, replicas int32, client klient.Client, ctx context.Context) error {\n\n\tlabels := map[string]string{\"app\": name}\n\n\tservice := &v1.Service{\n\t\tObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},\n\t\tSpec: v1.ServiceSpec{\n\t\t\tPorts: []v1.ServicePort{{Name: name, Protocol: \"TCP\", Port: 80}},\n\t\t\tSelector: labels,\n\t\t},\n\t}\n\n\tif err := client.Resources().Create(ctx, service); err != nil {\n\t\treturn err\n\t}\n\n\tdeploy := &appsv1.Deployment{\n\t\tObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},\n\t\tSpec: appsv1.DeploymentSpec{\n\t\t\tReplicas: &replicas,\n\t\t\tSelector: &metav1.LabelSelector{\n\t\t\t\tMatchLabels: labels,\n\t\t\t},\n\t\t\tTemplate: corev1.PodTemplateSpec{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Labels: labels},\n\t\t\t\tSpec: corev1.PodSpec{Containers: []corev1.Container{{Name: name, Image: \"nginx\"}}},\n\t\t\t},\n\t\t},\n\t}\n\n\tif err := client.Resources().Create(ctx, deploy); err != nil {\n\t\treturn err\n\t}\n\n\terr := wait.For(conditions.New(client.Resources()).DeploymentConditionMatch(deploy, appsv1.DeploymentAvailable, v1.ConditionTrue),\n\t\twait.WithTimeout(time.Minute*5))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n" }, { "path": "test/cases/netpol/np_test.go", "content": "//go:build e2e\n\npackage netpol\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"log\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tnetworking \"k8s.io/api/networking/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/apimachinery/pkg/util/intstr\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nfunc TestNetworkPolicyCases(t *testing.T) {\n\n\tprotocolTCP := corev1.ProtocolTCP\n\tprotocolUDP := corev1.ProtocolUDP\n\tnetworkPolicy := networking.NetworkPolicy{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"block-c-to-a\", Namespace: \"a\"},\n\t\tSpec: networking.NetworkPolicySpec{\n\t\t\tPodSelector: metav1.LabelSelector{MatchLabels: map[string]string{\"app\": \"a-server\"}},\n\t\t\tPolicyTypes: []networking.PolicyType{networking.PolicyTypeIngress, networking.PolicyTypeEgress},\n\t\t\tIngress: []networking.NetworkPolicyIngressRule{\n\t\t\t\t{\n\t\t\t\t\tFrom: []networking.NetworkPolicyPeer{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{\"app\": \"b-server\"}},\n\t\t\t\t\t\t\tNamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{\"ns\": \"b\"}},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tPorts: []networking.NetworkPolicyPort{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tProtocol: &protocolTCP,\n\t\t\t\t\t\t\tPort: &intstr.IntOrString{IntVal: 80},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\tEgress: []networking.NetworkPolicyEgressRule{\n\t\t\t\t{\n\t\t\t\t\tTo: []networking.NetworkPolicyPeer{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{\"app\": \"b-server\"}},\n\t\t\t\t\t\t\tNamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{\"ns\": \"b\"}},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tPorts: []networking.NetworkPolicyPort{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tProtocol: &protocolTCP,\n\t\t\t\t\t\t\tPort: &intstr.IntOrString{IntVal: 80},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tPorts: []networking.NetworkPolicyPort{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tProtocol: &protocolUDP,\n\t\t\t\t\t\t\tPort: &intstr.IntOrString{IntVal: 53},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tallowAll := features.New(\"allowAll\").\n\t\tWithLabel(\"suite\", \"netpol\").\n\t\tWithLabel(\"policy\", \"none\").\n\t\tAssess(\"curl from A to B succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tclient, err := cfg.NewClient()\n\t\t\tif err != nil {\n\t\t\t\treturn ctx\n\t\t\t}\n\t\t\tpods := &corev1.PodList{}\n\t\t\tnamespace := \"a\"\n\t\t\tcontainerName := \"a-server\"\n\t\t\terr = client.Resources(\"a\").List(context.TODO(), pods)\n\t\t\tif err != nil || pods.Items == nil {\n\t\t\t\tt.Error(\"error while getting pods\", err)\n\t\t\t}\n\t\t\tpodName := pods.Items[0].Name\n\n\t\t\tvar stdout, stderr bytes.Buffer\n\t\t\tcommand := []string{\"curl\", \"-m\", \"2\", \"-I\", \"http://b-server.b:80\"}\n\t\t\tclient.Resources().ExecInPod(context.TODO(), namespace, podName, containerName, command, &stdout, &stderr)\n\n\t\t\thttpStatus := strings.Split(stdout.String(), \"\\n\")[0]\n\t\t\tif !strings.Contains(httpStatus, \"200\") {\n\t\t\t\tt.Fatal(\"Couldn't connect to server B\")\n\t\t\t}\n\t\t\treturn ctx\n\n\t\t}).\n\t\tAssess(\"curl from C to A succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tclient, err := cfg.NewClient()\n\t\t\tif err != nil {\n\t\t\t\treturn ctx\n\t\t\t}\n\t\t\tnamespace := \"c\"\n\t\t\tcontainerName := \"c-server\"\n\t\t\tpods := &corev1.PodList{}\n\t\t\terr = client.Resources(\"c\").List(context.TODO(), pods)\n\t\t\tif err != nil || pods.Items == nil {\n\t\t\t\tt.Error(\"error while getting pods\", err)\n\t\t\t}\n\t\t\tpodName := pods.Items[0].Name\n\n\t\t\tvar stdout, stderr bytes.Buffer\n\t\t\tcommand := []string{\"curl\", \"-m\", \"2\", \"-I\", \"http://a-server.a:80\"}\n\t\t\tclient.Resources().ExecInPod(context.TODO(), namespace, podName, containerName, command, &stdout, &stderr)\n\n\t\t\thttpStatus := strings.Split(stdout.String(), \"\\n\")[0]\n\t\t\tif !strings.Contains(httpStatus, \"200\") {\n\t\t\t\tt.Fatal(\"Couldn't connect to server A\")\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\tblockCToA := features.New(\"blockCToA\").\n\t\tWithLabel(\"suite\", \"netpol\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tclient, err := cfg.NewClient()\n\t\t\tif err != nil {\n\t\t\t\treturn ctx\n\t\t\t}\n\n\t\t\tlog.Print(\"Applying Network Policy\")\n\t\t\tif err := client.Resources().Create(ctx, &networkPolicy); err != nil {\n\t\t\t\tt.Error(\"error while applying Network Policy\", err)\n\t\t\t\treturn ctx\n\t\t\t}\n\n\t\t\t// This time-wait is to account for Network Policy Controller to start up, run leader election in the control plane\n\t\t\t// and to apply the network policy\n\t\t\ttime.Sleep(1 * time.Minute)\n\n\t\t\treturn ctx\n\n\t\t}).\n\t\tAssess(\"curl from A to B succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tclient, err := cfg.NewClient()\n\t\t\tif err != nil {\n\t\t\t\treturn ctx\n\t\t\t}\n\t\t\tpods := &corev1.PodList{}\n\t\t\tnamespace := \"a\"\n\t\t\tcontainerName := \"a-server\"\n\t\t\terr = client.Resources(\"a\").List(context.TODO(), pods)\n\t\t\tif err != nil || pods.Items == nil {\n\t\t\t\tt.Error(\"error while getting pods\", err)\n\t\t\t}\n\t\t\tpodName := pods.Items[0].Name\n\n\t\t\tvar stdout, stderr bytes.Buffer\n\t\t\tcommand := []string{\"curl\", \"-m\", \"2\", \"-I\", \"http://b-server.b:80\"}\n\t\t\tclient.Resources().ExecInPod(context.TODO(), namespace, podName, containerName, command, &stdout, &stderr)\n\n\t\t\thttpStatus := strings.Split(stdout.String(), \"\\n\")[0]\n\t\t\tif !strings.Contains(httpStatus, \"200\") {\n\t\t\t\tt.Fatal(\"Couldn't connect to server B\")\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"curl from C to A fails\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tclient, err := cfg.NewClient()\n\t\t\tif err != nil {\n\t\t\t\treturn ctx\n\t\t\t}\n\t\t\tnamespace := \"c\"\n\t\t\tcontainerName := \"c-server\"\n\t\t\tpods := &corev1.PodList{}\n\t\t\terr = client.Resources(\"c\").List(context.TODO(), pods)\n\t\t\tif err != nil || pods.Items == nil {\n\t\t\t\tt.Error(\"error while getting pods\", err)\n\t\t\t}\n\t\t\tpodName := pods.Items[0].Name\n\n\t\t\tvar stdout, stderr bytes.Buffer\n\t\t\tcommand := []string{\"curl\", \"-m\", \"2\", \"-I\", \"http://a-server.a:80\"}\n\t\t\tclient.Resources().ExecInPod(context.TODO(), namespace, podName, containerName, command, &stdout, &stderr)\n\n\t\t\thttpStatus := strings.Split(stdout.String(), \"\\n\")[0]\n\t\t\tif strings.Contains(httpStatus, \"200\") {\n\t\t\t\tt.Fatal(\"Network Policy didn't block connection to server A\")\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tclient, err := cfg.NewClient()\n\t\t\tif err != nil {\n\t\t\t\treturn ctx\n\t\t\t}\n\n\t\t\tif err := client.Resources().Delete(ctx, &networkPolicy); err != nil {\n\t\t\t\tt.Error(\"error while deleting Network Policy\", err)\n\t\t\t\treturn ctx\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, allowAll, blockCToA)\n}\n" }, { "path": "test/cases/neuron/main_test.go", "content": "//go:build e2e\n\npackage neuron\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"slices\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nvar (\n\ttestenv env.Environment\n\tnodeType *string\n\tefaEnabled *bool\n\tnodeCount int\n\tneuronPerNode int\n\tneuronCorePerNode int\n\tefaPerNode int\n\tneuronTestImage *string\n\tinstallDevicePlugin *bool\n)\n\nfunc deployNeuronDevicePlugin(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tds := appsv1.DaemonSet{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"neuron-device-plugin-daemonset\", Namespace: \"kube-system\"},\n\t}\n\terr := wait.For(fwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&ds),\n\t\twait.WithContext(ctx))\n\tif err != nil {\n\t\treturn ctx, err\n\t}\n\treturn ctx, nil\n}\n\nfunc deployMPIOperator(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tdep := appsv1.Deployment{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"mpi-operator\", Namespace: \"mpi-operator\"},\n\t}\n\terr := wait.For(conditions.New(config.Client().Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, v1.ConditionTrue),\n\t\twait.WithContext(ctx))\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to deploy mpi-operator: %v\", err)\n\t}\n\treturn ctx, nil\n}\n\nfunc deployEFAPlugin(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\terr := fwext.ApplyManifests(config.Client().RESTConfig(), manifests.EfaDevicePluginManifest)\n\tif err != nil {\n\t\treturn ctx, err\n\t}\n\n\tds := appsv1.DaemonSet{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"aws-efa-k8s-device-plugin-daemonset\", Namespace: \"kube-system\"},\n\t}\n\terr = wait.For(fwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&ds),\n\t\twait.WithContext(ctx))\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to deploy efa-device-plugin: %v\", err)\n\t}\n\n\treturn ctx, nil\n}\n\nfunc checkNodeTypes(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\ttime.Sleep(time.Minute) // give node info time to populate\n\n\tclientset, err := kubernetes.NewForConfig(config.Client().RESTConfig())\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to create Kubernetes client: %w\", err)\n\t}\n\n\tnodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to list nodes: %w\", err)\n\t}\n\n\tif len(nodes.Items) == 0 {\n\t\treturn ctx, fmt.Errorf(\"no nodes found in the cluster\")\n\t}\n\n\tvar totalEfaCount, totalNeuronCoreCount, totalNeuronCount int\n\tif *nodeType == \"\" {\n\t\tnodeType = aws.String(nodes.Items[0].Labels[\"node.kubernetes.io/instance-type\"])\n\t\tlog.Printf(\"No node type specified. Using the node type %s in the node groups.\", *nodeType)\n\t}\n\tfor _, node := range nodes.Items {\n\t\tif node.Labels[\"node.kubernetes.io/instance-type\"] != *nodeType {\n\t\t\tcontinue\n\t\t}\n\t\tneuron, err := e2e.GetNonZeroResourceCapacity(&node, \"aws.amazon.com/neuron\")\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ttotalNeuronCount += neuron\n\n\t\t// Check for NeuronCore capacity\n\t\tneuronCore, err := e2e.GetNonZeroResourceCapacity(&node, \"aws.amazon.com/neuroncore\")\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ttotalNeuronCoreCount += neuronCore\n\n\t\t// Check for EFA capacity\n\t\tif *efaEnabled {\n\t\t\tefa, err := e2e.GetNonZeroResourceCapacity(&node, \"vpc.amazonaws.com/efa\")\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\ttotalEfaCount += efa\n\t\t}\n\t\tnodeCount += 1\n\t}\n\n\t// Update global capacities\n\tif nodeCount > 0 {\n\t\tneuronPerNode = totalNeuronCount / nodeCount\n\t\tneuronCorePerNode = totalNeuronCoreCount / nodeCount\n\t\tefaPerNode = totalEfaCount / nodeCount\n\t} else {\n\t\treturn nil, fmt.Errorf(\"no nodes of type %q found\", *nodeType)\n\t}\n\n\tlog.Printf(\"[INFO] Total Nodes: %d\", nodeCount)\n\tlog.Printf(\"[INFO] Total Neuron Count: %d, Neuron Per Node: %d\", totalNeuronCount, neuronPerNode)\n\tlog.Printf(\"[INFO] Total Neuron Core Count: %d, Neuron Core Per Node: %d\", totalNeuronCoreCount, neuronCorePerNode)\n\tlog.Printf(\"[INFO] Total EFA Count: %d, EFA Per Node: %d\", totalEfaCount, efaPerNode)\n\n\treturn ctx, nil\n}\n\nfunc TestMain(m *testing.M) {\n\tnodeType = flag.String(\"nodeType\", \"\", \"node type for the tests\")\n\tefaEnabled = flag.Bool(\"efaEnabled\", false, \"enable efa tests\")\n\tneuronTestImage = flag.String(\"neuronTestImage\", \"\", \"image for neuron single node test\")\n\tinstallDevicePlugin = flag.Bool(\"installDevicePlugin\", true, \"install neuron device plugin\")\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\ttestenv = env.NewWithConfig(cfg)\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = testenv.WithContext(ctx)\n\n\tdeploymentManifests := [][]byte{\n\t\tmanifests.MpiOperatorManifest,\n\t}\n\tsetUpFunctions := []env.Func{\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\terr := fwext.ApplyManifests(config.Client().RESTConfig(), deploymentManifests...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t\tdeployMPIOperator,\n\t}\n\n\tif *installDevicePlugin {\n\t\tdeploymentManifests = append(deploymentManifests, manifests.NeuronDevicePluginManifest, manifests.NeuronDevicePluginRbacManifest)\n\t\tsetUpFunctions = append(setUpFunctions, deployNeuronDevicePlugin)\n\t}\n\n\tif *efaEnabled {\n\t\tsetUpFunctions = append(setUpFunctions, deployEFAPlugin)\n\t}\n\n\tsetUpFunctions = append(setUpFunctions, checkNodeTypes)\n\ttestenv.Setup(setUpFunctions...)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\terr := fwext.DeleteManifests(cfg.Client().RESTConfig(), manifests.EfaDevicePluginManifest)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\tslices.Reverse(deploymentManifests)\n\t\t\terr = fwext.DeleteManifests(config.Client().RESTConfig(), deploymentManifests...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/neuron/manifests/multi-node-test-neuron.yaml", "content": "apiVersion: kubeflow.org/v2beta1\nkind: MPIJob\nmetadata:\n name: multi-node-nccom-test\nspec:\n slotsPerWorker: {{.NeuronPerNode}}\n runPolicy:\n backoffLimit: 20\n cleanPodPolicy: Running\n mpiReplicaSpecs:\n Launcher:\n replicas: 1\n template:\n spec:\n restartPolicy: OnFailure\n containers:\n - image: {{.NeuronTestImage}}\n imagePullPolicy: Always\n name: nccom-test-launcher\n env:\n - name: POD_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n command:\n - /bin/bash\n args:\n - -c\n - |\n WORKER_IPS=()\n for i in $(seq 0 $(({{.WorkerNodeCount}} - 1))); do\n WORKER_IP=$(getent hosts multi-node-nccom-test-worker-$i.multi-node-nccom-test | awk '{print $1}')\n WORKER_IPS+=(\"$WORKER_IP\")\n done\n\n export CCOM_SOCKET_IFNAME=eth0\n export NEURON_RT_ROOT_COMM_ID=${WORKER_IPS[0]}:63182\n nccom-test -r $(({{.NeuronCorePerNode}}*{{.WorkerNodeCount}})) -N {{.WorkerNodeCount}} -b \"8\" -e \"2G\" -f \"2\" -n \"5\" -w \"5\" -d \"fp32\" allr --hosts ${WORKER_IPS[*]} --data-collector-host $POD_IP --data-collector-port 60006 --debug\n Worker:\n replicas: {{.WorkerNodeCount}}\n template:\n spec:\n securityContext:\n runAsUser: 1000\n runAsGroup: 2000\n fsGroup: 3000\n containers:\n - image: {{.NeuronTestImage}}\n name: nccom-test-worker\n command: [\"/bin/bash\"]\n args: [\"-c\", \"echo password | sudo -S /usr/sbin/sshd -D\"]\n imagePullPolicy: Always\n resources:\n limits:\n aws.amazon.com/neuron: {{.NeuronPerNode}}\n aws.amazon.com/neuroncore: {{.NeuronCorePerNode}}\n vpc.amazonaws.com/efa: {{.EfaInterfacePerNode}}\n requests:\n aws.amazon.com/neuron: {{.NeuronPerNode}}\n aws.amazon.com/neuroncore: {{.NeuronCorePerNode}}\n vpc.amazonaws.com/efa: {{.EfaInterfacePerNode}}" }, { "path": "test/cases/neuron/manifests/single-node-test-neuronx.yaml", "content": "kind: Job\napiVersion: batch/v1\nmetadata:\n name: neuronx-single-node\n labels:\n app: neuronx-single-node\nspec:\n template:\n metadata:\n labels:\n app: neuronx-single-node\n spec:\n containers:\n - name: neuronx-single-node-test\n image: {{.NeuronTestImage}}\n command:\n - /bin/bash\n - ./tests/singleNodeTest.sh\n imagePullPolicy: Always\n resources:\n limits:\n cpu: \"4\"\n memory: 4Gi\n aws.amazon.com/neuron: \"1\"\n requests:\n cpu: \"1\"\n memory: 1Gi\n aws.amazon.com/neuron: \"1\"\n restartPolicy: Never\n securityContext:\n runAsUser: 1000\n runAsGroup: 2000\n fsGroup: 3000\n backoffLimit: 4\n" }, { "path": "test/cases/neuron/neuron_test.go", "content": "//go:build e2e\n\npackage neuron\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/internal/e2e/mpijobs\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\nvar (\n\t//go:embed manifests/single-node-test-neuronx.yaml\n\tneuronSingleNodeManifest []byte\n\t//go:embed manifests/multi-node-test-neuron.yaml\n\tneuronMultiNodeManifest []byte\n\trenderedNeuronSingleNodeManifest []byte\n\trenderedNeuronMultiNodeManifest []byte\n)\n\ntype neuronSingleNodeManifestTplVars struct {\n\tNeuronTestImage string\n}\n\ntype neuronMultiNodeTestManifestTplVars struct {\n\tWorkerNodeCount int\n\tWorkerNodeNeuronCount int\n\tNeuronPerNode int\n\tNeuronCorePerNode int\n\tNeuronTestImage string\n\tEfaInterfacePerNode int\n}\n\nfunc TestNeuronNodes(t *testing.T) {\n\tsingleNode := features.New(\"single-node\").\n\t\tWithLabel(\"suite\", \"neuron\").\n\t\tWithLabel(\"hardware\", \"neuron\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tif *neuronTestImage == \"\" {\n\t\t\t\tt.Fatal(fmt.Errorf(\"neuronTestImage must be set to run neuron single node test, use https://github.com/aws/aws-k8s-tester/blob/main/test/images/neuron/Dockerfile to build the image and -neuronTestImage to set the image url\"))\n\t\t\t}\n\t\t\tvar err error\n\t\t\trenderedNeuronSingleNodeManifest, err = fwext.RenderManifests(neuronSingleNodeManifest, neuronSingleNodeManifestTplVars{\n\t\t\t\tNeuronTestImage: *neuronTestImage,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tt.Log(\"Applying single node manifest\")\n\t\t\terr = fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedNeuronSingleNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tt.Log(\"Manifest applied successfully\")\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Single node test Job succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tjob := &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"neuronx-single-node\", Namespace: \"default\"},\n\t\t\t}\n\t\t\tt.Log(\"Waiting for single node job to complete\")\n\t\t\terr := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).JobSucceeded(job),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(time.Minute*20),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog, err := fwext.GetJobLogs(cfg.Client().RESTConfig(), &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"neuronx-single-node\", Namespace: \"default\"},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t} else {\n\t\t\t\tt.Log(\"Test log for neuronx-single-node:\")\n\t\t\t\tt.Log(log)\n\t\t\t}\n\t\t\terr = fwext.DeleteManifests(cfg.Client().RESTConfig(), renderedNeuronSingleNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\tmultiNode := features.New(\"multi-node\").\n\t\tWithLabel(\"suite\", \"neuron\").\n\t\tWithLabel(\"hardware\", \"neuron\").\n\t\tWithLabel(\"hardware\", \"efa\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tif *neuronTestImage == \"\" {\n\t\t\t\tt.Fatal(fmt.Errorf(\"neuronTestImage must be set to run unit test, use https://github.com/aws/aws-k8s-tester/blob/main/test/images/neuron/Dockerfile to build the image and -neuronTestImage to set the image url\"))\n\t\t\t}\n\t\t\trenderedNeuronMultiNodeManifest, err := fwext.RenderManifests(neuronMultiNodeManifest, neuronMultiNodeTestManifestTplVars{\n\t\t\t\t// one of the nodes will be used for the master pod\n\t\t\t\tWorkerNodeCount: nodeCount,\n\t\t\t\tWorkerNodeNeuronCount: nodeCount * neuronPerNode,\n\t\t\t\tNeuronPerNode: neuronPerNode,\n\t\t\t\tNeuronCorePerNode: neuronCorePerNode,\n\t\t\t\tNeuronTestImage: *neuronTestImage,\n\t\t\t\tEfaInterfacePerNode: efaPerNode,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tt.Log(\"Applying multi node manifest\")\n\t\t\terr = fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedNeuronMultiNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tt.Log(\"Applied manifest successfully\")\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"NCCOM test succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tmpiJob := mpijobs.NewUnstructured(\"multi-node-nccom-test\", \"default\")\n\t\t\tctx = context.WithValue(ctx, \"mpiJob\", mpiJob)\n\t\t\tt.Log(\"Waiting for MPIJob to complete\")\n\t\t\terr := wait.For(conditions.New(cfg.Client().Resources()).ResourceMatch(mpiJob, mpijobs.MPIJobSucceeded),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(time.Minute*30),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tlog, err := fwext.GetJobLogs(cfg.Client().RESTConfig(), mpiJob)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tt.Log(\"Test log for multi-node-nccom-test:\")\n\t\t\tt.Log(log)\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\terr := fwext.DeleteManifests(cfg.Client().RESTConfig(), renderedNeuronMultiNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, singleNode, multiNode)\n}\n" }, { "path": "test/cases/neuron-dra/main_test.go", "content": "//go:build e2e\n\npackage neuron_dra\n\nimport (\n\t\"context\"\n\t\"embed\"\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"path/filepath\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\t\"golang.org/x/sync/errgroup\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\n//go:embed rcts\nvar rctsFS embed.FS\n\nvar (\n\ttestenv env.Environment\n\tclientset kubernetes.Interface\n\tnodeType *string\n\trdmaDeviceDraDriverImage *string\n\tacceleratorDraDriverImage *string\n\tcontainerTestImage *string\n\tnodeCount int\n)\n\n// supportedRdmaTypes lists the recognized RDMA device types.\nvar supportedRdmaTypes = []string{\"efa\"}\n\nfunc validateConfig() error {\n\tif err := common.ValidateRequiredFlags(map[string]string{\n\t\t\"rdmaDeviceDraDriverImage\": *rdmaDeviceDraDriverImage,\n\t\t\"containerTestImage\": *containerTestImage,\n\t\t\"nodeType\": *nodeType,\n\t}); err != nil {\n\t\treturn err\n\t}\n\t// Validate that nodeType maps to a known topology (and thus a known RDMA type)\n\ttopo, err := GetTopologyForNodeType(*nodeType)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"invalid -nodeType: %w\", err)\n\t}\n\tif !slices.Contains(supportedRdmaTypes, topo.RdmaType) {\n\t\treturn fmt.Errorf(\"instance family %q has unsupported RDMA type %q; supported: %v\", topo.Family, topo.RdmaType, supportedRdmaTypes)\n\t}\n\t// Verify helm is available on the PATH.\n\tif _, err := exec.LookPath(\"helm\"); err != nil {\n\t\treturn fmt.Errorf(\"helm is required but not found on PATH: %w\", err)\n\t}\n\treturn nil\n}\n\nconst (\n\tneuronHelmReleaseName = \"neuron-helm-chart\"\n\tneuronHelmChartOCI = \"oci://public.ecr.aws/neuron/neuron-helm-chart\"\n\tneuronDRANamespace = \"neuron-dra-driver\"\n)\n\n// installNeuronDRADriverHelm installs the Neuron DRA driver via the public Helm chart.\n// If acceleratorDraDriverImage is non-empty, it splits on the last \":\" to extract\n// repository and tag and passes them as --set overrides.\nfunc installNeuronDRADriverHelm(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\targs := []string{\n\t\t\"upgrade\", \"--install\", neuronHelmReleaseName, neuronHelmChartOCI,\n\t\t\"--namespace\", neuronDRANamespace,\n\t\t\"--create-namespace\",\n\t\t\"--set\", \"devicePlugin.enabled=false\",\n\t\t\"--set\", \"npd.enabled=false\",\n\t\t\"--set\", \"draDriver.enabled=true\",\n\t\t\"--wait\",\n\t\t\"--timeout\", \"5m\",\n\t}\n\tif *acceleratorDraDriverImage != \"\" {\n\t\trepo, tag := common.SplitImageRepoTag(*acceleratorDraDriverImage)\n\t\targs = append(args,\n\t\t\t\"--set\", fmt.Sprintf(\"draDriver.image.repository=%s\", repo),\n\t\t\t\"--set\", fmt.Sprintf(\"draDriver.image.tag=%s\", tag),\n\t\t)\n\t}\n\tlog.Printf(\"[INFO] Installing Neuron DRA driver via Helm: helm %s\", strings.Join(args, \" \"))\n\tcmd := exec.CommandContext(ctx, \"helm\", args...)\n\tcmd.Stdout = os.Stdout\n\tcmd.Stderr = os.Stderr\n\tif err := cmd.Run(); err != nil {\n\t\treturn ctx, fmt.Errorf(\"helm install neuron-dra-driver failed: %w\", err)\n\t}\n\tlog.Println(\"Neuron DRA driver Helm release installed successfully.\")\n\treturn ctx, nil\n}\n\n// uninstallNeuronDRADriverHelm uninstalls the Neuron DRA driver Helm release.\nfunc uninstallNeuronDRADriverHelm(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\targs := []string{\n\t\t\"uninstall\", neuronHelmReleaseName,\n\t\t\"--namespace\", neuronDRANamespace,\n\t}\n\tlog.Printf(\"[INFO] Uninstalling Neuron DRA driver Helm release: helm %s\", strings.Join(args, \" \"))\n\tcmd := exec.CommandContext(ctx, \"helm\", args...)\n\tcmd.Stdout = os.Stdout\n\tcmd.Stderr = os.Stderr\n\tif err := cmd.Run(); err != nil {\n\t\tlog.Printf(\"[WARN] helm uninstall neuron-dra-driver failed (may already be removed): %v\", err)\n\t}\n\treturn ctx, nil\n}\n\nfunc deployNeuronDRADriver(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tds := appsv1.DaemonSet{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"neuron-dra-driver-kubelet-plugin\", Namespace: neuronDRANamespace},\n\t}\n\terr := wait.For(\n\t\tfwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&ds),\n\t\twait.WithTimeout(5*time.Minute),\n\t\twait.WithContext(ctx),\n\t)\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"neuron-dra-driver daemonset is not ready: %w\", err)\n\t}\n\tlog.Println(\"neuron-dra-driver daemonset is ready.\")\n\treturn ctx, nil\n}\n\nfunc TestMain(m *testing.M) {\n\tnodeType = flag.String(\"nodeType\", \"\", \"instance type for the cluster (e.g. trn1.32xlarge)\")\n\trdmaDeviceDraDriverImage = flag.String(\"rdmaDeviceDraDriverImage\", \"\", \"container image for the dranet DRA driver\")\n\tacceleratorDraDriverImage = flag.String(\"acceleratorDraDriverImage\", \"\", \"container image for the Neuron DRA driver\")\n\tcontainerTestImage = flag.String(\"containerTestImage\", \"\", \"container image for the nccom test workload\")\n\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\n\tif err := validateConfig(); err != nil {\n\t\tlog.Fatalf(\"invalid configuration: %v\", err)\n\t}\n\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = env.NewWithConfig(cfg).WithContext(ctx)\n\n\t// Build the manifest list and setup functions dynamically.\n\t// Resolve topology to determine RDMA type from nodeType.\n\ttopo, err := GetTopologyForNodeType(*nodeType)\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to resolve topology: %v\", err)\n\t}\n\n\tmanifestsList := [][]byte{\n\t\tmanifests.MpiOperatorManifest,\n\t}\n\tsetUpFunctions := []env.Func{\n\t\t// Run independent setup steps concurrently.\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tvar mu sync.Mutex\n\t\t\tg, gctx := errgroup.WithContext(ctx)\n\n\t\t\t// Deploy MPI operator.\n\t\t\tg.Go(func() error {\n\t\t\t\treturn common.DeployMPIOperator(gctx, config)\n\t\t\t})\n\n\t\t\t// Deploy dranet and RCTs based on topology's RDMA type.\n\t\t\tif topo.RdmaType == \"efa\" {\n\t\t\t\trctManifests, err := common.LoadRCTManifests(rctsFS, filepath.Join(\"rcts\", topo.RCTSubDir))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn ctx, fmt.Errorf(\"failed to load RCT manifests: %w\", err)\n\t\t\t\t}\n\t\t\t\tmu.Lock()\n\t\t\t\tmanifestsList = append(manifestsList, rctManifests...)\n\t\t\t\tmu.Unlock()\n\n\t\t\t\tg.Go(func() error {\n\t\t\t\t\trenderedDranet, err := common.DeployDranet(gctx, config, *rdmaDeviceDraDriverImage)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tmu.Lock()\n\t\t\t\t\tmanifestsList = append(manifestsList, renderedDranet)\n\t\t\t\t\tmu.Unlock()\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\n\t\t\t\tg.Go(func() error {\n\t\t\t\t\treturn fwext.ApplyManifests(config.Client().RESTConfig(), rctManifests...)\n\t\t\t\t})\n\t\t\t}\n\n\t\t\t// Install Neuron DRA driver via Helm chart.\n\t\t\tg.Go(func() error {\n\t\t\t\t_, err := installNeuronDRADriverHelm(gctx, config)\n\t\t\t\treturn err\n\t\t\t})\n\n\t\t\tif err := g.Wait(); err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t\tdeployNeuronDRADriver,\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tvar err error\n\t\t\tclientset, err = kubernetes.NewForConfig(config.Client().RESTConfig())\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\tnodeCount, err = common.CountNodesByType(ctx, clientset, *nodeType)\n\t\t\treturn ctx, err\n\t\t},\n\t}\n\ttestenv.Setup(setUpFunctions...)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\t// Uninstall Neuron DRA driver Helm release first.\n\t\t\tctx, _ = uninstallNeuronDRADriverHelm(ctx, config)\n\t\t\t// Delete remaining manifests in reverse order.\n\t\t\tslices.Reverse(manifestsList)\n\t\t\tif err := fwext.DeleteManifests(config.Client().RESTConfig(), manifestsList...); err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to delete manifests: %w\", err)\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/neuron-dra/neuron_dra_test.go", "content": "//go:build e2e\n\npackage neuron_dra\n\nimport (\n\t\"embed\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n)\n\n//go:embed testcases\nvar embeddedTestCases embed.FS\n\nfunc TestNeuronDRAMultiNode(t *testing.T) {\n\ttopo, err := GetTopologyForNodeType(*nodeType)\n\tif err != nil {\n\t\tt.Fatalf(\"resolving topology for %s: %v\", *nodeType, err)\n\t}\n\n\trctDir := filepath.Join(\"rcts\", topo.RCTSubDir)\n\trctIndex, err := common.LoadRCTIndex(rctsFS, rctDir)\n\tif err != nil {\n\t\tt.Fatalf(\"loading RCT index from %s: %v\", rctDir, err)\n\t}\n\n\ttcDir := filepath.Join(\"testcases\", topo.TestCaseSubDir)\n\n\tfeatureList, err := common.DiscoverAndBuildFeatures(\n\t\tembeddedTestCases,\n\t\ttcDir,\n\t\trctIndex,\n\t\t\"neuron-dra\",\n\t\t\"multi-node-nccom-test\",\n\t\tnodeCount,\n\t\tfunc(tc *common.TestCaseSpec, rctIndex map[string]*common.ResourceClaimTemplateSpec) ([]byte, error) {\n\t\t\tparams, err := ComputeMPIJobParamsFromTestCase(tc, rctIndex, topo, nodeCount, *containerTestImage)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn RenderMPIJobYAML(*params)\n\t\t},\n\t\tclientset,\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"discovering and building features: %v\", err)\n\t}\n\n\tif len(featureList) == 0 {\n\t\tt.Logf(\"no test cases found under %s, skipping\", tcDir)\n\t\treturn\n\t}\n\n\ttestenv.Test(t, featureList...)\n}\n" }, { "path": "test/cases/neuron-dra/rcts/trn1/rct-2-efas-4-neurons-wrong-match.yaml", "content": "apiVersion: resource.k8s.io/v1beta1\nkind: ResourceClaimTemplate\nmetadata:\n namespace: default\n name: rct-2-efas-4-neurons-wrong-match\nspec:\n spec:\n devices:\n requests:\n - name: 4-neurons\n deviceClassName: neuron.aws.com\n allocationMode: ExactCount\n count: 4\n - name: 2-efas\n deviceClassName: efa.networking.k8s.aws\n allocationMode: ExactCount\n count: 2\n constraints:\n - requests: [\"4-neurons\", \"2-efas\"]\n matchAttribute: \"resource.aws.com/devicegroup1_id\"\n" }, { "path": "test/cases/neuron-dra/rcts/trn1/rct-all-efas-all-neurons.yaml", "content": "apiVersion: resource.k8s.io/v1beta1\nkind: ResourceClaimTemplate\nmetadata:\n namespace: default\n name: rct-all-efas-all-neurons\nspec:\n spec:\n devices:\n requests:\n - name: all-neurons\n deviceClassName: neuron.aws.com\n allocationMode: All\n - name: all-efas\n deviceClassName: efa.networking.k8s.aws\n allocationMode: All\n" }, { "path": "test/cases/neuron-dra/templates/nccom-test-mpijob.yaml.tmpl", "content": "apiVersion: kubeflow.org/v2beta1\nkind: MPIJob\nmetadata:\n name: multi-node-nccom-test\nspec:\n slotsPerWorker: {{.SlotsPerWorker}}\n runPolicy:\n backoffLimit: 20\n cleanPodPolicy: Running\n mpiReplicaSpecs:\n Launcher:\n replicas: 1\n template:\n spec:\n restartPolicy: OnFailure\n containers:\n - name: nccom-test-launcher\n image: {{.ContainerTestImage}}\n imagePullPolicy: Always\n env:\n - name: POD_IP\n valueFrom:\n fieldRef:\n fieldPath: status.podIP\n command:\n - /bin/bash\n - -lc\n args:\n - |\n set -euo pipefail\n\n WORKER_IPS=()\n for i in $(seq 0 $(({{.WorkerReplicas}} - 1))); do\n WORKER_IP=$(getent hosts multi-node-nccom-test-worker-$i.multi-node-nccom-test | awk '{print $1}')\n WORKER_IPS+=(\"$WORKER_IP\")\n done\n\n export NCCOM_SOCKET_IFNAME=eth0\n export NEURON_RT_ROOT_COMM_ID=${WORKER_IPS[0]}:63182\n\n nccom-test \\\n -r {{.TotalRanks}} \\\n -N {{.WorkerReplicas}} \\\n -b 8 \\\n -e 2G \\\n -f 2 \\\n -n 5 \\\n -w 5 \\\n -d fp32 \\\n allr \\\n --hosts ${WORKER_IPS[*]} \\\n --data-collector-host \"${POD_IP}\" \\\n --data-collector-port 60006 \\\n --debug\n\n Worker:\n replicas: {{.WorkerReplicas}}\n template:\n spec:\n restartPolicy: OnFailure\n securityContext:\n runAsUser: 0\n containers:\n - name: nccom-test-worker\n image: {{.ContainerTestImage}}\n imagePullPolicy: Always\n securityContext:\n capabilities:\n add: [\"NET_ADMIN\"]\n env:\n - name: FI_EFA_USE_DEVICE_RDMA\n value: \"1\"\n command:\n - /bin/bash\n - -lc\n args:\n - |\n set -euo pipefail\n\n MY_IP=$(hostname -i)\n ip addr add ${MY_IP}/16 dev eth0 label eth0:ccom\n ip route del 192.168.0.0/16 dev eth0 2>/dev/null || true\n /usr/sbin/sshd -D\n resources:\n claims:\n{{- range .ResourceClaims}}\n - name: {{.Name}}\n{{- end}}\n resourceClaims:\n{{- range .ResourceClaims}}\n - name: {{.Name}}\n resourceClaimTemplateName: {{.TemplateName}}\n{{- end}}\n" }, { "path": "test/cases/neuron-dra/testcases/trn1/2-efas-4-neurons-wrong-match.yaml", "content": "expectFailure: true\nresourceClaims:\n- name: 2-efas-4-neurons-wrong-match\n resourceClaimTemplateName: rct-2-efas-4-neurons-wrong-match\n" }, { "path": "test/cases/neuron-dra/testcases/trn1/all-efas-all-neurons.yaml", "content": "resourceClaims:\n- name: all-efas-all-neurons\n resourceClaimTemplateName: rct-all-efas-all-neurons\n" }, { "path": "test/cases/neuron-dra/topology.go", "content": "package neuron_dra\n\nimport (\n\t\"bytes\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"strings\"\n\t\"text/template\"\n\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n)\n\n//go:embed templates/nccom-test-mpijob.yaml.tmpl\nvar mpijobTemplate string\n\n// ---------------------------------------------------------------------------\n// Instance topology\n// ---------------------------------------------------------------------------\n\n// InstanceTopology describes the Neuron/EFA hardware topology for an instance family.\ntype InstanceTopology struct {\n\tFamily string\n\tNeuronCoresPerDevice int\n\tAllNeuronCount int\n\tRdmaType string // RDMA device type (e.g. \"efa\")\n\tRCTSubDir string // subdirectory under rcts/\n\tTestCaseSubDir string // subdirectory under testcases/\n}\n\nvar instanceTopologies = map[string]InstanceTopology{\n\t\"trn1\": {\n\t\tFamily: \"trn1\",\n\t\tNeuronCoresPerDevice: 2,\n\t\tAllNeuronCount: 16,\n\t\tRdmaType: \"efa\",\n\t\tRCTSubDir: \"trn1\",\n\t\tTestCaseSubDir: \"trn1\",\n\t},\n}\n\n// GetTopologyForNodeType returns the InstanceTopology for a given node type\n// (e.g. \"trn1.32xlarge\"). It extracts the family prefix before the first \".\"\n// and looks it up in the registry.\nfunc GetTopologyForNodeType(nodeType string) (*InstanceTopology, error) {\n\tfamily := common.ExtractFamily(nodeType)\n\ttopo, ok := instanceTopologies[family]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"unsupported instance family %q (from %q); supported: %s\",\n\t\t\tfamily, nodeType, supportedFamilies())\n\t}\n\treturn &topo, nil\n}\n\nfunc supportedFamilies() string {\n\tfamilies := make([]string, 0, len(instanceTopologies))\n\tfor k := range instanceTopologies {\n\t\tfamilies = append(families, k)\n\t}\n\treturn strings.Join(families, \", \")\n}\n\n// ---------------------------------------------------------------------------\n// MPIJob rendering\n// ---------------------------------------------------------------------------\n\n// MPIJobParams holds all template parameters for rendering the MPIJob YAML.\ntype MPIJobParams struct {\n\tSlotsPerWorker int\n\tTotalRanks int\n\tWorkerReplicas int\n\tContainerTestImage string\n\tResourceClaims []common.ResourceClaimRef\n}\n\n// RenderMPIJobYAML renders the embedded MPIJob Go template with the given params\n// and returns the resulting YAML bytes.\nfunc RenderMPIJobYAML(params MPIJobParams) ([]byte, error) {\n\ttmpl, err := template.New(\"mpijob\").Parse(mpijobTemplate)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"parsing MPIJob template: %w\", err)\n\t}\n\tvar buf bytes.Buffer\n\tif err := tmpl.Execute(&buf, params); err != nil {\n\t\treturn nil, fmt.Errorf(\"rendering MPIJob template: %w\", err)\n\t}\n\treturn buf.Bytes(), nil\n}\n\n// ---------------------------------------------------------------------------\n// Neuron-specific helpers\n// ---------------------------------------------------------------------------\n\n// getNeuronCount returns the neuron device count from an RCT.\n// For AllocationMode \"All\" it returns the topology's AllNeuronCount;\n// otherwise it returns the explicit Count from the neuron request.\nfunc getNeuronCount(rct *common.ResourceClaimTemplateSpec, topo *InstanceTopology) int {\n\tfor _, req := range rct.Spec.Spec.Devices.Requests {\n\t\tif req.DeviceClassName != \"neuron.aws.com\" {\n\t\t\tcontinue\n\t\t}\n\t\tif req.AllocationMode == \"All\" {\n\t\t\treturn topo.AllNeuronCount\n\t\t}\n\t\treturn req.Count\n\t}\n\treturn 0\n}\n\n// ComputeMPIJobParamsFromTestCase computes MPIJob parameters from a test case spec.\n// It resolves each claim's resourceClaimTemplateName against the RCT index to\n// get the neuron count, then calculates SlotsPerWorker and TotalRanks.\nfunc ComputeMPIJobParamsFromTestCase(tc *common.TestCaseSpec, rctIndex map[string]*common.ResourceClaimTemplateSpec, topo *InstanceTopology, workerReplicas int, containerTestImage string) (*MPIJobParams, error) {\n\tif topo == nil {\n\t\treturn nil, fmt.Errorf(\"instance topology is required\")\n\t}\n\tif workerReplicas <= 0 {\n\t\treturn nil, fmt.Errorf(\"workerReplicas must be positive, got %d\", workerReplicas)\n\t}\n\tif containerTestImage == \"\" {\n\t\treturn nil, fmt.Errorf(\"containerTestImage is required\")\n\t}\n\n\ttotalNeurons := 0\n\tvar claims []common.ResourceClaimRef\n\n\tfor _, tcClaim := range tc.ResourceClaims {\n\t\trct, ok := rctIndex[tcClaim.ResourceClaimTemplateName]\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"resource claim template %q not found in RCT index\", tcClaim.ResourceClaimTemplateName)\n\t\t}\n\n\t\ttotalNeurons += getNeuronCount(rct, topo)\n\n\t\tclaims = append(claims, common.ResourceClaimRef{\n\t\t\tName: tcClaim.Name,\n\t\t\tTemplateName: tcClaim.ResourceClaimTemplateName,\n\t\t})\n\t}\n\n\tslotsPerWorker := totalNeurons * topo.NeuronCoresPerDevice\n\ttotalRanks := slotsPerWorker * workerReplicas\n\n\treturn &MPIJobParams{\n\t\tSlotsPerWorker: slotsPerWorker,\n\t\tTotalRanks: totalRanks,\n\t\tWorkerReplicas: workerReplicas,\n\t\tContainerTestImage: containerTestImage,\n\t\tResourceClaims: claims,\n\t}, nil\n}\n" }, { "path": "test/cases/neuron-inference/bert_inference_test.go", "content": "//go:build e2e\n\npackage inference\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\n//go:embed manifests/neuron-bert-inference.yaml\nvar neuronBertInferenceManifest []byte\n\nvar renderedManifest []byte\n\nfunc TestNeuronInference(t *testing.T) {\n\tif *bertInferenceImage == \"\" {\n\t\tt.Fatal(\"bertInferenceImage must be set to run the test\")\n\t}\n\n\tlog.Printf(\"[INFO] Using nodeType=%s, inferenceMode=%s\", *nodeType, *inferenceMode)\n\tlog.Printf(\"[INFO] Discovered neuronPerNode=%d, neuronCorePerNode=%d\", neuronPerNode, neuronCorePerNode)\n\n\trenderVars := map[string]string{\n\t\t\"BertInferenceImage\": *bertInferenceImage,\n\t\t\"NodeType\": *nodeType, // e.g. \"inf2.xlarge\"\n\t\t\"InferenceMode\": *inferenceMode, // \"throughput\" or \"latency\"\n\t\t\"NeuronPerNode\": fmt.Sprintf(\"%d\", neuronPerNode),\n\t\t\"NeuronCorePerNode\": fmt.Sprintf(\"%d\", neuronCorePerNode),\n\t}\n\n\t// Render the manifest\n\trenderedManifest, err := fwext.RenderManifests(neuronBertInferenceManifest, renderVars)\n\tif err != nil {\n\t\tt.Fatalf(\"[ERROR] Failed to render Neuron inference manifest: %v\", err)\n\t}\n\n\tfeature := features.New(\"neuron-inference\").\n\t\tWithLabel(\"suite\", \"neuron\").\n\t\tWithLabel(\"hardware\", \"neuron\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog.Println(\"[INFO] Applying rendered Neuron inference manifest.\")\n\t\t\terr := fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"[ERROR] Failed to apply Neuron inference manifest: %v\", err)\n\t\t\t}\n\t\t\tlog.Println(\"[INFO] Successfully applied Neuron inference manifest.\")\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"BERT inference Job succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog.Println(\"[INFO] Checking 'neuron-inference' job completion...\")\n\n\t\t\tjob := &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"neuron-inference\", Namespace: \"default\"},\n\t\t\t}\n\t\t\tif err := wait.For(\n\t\t\t\tfwext.NewConditionExtension(cfg.Client().Resources()).JobSucceeded(job),\n\t\t\t\twait.WithTimeout(60*time.Minute),\n\t\t\t); err != nil {\n\t\t\t\tlog.Println(\"[ERROR] Neuron inference job failed. Gathering logs...\")\n\t\t\t\tif err := printJobLogs(ctx, cfg, \"default\", \"neuron-inference\"); err != nil {\n\t\t\t\t\tt.Logf(\"[WARNING] Failed to retrieve neuron-inference job logs: %v\", err)\n\t\t\t\t}\n\t\t\t\tt.Fatalf(\"[ERROR] Neuron inference job did not succeed: %v\", err)\n\t\t\t}\n\n\t\t\tlog.Println(\"[INFO] Neuron inference job succeeded. Gathering logs...\")\n\t\t\tapplyTime := ctx.Value(\"applyTime\")\n\t\t\tif applyTime != nil {\n\t\t\t\tif start, ok := applyTime.(time.Time); ok {\n\t\t\t\t\tduration := time.Since(start)\n\t\t\t\t\tlog.Printf(\"[INFO] Neuron inference job completed in %s\", duration)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif err := printJobLogs(ctx, cfg, \"default\", \"neuron-inference\"); err != nil {\n\t\t\t\tt.Logf(\"[WARNING] Failed to retrieve neuron-inference job logs: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog.Println(\"[INFO] Cleaning up neuron-inference job resources...\")\n\t\t\tif err := fwext.DeleteManifests(cfg.Client().RESTConfig(), renderedManifest); err != nil {\n\t\t\t\tt.Fatalf(\"[ERROR] Failed to delete inference job resources: %v\", err)\n\t\t\t}\n\t\t\tlog.Println(\"[INFO] Inference job cleanup complete.\")\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, feature)\n}\n\nfunc printJobLogs(ctx context.Context, cfg *envconf.Config, namespace, jobName string) error {\n\tcs, err := getClientset(cfg.Client().RESTConfig())\n\tif err != nil {\n\t\treturn fmt.Errorf(\"[ERROR] failed to create kubernetes client: %w\", err)\n\t}\n\n\tpods, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{\n\t\tLabelSelector: fmt.Sprintf(\"job-name=%s\", jobName),\n\t})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"[ERROR] failed to list pods for job %s: %w\", jobName, err)\n\t}\n\tif len(pods.Items) == 0 {\n\t\treturn fmt.Errorf(\"[ERROR] no pods found for job %s\", jobName)\n\t}\n\n\tfor _, pod := range pods.Items {\n\t\tlog.Printf(\"[INFO] Pod %s is on node %s\", pod.Name, pod.Spec.NodeName)\n\t\tstream, err := cs.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{}).Stream(ctx)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"[ERROR] failed to get logs from pod %s: %w\", pod.Name, err)\n\t\t}\n\t\tdefer stream.Close()\n\n\t\tbuf := make([]byte, 4096)\n\t\tfor {\n\t\t\tn, readErr := stream.Read(buf)\n\t\t\tif n > 0 {\n\t\t\t\tlog.Printf(\"[INFO] Logs from Pod %s:\\n%s\", pod.Name, string(buf[:n]))\n\t\t\t}\n\t\t\tif readErr == io.EOF {\n\t\t\t\tlog.Printf(\"[INFO] Completed log stream for pod %s.\", pod.Name)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif readErr != nil {\n\t\t\t\treturn fmt.Errorf(\"[ERROR] reading logs from pod %s: %w\", pod.Name, readErr)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc getClientset(restConfig *rest.Config) (*kubernetes.Clientset, error) {\n\tcs, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"cannot create kubernetes clientset: %w\", err)\n\t}\n\treturn cs, nil\n}\n" }, { "path": "test/cases/neuron-inference/main_test.go", "content": "//go:build e2e\n\npackage inference\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"slices\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nfunc TestMain(m *testing.M) {\n\n\tflag.Parse()\n\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"[ERROR] Failed to create test environment: %v\", err)\n\t}\n\ttestenv = env.NewWithConfig(cfg)\n\n\tdeploymentManifests := [][]byte{\n\t\tmanifests.NeuronDevicePluginRbacManifest,\n\t\tmanifests.NeuronDevicePluginManifest,\n\t}\n\n\t// Setup steps: apply the device plugin, wait for DS readiness, discover capacity\n\ttestenv.Setup(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Applying Neuron device plugin RBAC and Neuron device plugin manifests.\")\n\t\t\terr := fwext.ApplyManifests(config.Client().RESTConfig(), deploymentManifests...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to apply manifests: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"Successfully applied Neuron device plugin RBAC and Neuron device plugin manifests.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Waiting for Neuron Device Plugin daemonset to be ready.\")\n\t\t\tdaemonset := appsv1.DaemonSet{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"neuron-device-plugin-daemonset\", Namespace: \"kube-system\"},\n\t\t\t}\n\t\t\terr := wait.For(\n\t\t\t\tfwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&daemonset),\n\t\t\t\twait.WithTimeout(time.Minute*5),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"Neuron Device Plugin daemonset is not ready: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"Neuron Device Plugin daemonset is ready.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t\tdiscoverNeuronCoreCapacity,\n\t\tgetNodeCapacity,\n\t)\n\n\t// Finish steps: remove device plugin if desired\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"[INFO] Cleaning up Neuron device plugin.\")\n\t\t\tslices.Reverse(deploymentManifests)\n\t\t\tif err := fwext.DeleteManifests(config.Client().RESTConfig(), deploymentManifests...); err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to delete neuron device plugin: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"[INFO] Neuron device plugin cleanup complete.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\texitCode := testenv.Run(m)\n\tlog.Printf(\"[INFO] Test environment finished with exit code %d\", exitCode)\n\tos.Exit(exitCode)\n}\n\n// discoverNeuronCoreCapacity sets neuronPerNode and neuronCorePerNode by scanning the cluster\nfunc discoverNeuronCoreCapacity(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tlog.Println(\"[INFO] Discovering cluster's Neuron capacity...\")\n\n\t// Check Neuron devices\n\tlog.Println(\"[INFO] Checking Neuron device capacity on nodes\")\n\terr := wait.For(\n\t\tfwext.NewConditionExtension(config.Client().Resources()).AllNodesHaveNonZeroResourceCapacity(\"aws.amazon.com/neuron\"),\n\t\twait.WithTimeout(time.Second*60),\n\t\twait.WithInterval(time.Second*5),\n\t)\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to verify Neuron device capacity on nodes: %w\", err)\n\t}\n\tlog.Println(\"[INFO] Neuron devices check passed - all nodes have non-zero capacity\")\n\n\t// Check Neuron cores\n\tlog.Println(\"[INFO] Checking Neuron core capacity on nodes\")\n\terr = wait.For(\n\t\tfwext.NewConditionExtension(config.Client().Resources()).AllNodesHaveNonZeroResourceCapacity(\"aws.amazon.com/neuroncore\"),\n\t\twait.WithTimeout(time.Second*60),\n\t\twait.WithInterval(time.Second*5),\n\t)\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to verify Neuron core capacity on nodes: %w\", err)\n\t}\n\tlog.Println(\"[INFO] Neuron cores check passed - all nodes have non-zero capacity\")\n\n\tlog.Println(\"[INFO] Neuron capacity discovery complete.\")\n\treturn ctx, nil\n}\n\nfunc getNodeCapacity(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tcs, err := kubernetes.NewForConfig(config.Client().RESTConfig())\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to create kubernetes client: %w\", err)\n\t}\n\n\tnodes, err := cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to list nodes: %w\", err)\n\t}\n\tif len(nodes.Items) == 0 {\n\t\treturn ctx, fmt.Errorf(\"no nodes found in the cluster\")\n\t}\n\tvar totalNeuron, totalNeuronCore, nodeCount int\n\t// if nodeType not set, use the instance type discovered\n\tif *nodeType == \"\" {\n\t\t*nodeType = nodes.Items[0].Labels[\"node.kubernetes.io/instance-type\"]\n\t}\n\tfor _, node := range nodes.Items {\n\t\tinstanceType := node.Labels[\"node.kubernetes.io/instance-type\"]\n\t\tneuronCap, hasNeuron := node.Status.Capacity[\"aws.amazon.com/neuron\"]\n\t\tneuronCoreCap, hasNeuronCore := node.Status.Capacity[\"aws.amazon.com/neuroncore\"]\n\t\tif instanceType == *nodeType {\n\t\t\tnodeCount++\n\t\t\tif hasNeuron {\n\t\t\t\ttotalNeuron += int(neuronCap.Value())\n\t\t\t} else {\n\t\t\t\tlog.Printf(\"[WARN] Node %s (type=%s) lacks 'aws.amazon.com/neuron'.\", node.Name, instanceType)\n\t\t\t}\n\t\t\tif hasNeuronCore {\n\t\t\t\ttotalNeuronCore += int(neuronCoreCap.Value())\n\t\t\t} else {\n\t\t\t\tlog.Printf(\"[WARN] Node %s (type=%s) lacks 'aws.amazon.com/neuroncore'.\", node.Name, instanceType)\n\t\t\t}\n\t\t}\n\t}\n\tif nodeCount > 0 {\n\t\tneuronPerNode = totalNeuron / nodeCount\n\t\tneuronCorePerNode = totalNeuronCore / nodeCount\n\t} else {\n\t\treturn ctx, fmt.Errorf(\"no nodes with %s node type found in the cluster\", *nodeType)\n\t}\n\tlog.Printf(\"[INFO] Discovered neuronPerNode=%d, neuronCorePerNode=%d (across %d node(s))\", neuronPerNode, neuronCorePerNode, nodeCount)\n\treturn ctx, nil\n}\n" }, { "path": "test/cases/neuron-inference/manifests/neuron-bert-inference.yaml", "content": "apiVersion: batch/v1\nkind: Job\nmetadata:\n name: neuron-inference\nspec:\n backoffLimit: 4\n template:\n spec:\n restartPolicy: OnFailure\n volumes:\n - name: dshm\n emptyDir:\n medium: Memory\n containers:\n - name: neuron-inference\n image: {{.BertInferenceImage}}\n imagePullPolicy: Always\n command: [\"python\", \"/app/infer.py\"]\n env:\n - name: INFERENCE_MODE\n value: \"{{.InferenceMode}}\"\n volumeMounts:\n - mountPath: /dev/shm\n name: dshm\n resources:\n requests:\n aws.amazon.com/neuroncore: \"{{.NeuronCorePerNode}}\"\n limits:\n aws.amazon.com/neuroncore: \"{{.NeuronCorePerNode}}\"\n nodeSelector:\n node.kubernetes.io/instance-type: {{.NodeType}}\n" }, { "path": "test/cases/neuron-inference/vars.go", "content": "//go:build e2e\n\npackage inference\n\nimport (\n\t\"flag\"\n\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n)\n\n// Shared global variables\nvar (\n\t// The e2e-framework environment\n\ttestenv env.Environment\n\n\t// Passed in as flags\n\tbertInferenceImage *string\n\tnodeType *string\n\tinferenceMode *string\n\n\t// Discovered in main_test.go\n\tneuronPerNode int\n\tneuronCorePerNode int\n)\n\n// init() runs before TestMain and sets up the flags\nfunc init() {\n\tbertInferenceImage = flag.String(\"bertInferenceImage\", \"\",\n\t\t\"[REQUIRED] Docker image used for Neuron-based BERT inference\")\n\tnodeType = flag.String(\"nodeType\", \"\",\n\t\t\"Node type label for K8s nodes, e.g., trn1.32xlarge or inf2.xlarge\")\n\tinferenceMode = flag.String(\"inferenceMode\", \"throughput\",\n\t\t\"Inference mode for BERT (throughput or latency)\")\n}\n" }, { "path": "test/cases/neuron-training/bert_training_test.go", "content": "//go:build e2e\n\npackage training\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"log\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient/k8s\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nvar (\n\t//go:embed manifests/bert-training.yaml\n\tbertTrainingJobManifest []byte\n\n\t//go:embed manifests/training-comm-service.yaml\n\ttrainingPodCommServiceManifest []byte\n\n\t// Regex to match lines like:\n\t// local_throughput=5.00 samples/s\n\trankThroughputRegex = regexp.MustCompile(\n\t\t`local_throughput\\s*=\\s*([\\d\\.]+)\\s+samples\\/s`,\n\t)\n\n\t// Regex to match lines like:\n\t// local_avg_epoch_time=12.50s\n\trankEpochTimeRegex = regexp.MustCompile(\n\t\t`local_avg_epoch_time=([\\d\\.]+)s`,\n\t)\n)\n\n// TestBertTraining runs the Neuron-based BERT training test\nfunc TestBertTraining(t *testing.T) {\n\tif *bertTrainingImage == \"\" {\n\t\tt.Fatal(\"bertTrainingImage must be set to run the test\")\n\t}\n\n\t// Render the templated manifest with dynamic variables\n\trenderVars := map[string]string{\n\t\t\"BertTrainingImage\": *bertTrainingImage,\n\t\t\"NodeType\": *nodeType,\n\t\t\"SlotsPerWorker\": fmt.Sprintf(\"%d\", nodeCount),\n\t\t\"NodeCount\": fmt.Sprintf(\"%d\", nodeCount),\n\t\t\"NeuronPerNode\": fmt.Sprintf(\"%d\", neuronPerNode),\n\t\t\"NeuronCorePerNode\": fmt.Sprintf(\"%d\", neuronCorePerNode),\n\t\t\"EFAPerNode\": fmt.Sprintf(\"%d\", efaPerNode),\n\t}\n\n\t// Render the manifest\n\trenderedManifest, err := fwext.RenderManifests(bertTrainingJobManifest, renderVars)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to render neuron BERT training manifest: %v\", err)\n\t}\n\n\trenderedCommServiceManifest, err := fwext.RenderManifests(trainingPodCommServiceManifest, renderVars)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to render pod communication manifest: %v\", err)\n\t}\n\n\t// Define a feature for the Neuron BERT training\n\tneuronTraining := features.New(\"bert-training\").\n\t\tWithLabel(\"suite\", \"neuron\").\n\t\tWithLabel(\"hardware\", \"neuron\").\n\t\tAssess(\"Neuron training Job succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tmanifests := [][]byte{renderedCommServiceManifest, renderedManifest}\n\t\t\tmaxAttempts := (*retries) + 1\n\n\t\t\tfor attempt := 0; attempt < maxAttempts; attempt++ {\n\t\t\t\tlog.Printf(\"Applying manifests for BERT training test (Attempt #%d)\", attempt+1)\n\n\t\t\t\tif err := applyManifests(cfg, manifests); err != nil {\n\t\t\t\t\tlog.Printf(\"Failed to apply manifests: %v\", err)\n\t\t\t\t\tcleanupManifests(cfg, manifests)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tjob, err := waitForJobCreation(cfg)\n\t\t\t\tif err != nil {\n\t\t\t\t\tlog.Printf(\"Failed to detect job creation: %v\", err)\n\t\t\t\t\tcleanupManifests(cfg, manifests)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tif err := waitForJobCompletion(job, cfg); err != nil {\n\t\t\t\t\tlog.Printf(\"Job did not complete successfully: %v\", err)\n\t\t\t\t\tlogsBuf, err := gatherJobLogs(ctx, cfg, \"default\", \"bert-training\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Printf(\"failed to get logs: %v\", err)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlog.Println(logsBuf.String())\n\t\t\t\t\t}\n\t\t\t\t\tcleanupManifests(cfg, manifests)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\t// Job completed successfully\n\t\t\t\tif err := processJobLogs(ctx, cfg); err != nil {\n\t\t\t\t\tlog.Printf(\"Failed to process job logs: %v\", err)\n\t\t\t\t\tcleanupManifests(cfg, manifests)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\t// Test succeeded, clean up and return\n\t\t\t\tcleanupManifests(cfg, manifests)\n\t\t\t\tlog.Printf(\"BERT training test succeeded on attempt #%d\", attempt+1)\n\t\t\t\treturn ctx\n\t\t\t}\n\n\t\t\t// If we've exhausted all attempts\n\t\t\tt.Fatalf(\"BERT training test did not succeed after %d attempts\", maxAttempts)\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\t// Run the feature\n\ttestenv.Test(t, neuronTraining)\n}\n\n// gatherJobLogs retrieves logs from all pods of the specified jobName, returning them as a buffer.\nfunc gatherJobLogs(ctx context.Context, cfg *envconf.Config, namespace, jobName string) (*bytes.Buffer, error) {\n\tclientset, err := getClientset(cfg.Client().RESTConfig())\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create kubernetes clientset: %w\", err)\n\t}\n\n\tpodList, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{\n\t\tLabelSelector: fmt.Sprintf(\"job-name=%s\", jobName),\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to list pods for job %s: %w\", jobName, err)\n\t}\n\tif len(podList.Items) == 0 {\n\t\treturn nil, fmt.Errorf(\"no pods found for job %s\", jobName)\n\t}\n\n\tvar out bytes.Buffer\n\tfor _, pod := range podList.Items {\n\t\treq := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{})\n\t\tlogStream, err := req.Stream(ctx)\n\t\tif err != nil {\n\t\t\treturn &out, fmt.Errorf(\"failed to get logs from pod %s: %w\", pod.Name, err)\n\t\t}\n\t\tdefer logStream.Close()\n\n\t\t// Copy logs into our buffer\n\t\tif _, err := out.ReadFrom(logStream); err != nil {\n\t\t\treturn &out, fmt.Errorf(\"failed to read logs from pod %s: %w\", pod.Name, err)\n\t\t}\n\t}\n\n\treturn &out, nil\n}\n\n// aggregateMetricFromLogs scans the log output for lines based on a provided RegEx.\n// The RegEx is assumed to take a sufficiently unique form like = to avoid\n// collisions, but also to simplify parsing.\n//\n// returns the average, sum, and count for all occurrences of the metric.\nfunc aggregateMetricFromLogs(metricRegex *regexp.Regexp, logs string) (avg float64, sum float64, count int) {\n\tmatches := metricRegex.FindAllStringSubmatch(logs, -1)\n\tfor _, match := range matches {\n\t\tval, err := strconv.ParseFloat(match[1], 64)\n\t\tif err == nil {\n\t\t\tsum += val\n\t\t\tcount++\n\t\t}\n\t}\n\tif count > 0 {\n\t\tavg = sum / float64(count)\n\t}\n\treturn avg, sum, count\n}\n\nfunc applyManifests(cfg *envconf.Config, manifests [][]byte) error {\n\tfwext.ApplyManifests(cfg.Client().RESTConfig(), manifests...)\n\tlog.Println(\"Successfully applied test manifests.\")\n\treturn nil\n}\n\nfunc waitForJobCreation(cfg *envconf.Config) (*batchv1.Job, error) {\n\tjob := &batchv1.Job{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: \"bert-training\",\n\t\t\tNamespace: \"default\",\n\t\t},\n\t}\n\n\tlog.Println(\"Waiting for the 'bert-training' Job resource to be created...\")\n\treturn job, wait.For(\n\t\tconditions.New(cfg.Client().Resources()).ResourceMatch(job, func(object k8s.Object) bool {\n\t\t\treturn true\n\t\t}),\n\t\twait.WithTimeout(time.Minute*5),\n\t)\n}\n\nfunc waitForJobCompletion(job *batchv1.Job, cfg *envconf.Config) error {\n\tlog.Println(\"Waiting for 'bert-training' Job to succeed...\")\n\treturn wait.For(\n\t\tfwext.NewConditionExtension(cfg.Client().Resources()).JobSucceeded(job),\n\t\twait.WithTimeout(30*time.Minute),\n\t)\n}\n\nfunc processJobLogs(ctx context.Context, cfg *envconf.Config) error {\n\tlogsBuf, err := gatherJobLogs(ctx, cfg, \"default\", \"bert-training\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to retrieve bert-training job logs: %v\", err)\n\t}\n\n\tlog.Println(\"== Raw Logs from the launcher pods ==\")\n\tlog.Println(logsBuf.String())\n\n\tprocessMetrics(logsBuf.String())\n\treturn nil\n}\n\nfunc processMetrics(logs string) {\n\t// Process throughput\n\tavgThru, sumThru, countThru := aggregateMetricFromLogs(rankThroughputRegex, logs)\n\tif countThru == 0 {\n\t\tlog.Printf(\"No throughput lines found. Possibly missing in logs.\")\n\t} else {\n\t\tlog.Printf(\"Parsed throughput from %d ranks. Total=%.2f samples/s, Average=%.2f samples/s\",\n\t\t\tcountThru, sumThru, avgThru)\n\t\tlog.Printf(\"Average Throughput: %.2f samples/second\", avgThru)\n\t}\n\n\t// Process epoch time\n\tavgEp, sumEp, countEp := aggregateMetricFromLogs(rankEpochTimeRegex, logs)\n\tif countEp == 0 {\n\t\tlog.Printf(\"No epoch time lines found. Possibly missing in logs.\")\n\t} else {\n\t\tlog.Printf(\"Parsed average epoch time from %d ranks. Sum=%.2fs, Average=%.2fs\",\n\t\t\tcountEp, sumEp, avgEp)\n\t}\n}\n\nfunc cleanupManifests(cfg *envconf.Config, manifests [][]byte) {\n\tlog.Println(\"Deleting test manifests.\")\n\tif err := fwext.DeleteManifests(cfg.Client().RESTConfig(), manifests...); err != nil {\n\t\tlog.Printf(\"Failed to delete manifests: %v\", err)\n\t}\n}\n\n// getClientset creates a Kubernetes clientset from the given REST config\nfunc getClientset(restConfig *rest.Config) (*kubernetes.Clientset, error) {\n\tclientset, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create kubernetes clientset: %w\", err)\n\t}\n\treturn clientset, nil\n}\n" }, { "path": "test/cases/neuron-training/main_test.go", "content": "//go:build e2e\n\npackage training\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"slices\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nfunc TestMain(m *testing.M) {\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\ttestenv = env.NewWithConfig(cfg)\n\n\tmanifests := [][]byte{\n\t\tmanifests.NeuronDevicePluginRbacManifest,\n\t\tmanifests.NeuronDevicePluginManifest,\n\t\tmanifests.EfaDevicePluginManifest,\n\t}\n\n\ttestenv.Setup(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Applying Neuron device plugin RBAC, Neuron device plugin and EFA device plugin manifests.\")\n\t\t\terr := fwext.ApplyManifests(config.Client().RESTConfig(), manifests...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to apply manifests: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"Successfully applied Neuron device plugin RBAC, Neuron device plugin and EFA device plugin manifests.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Waiting for Neuron Device Plugin daemonset to be ready.\")\n\t\t\tdaemonset := appsv1.DaemonSet{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"neuron-device-plugin-daemonset\", Namespace: \"kube-system\"},\n\t\t\t}\n\t\t\terr := wait.For(\n\t\t\t\tfwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&daemonset),\n\t\t\t\twait.WithTimeout(time.Minute*5),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"Neuron Device Plugin daemonset is not ready: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"Neuron Device Plugin daemonset is ready.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Waiting for EFA Device Plugin daemonset to be ready.\")\n\t\t\tdaemonset := appsv1.DaemonSet{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"aws-efa-k8s-device-plugin-daemonset\", Namespace: \"kube-system\"},\n\t\t\t}\n\t\t\terr := wait.For(\n\t\t\t\tfwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&daemonset),\n\t\t\t\twait.WithTimeout(time.Minute*5),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"EFA Device Plugin daemonset is not ready: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"EFA Device Plugin daemonset is ready.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t\tcheckNonZeroResourceCapacity,\n\t\tcheckNodeTypes,\n\t)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Deleting Neuron device plugin and EFA device plugin manifests.\")\n\t\t\tslices.Reverse(manifests)\n\t\t\terr := fwext.DeleteManifests(config.Client().RESTConfig(), manifests...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to delete manifests: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"Successfully deleted Neuron device plugin and EFA device plugin manifests.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tlog.Println(\"Starting tests...\")\n\texitCode := testenv.Run(m)\n\tlog.Printf(\"Tests finished with exit code %d\", exitCode)\n\tos.Exit(exitCode)\n}\n\nfunc checkNodeTypes(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tclientset, err := kubernetes.NewForConfig(config.Client().RESTConfig())\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to create Kubernetes client: %w\", err)\n\t}\n\n\tnodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to list nodes: %w\", err)\n\t}\n\n\tif len(nodes.Items) == 0 {\n\t\treturn ctx, fmt.Errorf(\"no nodes found in the cluster\")\n\t}\n\n\t// Check if all nodes have the same instance type\n\tfor i := 1; i < len(nodes.Items); i++ {\n\t\tcurrentInstanceType := nodes.Items[i].Labels[\"node.kubernetes.io/instance-type\"]\n\t\tif currentInstanceType != nodes.Items[i-1].Labels[\"node.kubernetes.io/instance-type\"] {\n\t\t\treturn ctx, fmt.Errorf(\"inconsistent node types detected, all nodes must have the same instance type\")\n\t\t} else if *nodeType == \"\" {\n\t\t\tlog.Printf(\"[INFO] nodeType was not set, discovered type %s\", currentInstanceType)\n\t\t\t*nodeType = currentInstanceType\n\t\t}\n\t}\n\n\t// Calculate capacities for all nodes\n\ttotalNeuronCount := 0\n\ttotalNeuronCoreCount := 0\n\ttotalEfaCount := 0\n\tnodeCount = len(nodes.Items) // Store global node count\n\n\tfor _, node := range nodes.Items {\n\t\tlog.Printf(\"[INFO] Processing node %s\", node.Name)\n\n\t\t// Check for Neuron capacity\n\t\tneuron, ok := node.Status.Capacity[\"aws.amazon.com/neuron\"]\n\t\tif ok {\n\t\t\ttotalNeuronCount += int(neuron.Value())\n\t\t} else {\n\t\t\tlog.Printf(\"[WARN] Node %s does not have 'aws.amazon.com/neuron' capacity\", node.Name)\n\t\t}\n\n\t\t// Check for NeuronCore capacity\n\t\tneuronCore, ok := node.Status.Capacity[\"aws.amazon.com/neuroncore\"]\n\t\tif ok {\n\t\t\ttotalNeuronCoreCount += int(neuronCore.Value())\n\t\t} else {\n\t\t\tlog.Printf(\"[WARN] Node %s does not have 'aws.amazon.com/neuroncore' capacity\", node.Name)\n\t\t}\n\n\t\t// Check for EFA capacity\n\t\tefa, ok := node.Status.Capacity[\"vpc.amazonaws.com/efa\"]\n\t\tif ok {\n\t\t\ttotalEfaCount += int(efa.Value())\n\t\t} else {\n\t\t\tlog.Printf(\"[WARN] Node %s does not have 'vpc.amazonaws.com/efa' capacity\", node.Name)\n\t\t}\n\t}\n\n\t// Update global capacities\n\tif nodeCount > 0 {\n\t\tneuronPerNode = totalNeuronCount / nodeCount\n\t\tneuronCorePerNode = totalNeuronCoreCount / nodeCount\n\t\tefaPerNode = totalEfaCount / nodeCount\n\t} else {\n\t\tlog.Printf(\"[WARN] No nodes found, setting capacities to 0\")\n\t\tneuronPerNode = 0\n\t\tneuronCorePerNode = 0\n\t\tefaPerNode = 0\n\t}\n\n\tlog.Printf(\"[INFO] Total Nodes: %d\", nodeCount)\n\tlog.Printf(\"[INFO] Total Neuron Count: %d, Neuron Per Node: %d\", totalNeuronCount, neuronPerNode)\n\tlog.Printf(\"[INFO] Total Neuron Core Count: %d, Neuron Core Per Node: %d\", totalNeuronCoreCount, neuronCorePerNode)\n\tlog.Printf(\"[INFO] Total EFA Count: %d, EFA Per Node: %d\", totalEfaCount, efaPerNode)\n\n\treturn ctx, nil\n}\n\nfunc checkNonZeroResourceCapacity(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tlog.Println(\"[INFO] Starting resource capacity checks\")\n\n\t// Check Neuron devices\n\tlog.Println(\"Checking Neuron device capacity on nodes\")\n\terr := wait.For(\n\t\tfwext.NewConditionExtension(config.Client().Resources()).AllNodesHaveNonZeroResourceCapacity(\"aws.amazon.com/neuron\"),\n\t\twait.WithTimeout(time.Second*60),\n\t\twait.WithInterval(time.Second*5),\n\t)\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to verify Neuron device capacity on nodes: %w\", err)\n\t}\n\tlog.Println(\"Neuron devices check passed - all nodes have non-zero capacity\")\n\n\t// Check Neuron cores\n\tlog.Println(\"Checking Neuron core capacity on nodes\")\n\terr = wait.For(\n\t\tfwext.NewConditionExtension(config.Client().Resources()).AllNodesHaveNonZeroResourceCapacity(\"aws.amazon.com/neuroncore\"),\n\t\twait.WithTimeout(time.Second*60),\n\t\twait.WithInterval(time.Second*5),\n\t)\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to verify Neuron core capacity on nodes: %w\", err)\n\t}\n\tlog.Println(\"Neuron cores check passed - all nodes have non-zero capacity\")\n\n\t// Check EFA devices\n\tlog.Println(\"Checking EFA device capacity on nodes\")\n\terr = wait.For(\n\t\tfwext.NewConditionExtension(config.Client().Resources()).AllNodesHaveNonZeroResourceCapacity(\"vpc.amazonaws.com/efa\"),\n\t\twait.WithTimeout(time.Second*60),\n\t\twait.WithInterval(time.Second*5),\n\t)\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to verify EFA device capacity on nodes: %w\", err)\n\t}\n\tlog.Println(\"EFA devices check passed - all nodes have non-zero capacity\")\n\n\tlog.Println(\"[INFO] All resource capacity checks completed successfully\")\n\treturn ctx, nil\n}\n" }, { "path": "test/cases/neuron-training/manifests/bert-training.yaml", "content": "apiVersion: batch/v1\nkind: Job\nmetadata:\n labels:\n app: bert-training\n name: bert-training\nspec:\n completionMode: Indexed\n completions: {{.NodeCount}}\n parallelism: {{.NodeCount}}\n backoffLimit: 0\n template:\n spec:\n restartPolicy: Never\n containers:\n - image: {{.BertTrainingImage}}\n name: bert-training\n env:\n - name: MASTER_ADDR\n value: bert-training-0.training\n args:\n - sh\n - -c \n - |\n # Enable EFA https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-runtime/nrt-troubleshoot.html#fi-efa-fork-safe (AL2 legacy requirement)\n export FI_EFA_FORK_SAFE=1\n export CCOM_SOCKET_IFNAME=eth0\n export NCCL_DEBUG=ERROR\n torchrun --nproc_per_node {{.NeuronCorePerNode}} --nnodes {{.NodeCount}} --node_rank $JOB_COMPLETION_INDEX --master_addr $MASTER_ADDR train.py\n volumeMounts:\n - name: dshm\n mountPath: /dev/shm \n resources:\n requests:\n aws.amazon.com/neuron: {{.NeuronPerNode}}\n aws.amazon.com/neuroncore: {{.NeuronCorePerNode}}\n vpc.amazonaws.com/efa: {{.EFAPerNode}}\n limits:\n aws.amazon.com/neuron: {{.NeuronPerNode}}\n aws.amazon.com/neuroncore: {{.NeuronCorePerNode}}\n vpc.amazonaws.com/efa: {{.EFAPerNode}}\n nodeSelector:\n node.kubernetes.io/instance-type: {{.NodeType}}\n subdomain: training\n volumes:\n - name: dshm\n emptyDir:\n medium: Memory\n" }, { "path": "test/cases/neuron-training/manifests/training-comm-service.yaml", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: training\n labels:\n app: training\nspec:\n clusterIP: None\n selector:\n job-name: bert-training\n" }, { "path": "test/cases/neuron-training/vars.go", "content": "package training\n\nimport (\n\t\"flag\"\n\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n)\n\n// Shared global variables\nvar (\n\ttestenv env.Environment\n\n\tbertTrainingImage *string\n\tefaEnabled *bool\n\tnodeType *string\n\tnodeCount int\n\tefaPerNode int\n\tneuronPerNode int\n\tneuronCorePerNode int\n\tretries *int\n)\n\nfunc init() {\n\t// Define command-line flags\n\tbertTrainingImage = flag.String(\"bertTrainingImage\", \"\", \"Docker image used for BERT training workload\")\n\tefaEnabled = flag.Bool(\"efaEnabled\", false, \"Enable Elastic Fabric Adapter (EFA)\")\n\tnodeType = flag.String(\"nodeType\", \"\", \"Instance type for cluster nodes (e.g., inf1.24xlarge)\")\n\tretries = flag.Int(\"retries\", 2, \"Number of retries to attempt before marking the test as failed.\")\n}\n" }, { "path": "test/cases/nvidia/capabilities_test.go", "content": "//go:build e2e\n\npackage nvidia\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\t\"k8s.io/apimachinery/pkg/util/wait\"\n\te2ewait \"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n\n\t_ \"embed\"\n)\n\n//go:embed manifests/nvidia-driver-capabilities-check.yaml\nvar capabilitiesCheckPod []byte\n\nconst (\n\tPodName = \"moderngl-pod\"\n\tPodNamespace = \"default\"\n)\n\nfunc TestNvidiaDriverCapabilities(t *testing.T) {\n\tfeat := features.New(\"nvidia-driver-capabilities-check\").\n\t\tWithLabel(\"suite\", \"nvidia\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tt.Log(\"Applying nvidia driver capabilities check pod manifest.\")\n\t\t\t// capabilitiesCheckPod only run moderngl.create_standalone_context() with NVIDIA_DRIVER_CAPABILITIES=all to load all capabilities enabled by nvidia driver.\n\t\t\t// If any lib required by any of nvidia driver capabilities is missing, it will failed with exception.\n\t\t\tif err := e2e.ApplyManifests(cfg.Client().RESTConfig(), capabilitiesCheckPod); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to apply capabilities check pod manifest: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Check Pod becomes ready\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tt.Log(\"Waiting up to 5 minute for pod to complete...\")\n\t\t\tpod := &v1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tName: PodName,\n\t\t\t\t\tNamespace: PodNamespace,\n\t\t\t\t},\n\t\t\t}\n\t\t\terr := e2ewait.For(\n\t\t\t\te2e.NewConditionExtension(cfg.Client().Resources()).PodSucceeded(pod),\n\t\t\t\te2ewait.WithTimeout(5*time.Minute),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tif err == wait.ErrWaitTimeout {\n\t\t\t\t\tt.Fatalf(\"nvidia capabilities check pod not in compeleted phase (succeeded or failed) within 5 minute and waiter timeout: %v\", err)\n\t\t\t\t}\n\t\t\t\tt.Fatalf(\"nvidia capabilities pod in Failed status, ModernGL check failed. Could be caused by required library missing\")\n\t\t\t}\n\t\t\tt.Log(\"nvidia driver capabilities check succeeded.\")\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tt.Log(\"Removing nvidia driver capabilities check pod.\")\n\t\t\tif err := e2e.DeleteManifests(cfg.Client().RESTConfig(), capabilitiesCheckPod); err != nil {\n\t\t\t\tt.Errorf(\"Failed to delete pod: %v\", err)\n\t\t\t}\n\t\t\tt.Log(\"all test resources removed successfully.\")\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, feat)\n}\n" }, { "path": "test/cases/nvidia/containerd_test.go", "content": "//go:build e2e\n\npackage nvidia\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n\n\t_ \"embed\"\n)\n\n//go:embed manifests/daemonset-containerd-check.yaml\nvar containerdCheckDS []byte\n\nfunc TestContainerdConfig(t *testing.T) {\n\tfeat := features.New(\"containerd-config-check\").\n\t\tWithLabel(\"suite\", \"nvidia\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog.Println(\"[Setup] Applying containerd-check DaemonSet manifest.\")\n\t\t\tif err := e2e.ApplyManifests(cfg.Client().RESTConfig(), containerdCheckDS); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to apply containerd-check DS: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"DaemonSet becomes ready\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tdsName := \"containerd-check\"\n\t\t\tdsNS := \"default\"\n\n\t\t\tlog.Println(\"[Assess] Waiting up to 1 minute for containerd-check DS to become Ready...\")\n\t\t\tds := &appsv1.DaemonSet{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tName: dsName,\n\t\t\t\t\tNamespace: dsNS,\n\t\t\t\t},\n\t\t\t}\n\t\t\terr := wait.For(\n\t\t\t\te2e.NewConditionExtension(cfg.Client().Resources()).DaemonSetReady(ds),\n\t\t\t\twait.WithTimeout(1*time.Minute),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"[Assess] containerd-check DS did not become Ready: %v\", err)\n\t\t\t\te2e.PrintDaemonSetPodLogs(t, ctx, cfg.Client().RESTConfig(), dsNS, \"app=containerd-check\")\n\t\t\t\tt.Fatalf(\"containerd-check DS not Ready within 1 minute\")\n\t\t\t}\n\n\t\t\tlog.Println(\"[Assess] containerd-check DS is Ready.\")\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tt.Log(\"[Teardown] Removing containerd-check DS (no additional logs).\")\n\t\t\tif err := e2e.DeleteManifests(cfg.Client().RESTConfig(), containerdCheckDS); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to delete containerd-check DS: %v\", err)\n\t\t\t}\n\t\t\tt.Log(\"[Teardown] containerd-check DS removed successfully.\")\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, feat)\n}\n" }, { "path": "test/cases/nvidia/main_test.go", "content": "//go:build e2e\n\npackage nvidia\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"slices\"\n\t\"testing\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\ntype Config struct {\n\tcommon.MetricOps\n\tNodeType string `flag:\"nodeType\" desc:\"node type for the tests\"`\n\tInstallDevicePlugin bool `flag:\"installDevicePlugin\" desc:\"install nvidia device plugin\"`\n\tEfaEnabled bool `flag:\"efaEnabled\" desc:\"enable efa tests\"`\n\tNvidiaTestImage string `flag:\"nvidiaTestImage\" desc:\"nccl test image for nccl tests\"`\n\tPytorchImage string `flag:\"pytorchImage\" desc:\"pytorch cuda image for single node tests\"`\n\tSkipUnitTestSubcommand string `flag:\"skipUnitTestSubcommand\" desc:\"optional command to skip specified unit test\"`\n}\n\nvar (\n\ttestenv env.Environment\n\ttestConfig Config\n\tnodeCount int\n\tgpuPerNode int\n\tefaPerNode int\n)\n\nfunc deployMPIOperator(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tdep := appsv1.Deployment{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"mpi-operator\", Namespace: \"mpi-operator\"},\n\t}\n\terr := wait.For(conditions.New(config.Client().Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, v1.ConditionTrue),\n\t\twait.WithContext(ctx))\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to deploy mpi-operator: %v\", err)\n\t}\n\treturn ctx, nil\n}\n\nfunc checkNodeTypes(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tclientset, err := kubernetes.NewForConfig(config.Client().RESTConfig())\n\tif err != nil {\n\t\treturn ctx, err\n\t}\n\n\tnodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\tif err != nil {\n\t\treturn ctx, err\n\t}\n\n\tfor i := 1; i < len(nodes.Items)-1; i++ {\n\t\tif nodes.Items[i].Labels[\"node.kubernetes.io/instance-type\"] != nodes.Items[i-1].Labels[\"node.kubernetes.io/instance-type\"] {\n\t\t\treturn ctx, fmt.Errorf(\"Node types are not the same, all node types must be the same in the cluster\")\n\t\t}\n\t}\n\n\tif testConfig.NodeType != \"\" {\n\t\tfor _, v := range nodes.Items {\n\t\t\tif v.Labels[\"node.kubernetes.io/instance-type\"] == testConfig.NodeType {\n\t\t\t\tnodeCount++\n\t\t\t\tgpu := v.Status.Capacity[\"nvidia.com/gpu\"]\n\t\t\t\tgpuPerNode = int(gpu.Value())\n\t\t\t\tefa := v.Status.Capacity[\"vpc.amazonaws.com/efa\"]\n\t\t\t\tefaPerNode = int(efa.Value())\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlog.Printf(\"No node type specified. Using the node type %s in the node groups.\", nodes.Items[0].Labels[\"node.kubernetes.io/instance-type\"])\n\t\ttestConfig.NodeType = nodes.Items[0].Labels[\"node.kubernetes.io/instance-type\"]\n\t\tnodeCount = len(nodes.Items)\n\t\tgpu := nodes.Items[0].Status.Capacity[\"nvidia.com/gpu\"]\n\t\tgpuPerNode = int(gpu.Value())\n\t\tefa := nodes.Items[0].Status.Capacity[\"vpc.amazonaws.com/efa\"]\n\t\tefaPerNode = int(efa.Value())\n\t}\n\n\treturn ctx, nil\n}\n\nfunc TestMain(m *testing.M) {\n\ttestConfig = Config{\n\t\tInstallDevicePlugin: true,\n\t\tPytorchImage: \"763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:2.1.0-gpu-py310-cu121-ubuntu20.04-ec2\",\n\t}\n\n\t_, err := common.ParseFlags(&testConfig)\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to parse flags: %v\", err)\n\t}\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = env.NewWithConfig(cfg).WithContext(ctx)\n\n\tmanifestsList := [][]byte{\n\t\tmanifests.MpiOperatorManifest,\n\t}\n\n\tsetUpFunctions := []env.Func{\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\terr := fwext.ApplyManifests(config.Client().RESTConfig(), manifestsList...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t\tdeployMPIOperator,\n\t}\n\n\tif testConfig.InstallDevicePlugin {\n\t\tmanifestsList = append(manifestsList, manifests.NvidiaDevicePluginManifest)\n\t\tsetUpFunctions = append(setUpFunctions, func(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\treturn common.DeployDaemonSet(\"nvidia-device-plugin-daemonset\", \"kube-system\")(ctx, config)\n\t\t})\n\t}\n\n\tif testConfig.EfaEnabled {\n\t\tmanifestsList = append(manifestsList, manifests.EfaDevicePluginManifest)\n\t\tsetUpFunctions = append(setUpFunctions, func(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\treturn common.DeployDaemonSet(\"aws-efa-k8s-device-plugin-daemonset\", \"kube-system\")(ctx, config)\n\t\t})\n\t}\n\n\tif len(testConfig.MetricDimensions) > 0 {\n\t\trenderedCloudWatchAgentManifest, err := manifests.RenderCloudWatchAgentManifest(testConfig.MetricDimensions)\n\t\tif err != nil {\n\t\t\tlog.Printf(\"Warning: failed to render CloudWatch Agent manifest: %v\", err)\n\t\t}\n\t\tmanifestsList = append(manifestsList, manifests.DCGMExporterManifest, renderedCloudWatchAgentManifest)\n\t\tsetUpFunctions = append(setUpFunctions, func(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tif ctx, err := common.DeployDaemonSet(\"dcgm-exporter\", \"kube-system\")(ctx, config); err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\tif ctx, err := common.DeployDaemonSet(\"cwagent\", \"amazon-cloudwatch\")(ctx, config); err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t})\n\t}\n\n\tsetUpFunctions = append(setUpFunctions, checkNodeTypes)\n\ttestenv.Setup(setUpFunctions...)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tslices.Reverse(manifestsList)\n\t\t\terr := fwext.DeleteManifests(config.Client().RESTConfig(), manifestsList...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/nvidia/manifests/daemonset-containerd-check.yaml", "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: containerd-check\n namespace: default\n labels:\n app: containerd-check\nspec:\n selector:\n matchLabels:\n app: containerd-check\n template:\n metadata:\n labels:\n app: containerd-check\n spec:\n containers:\n - name: containerd-check\n image: public.ecr.aws/amazonlinux/amazonlinux:latest\n command:\n - sh\n - -c\n - |\n # 1. Ensure the script fails on any command or pipeline error\n set -e\n set -o pipefail\n\n echo \"=== content read by the container ===\"\n cat /host-etc/containerd/config.toml\n\n # 2. Check containerd config version and look for appropriate sandbox field\n # In containerd config version = 2 expect to find pattern `sandbox_image = \"registry.k8s.io/pause:3.10.1\"`\n # In containerd config version = 3 expect to find pattern `sandbox = \"registry.k8s.io/pause:3.10.1\"`\n # For more details: https://github.com/containerd/containerd/blob/main/docs/cri/config.md\n version_line=$(grep -E '^version\\s*=' /host-etc/containerd/config.toml || true)\n if [ -z \"$version_line\" ]; then\n echo \"FAIL: no version line found in containerd config\"\n exit 1\n fi\n\n version=$(echo \"$version_line\" | cut -d'=' -f2 | tr -d ' ')\n echo \"INFO: containerd config version = $version\"\n if [ \"$version\" = \"2\" ]; then\n sandbox_line=$(grep -E 'sandbox_image\\s*=' /host-etc/containerd/config.toml || true)\n elif [ \"$version\" = \"3\" ]; then\n sandbox_line=$(grep -E 'sandbox\\s*=' /host-etc/containerd/config.toml || true)\n else\n echo \"FAIL: unsupported containerd config version: $version\"\n exit 1\n fi\n\n # 3. If no sandbox configuration is found, fail explicitly\n if [ -z \"$sandbox_line\" ]; then\n echo \"FAIL: no sandbox_image or sandbox line found\"\n echo \"=== debug ===\"\n exit 1\n fi\n sandbox_image=$(echo \"$sandbox_line\" | cut -d'\"' -f2)\n\n # 4. Check that $sandbox_image references .ecr. or is provided on the instance\n if [[ \"$sandbox_image\" == \"localhost\"* ]]; then\n echo \"INFO: skipping .ecr. check for localhost sandbox image\"\n else\n if [[ \"$sandbox_image\" != *\".ecr.\"* ]]; then\n echo \"FAIL: no .ecr. reference in $sandbox_image\"\n echo \"=== debug ===\"\n exit 1\n fi\n fi\n\n # 5. Check for 'nvidia-container-runtime'\n if ! grep -q \"nvidia-container-runtime\" /host-etc/containerd/config.toml; then\n echo \"FAIL: no nvidia-container-runtime found\"\n echo \"=== debug ===\"\n exit 1\n fi\n\n # 6. Check for 'systemd_cgroup = true' or 'SystemdCgroup = true'\n if ! ( grep -q 'systemd_cgroup = true' /host-etc/containerd/config.toml || \\\n grep -q 'SystemdCgroup = true' /host-etc/containerd/config.toml ); then\n echo \"FAIL: no systemd cgroup setting\"\n echo \"=== debug ===\"\n exit 1\n fi\n\n echo \"containerd config check PASSED.\"\n # Keep container running so DS can be marked Ready\n tail -f /dev/null\n volumeMounts:\n - name: containerd-config\n mountPath: /host-etc/containerd\n readOnly: true\n volumes:\n - name: containerd-config\n hostPath:\n path: /etc/containerd\n" }, { "path": "test/cases/nvidia/manifests/job-hpc-benchmarks.yaml", "content": "kind: Job\napiVersion: batch/v1\nmetadata:\n name: hpc-benckmarks-job\n labels:\n app: hpc-benckmarks-job\nspec:\n completions: 1\n parallelism: 1\n template:\n metadata:\n labels:\n app: hpc-benckmarks-job\n spec:\n volumes:\n - name: dshm\n emptyDir:\n medium: Memory\n containers:\n - name: hpc-benchmarks\n image: \"nvcr.io/nvidia/hpc-benchmarks:25.04\"\n command:\n - mpirun\n - --allow-run-as-root\n - -np\n - \"{{.GpuPerNode}}\"\n - -bind-to\n - none\n - -x\n - NCCL_DEBUG=INFO\n - -x \n - HPL_FCT_COMM_POLICY=1 \n - -x \n - HPL_USE_NVSHMEM=0\n # TODO: for arm it will be\n # - hpl-aarch64.sh\n - hpl.sh \n - --mem-affinity \n - 0:0:0:0:1:1:1:1 \n # --cpu-affinity needs to be tuned depending on the number of CPUs\n # available on the instance type.\n - --cpu-affinity \n - 0-13:14-27:28-41:42-55:56-69:70-83:84-97:98-111\n - --no-multinode \n - --dat \n - hpl-linux-x86_64/sample-dat/HPL-dgx-1N.dat\n # TODO: the path differs for arm64\n # - hpl-linux-aarch64-gpu/sample-dat/HPL-dgx-1N.dat\n volumeMounts:\n - mountPath: /dev/shm\n name: dshm\n imagePullPolicy: Always\n resources:\n limits:\n nvidia.com/gpu: {{.GpuPerNode}}\n env:\n - name: UCX_TLS\n value: \"^sysv\"\n restartPolicy: Never\n backoffLimit: 4\n" }, { "path": "test/cases/nvidia/manifests/job-unit-test-single-node.yaml", "content": "kind: Job\napiVersion: batch/v1\nmetadata:\n name: unit-test-job\n labels:\n app: unit-test-job\nspec:\n template:\n metadata:\n labels:\n app: unit-test-job\n spec:\n containers:\n - name: unit-test-container\n image: {{.NvidiaTestImage}}\n command: \n - /bin/bash\n - ./gpu_unit_tests/unit_test\n env:\n - name: SKIP_TESTS_SUBCOMMAND\n value: {{.SkipTestSubcommand}}\n # because we started building these from source, this is just a\n # regular binary.\n - name: DEMO_SUITE_DIR\n value: /usr/bin\n - name: EC2_INSTANCE_TYPE\n value: {{.NodeType}}\n imagePullPolicy: Always\n resources:\n limits:\n nvidia.com/gpu: {{.GpuPerNode}}\n requests:\n cpu: \"1\"\n memory: 1Gi\n restartPolicy: Never\n backoffLimit: 1\n" }, { "path": "test/cases/nvidia/manifests/mpi-job-nccl-test-multi-node.yaml", "content": "apiVersion: kubeflow.org/v2beta1\nkind: MPIJob\nmetadata:\n name: {{.JobName}}\nspec:\n slotsPerWorker: {{.GpuPerNode}}\n runPolicy:\n # it may take a bit for the workers to get ready (the container image is heavy)\n # and we don't want the launcher to reach it's CrashLoopBackoff limit in the meantime\n backoffLimit: 20\n cleanPodPolicy: Running\n mpiReplicaSpecs:\n Launcher:\n replicas: 1\n template:\n spec:\n restartPolicy: OnFailure\n containers:\n - image: {{.NvidiaTestImage}}\n imagePullPolicy: Always\n name: nccl-test-launcher\n env:\n command:\n - mpirun\n - --allow-run-as-root\n - --tag-output\n - -np\n - \"{{.WorkerNodeGpuCount}}\"\n - -bind-to\n - none\n - -map-by\n - slot\n - -x\n - PATH\n - -x\n - LD_LIBRARY_PATH\n - -x\n - NCCL_DEBUG=INFO\n - -x\n - NCCL_BUFFSIZE={{.NcclBuffSize}}\n - -x\n - NCCL_TUNER_PLUGIN=/opt/aws-ofi-nccl/install/lib/libnccl-ofi-tuner.so\n - --mca\n - pml\n - ^cm,ucx\n - --mca\n - btl\n - tcp,self\n - --mca\n - btl_tcp_if_exclude\n - lo,docker0,veth_def_agent\n - /opt/nccl-tests/build/{{.TestName}}\n - -b\n - \"8\"\n - -e\n - {{.MaxBytes}}\n - -f\n - \"2\"\n - -c\n - \"1\"\n - -n\n - \"10\"\n Worker:\n replicas: {{.WorkerNodeCount}}\n template:\n spec:\n volumes:\n - name: dshm\n emptyDir:\n medium: Memory\n containers:\n - image: {{.NvidiaTestImage}}\n imagePullPolicy: Always\n name: nccl-test-worker\n volumeMounts:\n - mountPath: /dev/shm\n name: dshm\n resources:\n requests:\n nvidia.com/gpu: {{.GpuPerNode}}\n vpc.amazonaws.com/efa: {{.EfaInterfacePerNode}}\n limits:\n nvidia.com/gpu: {{.GpuPerNode}}\n vpc.amazonaws.com/efa: {{.EfaInterfacePerNode}}\n" }, { "path": "test/cases/nvidia/manifests/mpi-job-pytorch-training-single-node.yaml", "content": "---\n# container image from: https://github.com/aws/deep-learning-containers/blob/master/available_images.md\napiVersion: kubeflow.org/v2beta1\nkind: MPIJob\nmetadata:\n name: pytorch-training-single-node\nspec:\n slotsPerWorker: 4\n runPolicy:\n cleanPodPolicy: Running\n mpiImplementation: OpenMPI\n mpiReplicaSpecs:\n Launcher:\n replicas: 1\n template:\n spec:\n restartPolicy: OnFailure\n containers:\n - image: {{.PytorchTestImage}}\n name: gpu-test\n command:\n - mpirun\n - --allow-run-as-root\n - -np\n - \"1\"\n - -mca\n - btl_tcp_if_exclude\n - lo\n - -mca\n - pml\n - ob1\n - -mca\n - btl\n - ^openib\n - --bind-to\n - none\n - -map-by\n - slot\n - -x\n - LD_LIBRARY_PATH\n - -x\n - PATH\n - -x\n - NCCL_SOCKET_IFNAME=eth0\n - -x\n - NCCL_DEBUG=INFO\n - -x\n - MXNET_CUDNN_AUTOTUNE_DEFAULT=0\n - python\n - -c\n - import os; os.system(\"git clone https://github.com/pytorch/examples.git pytorch-examples\"); os.system(\"git -C pytorch-examples checkout 0f0c9131ca5c79d1332dce1f4c06fe942fbdc665\"); os.system(\"python pytorch-examples/mnist/main.py --epochs 1\")\n resources:\n limits:\n nvidia.com/gpu: 1\n" }, { "path": "test/cases/nvidia/manifests/nvidia-driver-capabilities-check.yaml", "content": "apiVersion: v1\nkind: Pod\nmetadata:\n name: moderngl-pod\nspec:\n restartPolicy: Never\n tolerations:\n - key: \"nvidia.com/gpu\"\n operator: \"Exists\"\n effect: \"NoSchedule\"\n containers:\n - name: moderngl-container\n env:\n - name: NVIDIA_DRIVER_CAPABILITIES\n value: \"all\"\n image: public.ecr.aws/ubuntu/ubuntu:22.04\n command: [\"/bin/bash\"]\n args:\n - -c\n - |\n set -e\n apt-get update\n apt-get install -y \\\n python3 \\\n python3-pip \\\n libgl1-mesa-glx \\\n libegl1-mesa-dev \\\n libgles2-mesa-dev \\\n mesa-utils \\\n xvfb\n pip3 install moderngl\n sleep 60\n cat <<'EOF' > moderngl-script.py\n import moderngl\n moderngl.create_standalone_context(backend='egl')\n EOF\n python3 moderngl-script.py\n resources:\n requests:\n memory: \"50Gi\"\n cpu: \"15\"\n \"nvidia.com/gpu\": \"1\"\n limits:\n memory: \"50Gi\"\n \"nvidia.com/gpu\": \"1\"\n" }, { "path": "test/cases/nvidia/mpi_test.go", "content": "//go:build e2e\n\npackage nvidia\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/internal/e2e/mpijobs\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n\n\t\"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured\"\n\t\"k8s.io/utils/strings/slices\"\n)\n\nvar (\n\tinstanceSupportsRdmaRead = []string{\"p5.48xlarge\", \"p4d.24xlarge\", \"p4de.24xlarge\", \"p5e.48xlarge\", \"p5en.48xlarge\"}\n)\n\nvar (\n\t//go:embed manifests/mpi-job-pytorch-training-single-node.yaml\n\tmpiJobPytorchTrainingSingleNodeManifest []byte\n\t//go:embed manifests/mpi-job-nccl-test-multi-node.yaml\n\tmpiJobNcclTestMultiNodeManifest []byte\n)\n\ntype ncclTestManifestTplVars struct {\n\tWorkerNodeCount int\n\tWorkerNodeGpuCount int\n\tGpuPerNode int\n\tNvidiaTestImage string\n\tEfaInterfacePerNode int\n\tMaxBytes string\n\tNcclBuffSize string\n\tTestName string\n\tJobName string\n}\n\nfunc TestMPIJobPytorchTraining(t *testing.T) {\n\ttestenv.Test(t,\n\t\tsingleNode(),\n\t\tmultiNode(\"all_reduce_perf\"),\n\t\tmultiNode(\"all_gather_perf\"),\n\t\tmultiNode(\"alltoall_perf\"),\n\t)\n}\n\nfunc multiNode(testName string) features.Feature {\n\tvar renderedMpiJobNcclTestMultiNodeManifest []byte\n\tjobName := strings.ReplaceAll(fmt.Sprintf(\"multi-node-%s\", testName), \"_\", \"-\")\n\n\treturn features.New(fmt.Sprintf(\"multi-node:%s\", testName)).\n\t\tWithLabel(\"suite\", \"nvidia\").\n\t\tWithLabel(\"hardware\", \"gpu\").\n\t\tWithLabel(\"hardware\", \"efa\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tif testConfig.NvidiaTestImage == \"\" {\n\t\t\t\tt.Fatal(fmt.Errorf(\"nvidiaTestImage must be set to run unit test, use https://github.com/aws/aws-k8s-tester/blob/main/test/images/nvidia/Dockerfile to build the image and -nvidiaTestImage to set the image url\"))\n\t\t\t}\n\t\t\tmaxBytes := \"2G\"\n\t\t\tncclBuffSize := \"4194304\"\n\t\t\tif slices.Contains(instanceSupportsRdmaRead, testConfig.NodeType) {\n\t\t\t\tt.Log(\"Instance supports RDMA\")\n\t\t\t\t// TODO: revisit this with some kind of per-instance optimizer, or maybe use the defaults for all instance types unless specified\n\t\t\t\tif testName == \"alltoall_perf\" && strings.Contains(testConfig.NodeType, \"p4\") {\n\t\t\t\t\t// Keep default values for P4 running all-to-all\n\t\t\t\t} else {\n\t\t\t\t\tmaxBytes = \"16G\"\n\t\t\t\t\tncclBuffSize = \"8388608\"\n\t\t\t\t}\n\t\t\t}\n\t\t\tvar err error\n\t\t\trenderedMpiJobNcclTestMultiNodeManifest, err = fwext.RenderManifests(mpiJobNcclTestMultiNodeManifest, ncclTestManifestTplVars{\n\t\t\t\t// one of the nodes will be used for the master pod\n\t\t\t\tWorkerNodeCount: nodeCount,\n\t\t\t\tWorkerNodeGpuCount: nodeCount * gpuPerNode,\n\t\t\t\tGpuPerNode: gpuPerNode,\n\t\t\t\tNvidiaTestImage: testConfig.NvidiaTestImage,\n\t\t\t\tEfaInterfacePerNode: efaPerNode,\n\t\t\t\tMaxBytes: maxBytes,\n\t\t\t\tNcclBuffSize: ncclBuffSize,\n\t\t\t\tTestName: testName,\n\t\t\t\tJobName: jobName,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tt.Log(\"Applying multi node manifest\")\n\t\t\terr = fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedMpiJobNcclTestMultiNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tt.Log(\"Manifest applied successfully\")\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"MPIJob succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tmpiJob := mpijobs.NewUnstructured(jobName, \"default\")\n\t\t\tt.Log(\"Waiting for multi node job to complete\")\n\t\t\terr := wait.For(conditions.New(cfg.Client().Resources()).ResourceMatch(mpiJob, mpijobs.MPIJobSucceeded),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(60*time.Minute),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tt.Logf(\"final mpijob resource: %v\", mpiJob)\n\t\t\tlog, err := fwext.GetJobLogs(cfg.Client().RESTConfig(), mpiJob)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"failed to get job logs: %v\", err)\n\t\t\t}\n\t\t\tt.Logf(\"Test log for %s:\", jobName)\n\t\t\tt.Log(log)\n\n\t\t\tif !t.Failed() {\n\t\t\t\tt.Log(\"Multi node job completed\")\n\t\t\t\t// Verify GPU Direct RDMA is used on P4/P5\n\t\t\t\tif testConfig.EfaEnabled && slices.Contains(instanceSupportsRdmaRead, testConfig.NodeType) {\n\t\t\t\t\tpattern := regexp.MustCompile(`\\[send\\] via NET/.*Libfabric/.*/GDRDMA`)\n\t\t\t\t\tif !pattern.MatchString(log) {\n\t\t\t\t\t\tt.Errorf(\"GPU Direct RDMA is not utilized for inter-node communication in NCCL tests on instances that support GDRDMA: %s\", testConfig.NodeType)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\terr := fwext.DeleteManifests(cfg.Client().RESTConfig(), renderedMpiJobNcclTestMultiNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n}\n\nfunc singleNode() features.Feature {\n\tvar renderedSingleNodeManifest []byte\n\n\treturn features.New(\"single-node\").\n\t\tWithLabel(\"suite\", \"nvidia\").\n\t\tWithLabel(\"hardware\", \"gpu\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tt.Log(\"Applying single node manifest\")\n\t\t\tvar err error\n\t\t\trenderedSingleNodeManifest, err = fwext.RenderManifests(mpiJobPytorchTrainingSingleNodeManifest, struct {\n\t\t\t\tPytorchTestImage string\n\t\t\t}{\n\t\t\t\tPytorchTestImage: testConfig.PytorchImage,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\terr = fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedSingleNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tt.Log(\"Manifest applied successfully\")\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"MPIJob succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tmpiJob := mpijobs.NewUnstructured(\"pytorch-training-single-node\", \"default\")\n\t\t\tctx = context.WithValue(ctx, \"mpiJob\", mpiJob)\n\t\t\tt.Log(\"Waiting for single node job to complete\")\n\t\t\terr := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).ResourceMatch(mpiJob, mpijobs.MPIJobSucceeded),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(30*time.Minute),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t} else {\n\t\t\t\tt.Log(\"Single node job completed\")\n\t\t\t}\n\t\t\tt.Logf(\"final mpijob resource: %v\", mpiJob)\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tjob := ctx.Value(\"mpiJob\")\n\t\t\tif job == nil {\n\t\t\t\t// nothing to do\n\t\t\t\treturn ctx\n\t\t\t}\n\t\t\tu, ok := job.(*unstructured.Unstructured)\n\t\t\tif !ok {\n\t\t\t\tt.Errorf(\"mpiJob in context is not unstructured: %v\", job)\n\t\t\t}\n\t\t\tlog, err := fwext.GetJobLogs(cfg.Client().RESTConfig(), u)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"failed to get job logs: %v\", err)\n\t\t\t}\n\t\t\tt.Log(\"Test log for pytorch-training-single-node:\")\n\t\t\tt.Log(log)\n\t\t\terr = fwext.DeleteManifests(cfg.Client().RESTConfig(), renderedSingleNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n}\n" }, { "path": "test/cases/nvidia/unit_test.go", "content": "//go:build e2e\n\npackage nvidia\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n)\n\nvar (\n\t//go:embed manifests/job-unit-test-single-node.yaml\n\tjobUnitTestSingleNodeManifest []byte\n\trenderedJobUnitTestSingleNodeManifest []byte\n\t//go:embed manifests/job-hpc-benchmarks.yaml\n\tjobHpcBenchmarksSingleNodeManifest []byte\n\trenderedJobHpcBenchmarksSingleNodeManifest []byte\n)\n\ntype unitTestManifestTplVars struct {\n\tNvidiaTestImage string\n\tSkipTestSubcommand string\n\tGpuPerNode int\n\tNodeType string\n}\n\ntype hpcTestManifestTplVars struct {\n\tGpuPerNode int\n}\n\nfunc TestSingleNodeUnitTest(t *testing.T) {\n\tunitTest := features.New(\"unit-test\").\n\t\tWithLabel(\"suite\", \"nvidia\").\n\t\tWithLabel(\"hardware\", \"gpu\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tif testConfig.NvidiaTestImage == \"\" {\n\t\t\t\tt.Fatal(fmt.Errorf(\"nvidiaTestImage must be set to run unit test, use https://github.com/aws/aws-k8s-tester/blob/main/test/images/nvidia/Dockerfile to build the image and -nvidiaTestImage to set the image url\"))\n\t\t\t}\n\t\t\tvar err error\n\t\t\trenderedJobUnitTestSingleNodeManifest, err = fwext.RenderManifests(jobUnitTestSingleNodeManifest, unitTestManifestTplVars{\n\t\t\t\tNvidiaTestImage: testConfig.NvidiaTestImage,\n\t\t\t\tSkipTestSubcommand: testConfig.SkipUnitTestSubcommand,\n\t\t\t\tGpuPerNode: gpuPerNode,\n\t\t\t\tNodeType: testConfig.NodeType,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\terr = fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedJobUnitTestSingleNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Unit test Job succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tjob := &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"unit-test-job\", Namespace: \"default\"},\n\t\t\t}\n\t\t\terr := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).JobSucceeded(job),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(60*time.Minute))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog, err := fwext.GetJobLogs(cfg.Client().RESTConfig(), &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"unit-test-job\", Namespace: \"default\"},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tt.Log(\"Test log for unit-test-job:\")\n\t\t\tt.Log(log)\n\t\t\terr = fwext.DeleteManifests(cfg.Client().RESTConfig(), renderedJobUnitTestSingleNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\thpcTest := features.New(\"hpc-benckmarks\").\n\t\tWithLabel(\"suite\", \"nvidia\").\n\t\tWithLabel(\"hardware\", \"gpu\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tvar err error\n\t\t\trenderedJobHpcBenchmarksSingleNodeManifest, err = fwext.RenderManifests(jobHpcBenchmarksSingleNodeManifest, hpcTestManifestTplVars{\n\t\t\t\tGpuPerNode: gpuPerNode,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\terr = fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedJobHpcBenchmarksSingleNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"HPC test Job succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tjob := &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"hpc-benckmarks-job\", Namespace: \"default\"},\n\t\t\t}\n\t\t\terr := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).JobSucceeded(job),\n\t\t\t\twait.WithContext(ctx))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog, err := fwext.GetJobLogs(cfg.Client().RESTConfig(), &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"hpc-benckmarks-job\", Namespace: \"default\"},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tt.Log(\"Test log for hpc-benckmarks-job:\")\n\t\t\tt.Log(log)\n\t\t\terr = fwext.DeleteManifests(cfg.Client().RESTConfig(), renderedJobHpcBenchmarksSingleNodeManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, unitTest, hpcTest)\n}\n" }, { "path": "test/cases/nvidia-dra/main_test.go", "content": "//go:build e2e\n\npackage nvidia_dra\n\nimport (\n\t\"context\"\n\t\"embed\"\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"path/filepath\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\t\"golang.org/x/sync/errgroup\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\n//go:embed rcts\nvar rctsFS embed.FS\n\nvar (\n\ttestenv env.Environment\n\tclientset kubernetes.Interface\n\tnodeType *string\n\trdmaDeviceDraDriverImage *string\n\tacceleratorDraDriverImage *string\n\tcontainerTestImage *string\n\tnodeCount int\n)\n\n// supportedRdmaTypes lists the recognized RDMA device types.\nvar supportedRdmaTypes = []string{\"efa\"}\n\nfunc validateConfig() error {\n\tif err := common.ValidateRequiredFlags(map[string]string{\n\t\t\"rdmaDeviceDraDriverImage\": *rdmaDeviceDraDriverImage,\n\t\t\"containerTestImage\": *containerTestImage,\n\t\t\"nodeType\": *nodeType,\n\t}); err != nil {\n\t\treturn err\n\t}\n\t// Validate that nodeType maps to a known topology (and thus a known RDMA type).\n\ttopo, err := GetTopologyForNodeType(*nodeType)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"invalid -nodeType: %w\", err)\n\t}\n\tif !slices.Contains(supportedRdmaTypes, topo.RdmaType) {\n\t\treturn fmt.Errorf(\"instance family %q has unsupported RDMA type %q; supported: %v\", topo.Family, topo.RdmaType, supportedRdmaTypes)\n\t}\n\t// Verify helm is available on the PATH.\n\tif _, err := exec.LookPath(\"helm\"); err != nil {\n\t\treturn fmt.Errorf(\"helm is required but not found on PATH: %w\", err)\n\t}\n\t// Verify kubectl is available on the PATH.\n\tif _, err := exec.LookPath(\"kubectl\"); err != nil {\n\t\treturn fmt.Errorf(\"kubectl is required but not found on PATH: %w\", err)\n\t}\n\treturn nil\n}\n\nconst (\n\tnvidiaDRAHelmReleaseName = \"nvidia-dra-driver-gpu\"\n\tnvidiaDRAHelmRepoName = \"nvidia-dra\"\n\tnvidiaDRAHelmRepoURL = \"https://helm.ngc.nvidia.com/nvidia\"\n\tnvidiaDRANamespace = \"nvidia-dra-driver-gpu\"\n\tnvidiaDRAHelmChartVer = \"25.8.1\"\n)\n\n// labelNodesGPUPresent labels all nodes with nvidia.com/gpu.present=true.\nfunc labelNodesGPUPresent(ctx context.Context) error {\n\targs := []string{\n\t\t\"label\", \"nodes\", \"--all\",\n\t\t\"nvidia.com/gpu.present=true\",\n\t\t\"--overwrite\",\n\t}\n\tlog.Printf(\"[INFO] Labeling nodes: kubectl %s\", strings.Join(args, \" \"))\n\tcmd := exec.CommandContext(ctx, \"kubectl\", args...)\n\tcmd.Stdout = os.Stdout\n\tcmd.Stderr = os.Stderr\n\tif err := cmd.Run(); err != nil {\n\t\treturn fmt.Errorf(\"kubectl label nodes failed: %w\", err)\n\t}\n\tlog.Println(\"All nodes labeled with nvidia.com/gpu.present=true.\")\n\treturn nil\n}\n\n// installNvidiaDRADriverHelm adds the NVIDIA Helm repo and installs the NVIDIA DRA driver.\n// If acceleratorDraDriverImage is non-empty, it splits on the last \":\" to extract\n// repository and tag and passes them as --set overrides.\nfunc installNvidiaDRADriverHelm(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t// Add the Helm repo.\n\trepoArgs := []string{\"repo\", \"add\", nvidiaDRAHelmRepoName, nvidiaDRAHelmRepoURL}\n\tlog.Printf(\"[INFO] Adding NVIDIA Helm repo: helm %s\", strings.Join(repoArgs, \" \"))\n\trepoCmd := exec.CommandContext(ctx, \"helm\", repoArgs...)\n\trepoCmd.Stdout = os.Stdout\n\trepoCmd.Stderr = os.Stderr\n\tif err := repoCmd.Run(); err != nil {\n\t\treturn ctx, fmt.Errorf(\"helm repo add nvidia-dra failed: %w\", err)\n\t}\n\n\t// Install (or upgrade) the chart.\n\targs := []string{\n\t\t\"upgrade\", \"--install\", nvidiaDRAHelmReleaseName,\n\t\tfmt.Sprintf(\"%s/%s\", nvidiaDRAHelmRepoName, nvidiaDRAHelmReleaseName),\n\t\t\"--version\", nvidiaDRAHelmChartVer,\n\t\t\"--create-namespace\",\n\t\t\"--namespace\", nvidiaDRANamespace,\n\t\t\"--set\", \"resources.gpus.enabled=true\",\n\t\t\"--set\", \"gpuResourcesEnabledOverride=true\",\n\t\t\"--timeout\", \"5m\",\n\t}\n\tif *acceleratorDraDriverImage != \"\" {\n\t\trepo, tag := common.SplitImageRepoTag(*acceleratorDraDriverImage)\n\t\targs = append(args,\n\t\t\t\"--set\", fmt.Sprintf(\"image.repository=%s\", repo),\n\t\t\t\"--set\", fmt.Sprintf(\"image.tag=%s\", tag),\n\t\t)\n\t}\n\tlog.Printf(\"[INFO] Installing NVIDIA DRA driver via Helm: helm %s\", strings.Join(args, \" \"))\n\tcmd := exec.CommandContext(ctx, \"helm\", args...)\n\tcmd.Stdout = os.Stdout\n\tcmd.Stderr = os.Stderr\n\tif err := cmd.Run(); err != nil {\n\t\treturn ctx, fmt.Errorf(\"helm install nvidia-dra-driver-gpu failed: %w\", err)\n\t}\n\tlog.Println(\"NVIDIA DRA driver Helm release installed successfully.\")\n\treturn ctx, nil\n}\n\n// uninstallNvidiaDRADriverHelm uninstalls the NVIDIA DRA driver Helm release.\nfunc uninstallNvidiaDRADriverHelm(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\targs := []string{\n\t\t\"uninstall\", nvidiaDRAHelmReleaseName,\n\t\t\"--namespace\", nvidiaDRANamespace,\n\t}\n\tlog.Printf(\"[INFO] Uninstalling NVIDIA DRA driver Helm release: helm %s\", strings.Join(args, \" \"))\n\tcmd := exec.CommandContext(ctx, \"helm\", args...)\n\tcmd.Stdout = os.Stdout\n\tcmd.Stderr = os.Stderr\n\tif err := cmd.Run(); err != nil {\n\t\tlog.Printf(\"[WARN] helm uninstall nvidia-dra-driver-gpu failed (may already be removed): %v\", err)\n\t}\n\treturn ctx, nil\n}\n\nfunc waitForNvidiaDRADriverReady(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tds := appsv1.DaemonSet{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"nvidia-dra-driver-gpu-kubelet-plugin\", Namespace: nvidiaDRANamespace},\n\t}\n\terr := wait.For(\n\t\tfwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&ds),\n\t\twait.WithTimeout(5*time.Minute),\n\t\twait.WithContext(ctx),\n\t)\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"nvidia-dra-driver daemonset is not ready: %w\", err)\n\t}\n\tlog.Println(\"nvidia-dra-driver daemonset is ready.\")\n\treturn ctx, nil\n}\n\nfunc TestMain(m *testing.M) {\n\tnodeType = flag.String(\"nodeType\", \"\", \"instance type for the cluster (e.g. p5.48xlarge)\")\n\trdmaDeviceDraDriverImage = flag.String(\"rdmaDeviceDraDriverImage\", \"\", \"container image for the dranet DRA driver\")\n\tacceleratorDraDriverImage = flag.String(\"acceleratorDraDriverImage\", \"\", \"container image for the NVIDIA DRA driver\")\n\tcontainerTestImage = flag.String(\"containerTestImage\", \"\", \"container image for the NCCL test workload\")\n\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\n\tif err := validateConfig(); err != nil {\n\t\tlog.Fatalf(\"invalid configuration: %v\", err)\n\t}\n\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = env.NewWithConfig(cfg).WithContext(ctx)\n\n\t// Resolve topology to determine RDMA type from nodeType.\n\ttopo, err := GetTopologyForNodeType(*nodeType)\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to resolve topology: %v\", err)\n\t}\n\n\tmanifestsList := [][]byte{\n\t\tmanifests.MpiOperatorManifest,\n\t}\n\tsetUpFunctions := []env.Func{\n\t\t// Run independent setup steps concurrently.\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tvar mu sync.Mutex\n\t\t\tg, gctx := errgroup.WithContext(ctx)\n\n\t\t\t// Deploy MPI operator.\n\t\t\tg.Go(func() error {\n\t\t\t\treturn common.DeployMPIOperator(gctx, config)\n\t\t\t})\n\n\t\t\t// Deploy dranet and RCTs based on topology's RDMA type.\n\t\t\tif topo.RdmaType == \"efa\" {\n\t\t\t\trctManifests, err := common.LoadRCTManifests(rctsFS, filepath.Join(\"rcts\", topo.RCTSubDir))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn ctx, fmt.Errorf(\"failed to load RCT manifests: %w\", err)\n\t\t\t\t}\n\t\t\t\tmu.Lock()\n\t\t\t\tmanifestsList = append(manifestsList, rctManifests...)\n\t\t\t\tmu.Unlock()\n\n\t\t\t\tg.Go(func() error {\n\t\t\t\t\trenderedDranet, err := common.DeployDranet(gctx, config, *rdmaDeviceDraDriverImage)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tmu.Lock()\n\t\t\t\t\tmanifestsList = append(manifestsList, renderedDranet)\n\t\t\t\t\tmu.Unlock()\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\n\t\t\t\tg.Go(func() error {\n\t\t\t\t\treturn fwext.ApplyManifests(config.Client().RESTConfig(), rctManifests...)\n\t\t\t\t})\n\t\t\t}\n\n\t\t\t// Label all nodes with nvidia.com/gpu.present=true.\n\t\t\tg.Go(func() error {\n\t\t\t\treturn labelNodesGPUPresent(gctx)\n\t\t\t})\n\n\t\t\t// Add NVIDIA Helm repo and install NVIDIA DRA driver.\n\t\t\tg.Go(func() error {\n\t\t\t\t_, err := installNvidiaDRADriverHelm(gctx, config)\n\t\t\t\treturn err\n\t\t\t})\n\n\t\t\tif err := g.Wait(); err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t\twaitForNvidiaDRADriverReady,\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tvar err error\n\t\t\tclientset, err = kubernetes.NewForConfig(config.Client().RESTConfig())\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, err\n\t\t\t}\n\t\t\tnodeCount, err = common.CountNodesByType(ctx, clientset, *nodeType)\n\t\t\treturn ctx, err\n\t\t},\n\t}\n\ttestenv.Setup(setUpFunctions...)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\t// Uninstall NVIDIA DRA driver Helm release first.\n\t\t\tctx, _ = uninstallNvidiaDRADriverHelm(ctx, config)\n\t\t\t// Delete remaining manifests in reverse order.\n\t\t\tslices.Reverse(manifestsList)\n\t\t\tif err := fwext.DeleteManifests(config.Client().RESTConfig(), manifestsList...); err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to delete manifests: %w\", err)\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/nvidia-dra/nvidia_dra_test.go", "content": "//go:build e2e\n\npackage nvidia_dra\n\nimport (\n\t\"embed\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n)\n\n//go:embed testcases\nvar embeddedTestCases embed.FS\n\nfunc TestNvidiaDRAMultiNode(t *testing.T) {\n\ttopo, err := GetTopologyForNodeType(*nodeType)\n\tif err != nil {\n\t\tt.Fatalf(\"resolving topology for %s: %v\", *nodeType, err)\n\t}\n\n\trctDir := filepath.Join(\"rcts\", topo.RCTSubDir)\n\trctIndex, err := common.LoadRCTIndex(rctsFS, rctDir)\n\tif err != nil {\n\t\tt.Fatalf(\"loading RCT index from %s: %v\", rctDir, err)\n\t}\n\n\ttcDir := filepath.Join(\"testcases\", topo.TestCaseSubDir)\n\n\tfeatureList, err := common.DiscoverAndBuildFeatures(\n\t\tembeddedTestCases,\n\t\ttcDir,\n\t\trctIndex,\n\t\t\"nvidia-dra\",\n\t\t\"multi-node-nccl-test\",\n\t\tnodeCount,\n\t\tfunc(tc *common.TestCaseSpec, rctIndex map[string]*common.ResourceClaimTemplateSpec) ([]byte, error) {\n\t\t\tparams, err := ComputeNvidiaMPIJobParams(tc, rctIndex, topo, nodeCount, *containerTestImage)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn RenderNvidiaMPIJobYAML(*params)\n\t\t},\n\t\tclientset,\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"discovering and building features: %v\", err)\n\t}\n\n\tif len(featureList) == 0 {\n\t\tt.Logf(\"no test cases found under %s, skipping\", tcDir)\n\t\treturn\n\t}\n\n\ttestenv.Test(t, featureList...)\n}\n" }, { "path": "test/cases/nvidia-dra/rcts/p5/rct-all-efas.yaml", "content": "apiVersion: resource.k8s.io/v1beta1\nkind: ResourceClaimTemplate\nmetadata:\n namespace: default\n name: rct-all-efas\nspec:\n spec:\n devices:\n requests:\n - name: all-efas\n deviceClassName: efa.networking.k8s.aws\n allocationMode: All\n" }, { "path": "test/cases/nvidia-dra/rcts/p5/rct-all-gpus.yaml", "content": "apiVersion: resource.k8s.io/v1beta1\nkind: ResourceClaimTemplate\nmetadata:\n namespace: default\n name: rct-all-gpus\nspec:\n spec:\n devices:\n requests:\n - name: all-gpus\n deviceClassName: gpu.nvidia.com\n allocationMode: All\n" }, { "path": "test/cases/nvidia-dra/rcts/p5/rct-five-efas-one-gpu.yaml", "content": "apiVersion: resource.k8s.io/v1beta1\nkind: ResourceClaimTemplate\nmetadata:\n namespace: default\n name: rct-five-efas-one-gpu\nspec:\n spec:\n devices:\n requests:\n - name: five-efas\n deviceClassName: efa.networking.k8s.aws\n allocationMode: ExactCount\n count: 5\n - name: one-gpu\n deviceClassName: gpu.nvidia.com\n allocationMode: ExactCount\n count: 1\n constraints:\n - requests: [\"five-efas\", \"one-gpu\"]\n matchAttribute: \"resource.kubernetes.io/pcieRoot\"\n" }, { "path": "test/cases/nvidia-dra/templates/nccl-test-mpijob.yaml.tmpl", "content": "apiVersion: kubeflow.org/v2beta1\nkind: MPIJob\nmetadata:\n name: multi-node-nccl-test\nspec:\n slotsPerWorker: {{.SlotsPerWorker}}\n runPolicy:\n backoffLimit: 20\n cleanPodPolicy: Running\n mpiReplicaSpecs:\n Launcher:\n replicas: 1\n template:\n spec:\n restartPolicy: OnFailure\n containers:\n - name: nccl-test-launcher\n image: {{.ContainerTestImage}}\n imagePullPolicy: IfNotPresent\n env:\n - name: PATH\n value: $PATH:/opt/amazon/efa/bin:/usr/bin\n command:\n - /opt/amazon/openmpi/bin/mpirun\n - --allow-run-as-root\n - --tag-output\n - -np\n - \"{{.TotalProcesses}}\"\n - -N\n - \"{{.SlotsPerWorker}}\"\n - --bind-to\n - none\n - -x\n - PATH\n - -x\n - LD_LIBRARY_PATH\n - -x\n - NCCL_DEBUG=INFO\n - -x\n - NCCL_BUFFSIZE=8388608\n - -x\n - NCCL_P2P_NET_CHUNKSIZE=524288\n - -x\n - NCCL_TUNER_PLUGIN=/opt/amazon/ofi-nccl/lib/x86_64-linux-gnu/libnccl-ofi-tuner.so\n - --mca\n - pml\n - ^cm,ucx\n - --mca\n - btl\n - tcp,self\n - --mca\n - btl_tcp_if_exclude\n - lo,docker0,veth_def_agent\n - /opt/nccl-tests/build/all_reduce_perf\n - -b\n - \"8\"\n - -e\n - \"16G\"\n - -f\n - \"2\"\n - -g\n - \"1\"\n - -c\n - \"1\"\n - -n\n - \"100\"\n Worker:\n replicas: {{.WorkerReplicas}}\n template:\n spec:\n containers:\n - name: nccl-tests-worker\n image: {{.ContainerTestImage}}\n imagePullPolicy: IfNotPresent\n volumeMounts:\n - name: shmem\n mountPath: /dev/shm\n resources:\n claims:\n{{- range .ResourceClaims}}\n - name: {{.Name}}\n{{- end}}\n resourceClaims:\n{{- range .ResourceClaims}}\n - name: {{.Name}}\n resourceClaimTemplateName: {{.TemplateName}}\n{{- end}}\n volumes:\n - name: shmem\n hostPath:\n path: /dev/shm\n" }, { "path": "test/cases/nvidia-dra/testcases/p5/all-efas-all-gpus.yaml", "content": "resourceClaims:\n- name: all-efas\n resourceClaimTemplateName: rct-all-efas\n- name: all-gpus\n resourceClaimTemplateName: rct-all-gpus\n" }, { "path": "test/cases/nvidia-dra/testcases/p5/five-efas-one-gpu-negative-test.yaml", "content": "expectFailure: true\nresourceClaims:\n- name: five-efas-one-gpu\n resourceClaimTemplateName: rct-five-efas-one-gpu\n" }, { "path": "test/cases/nvidia-dra/topology.go", "content": "package nvidia_dra\n\nimport (\n\t\"bytes\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"log\"\n\t\"strings\"\n\t\"text/template\"\n\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n)\n\n//go:embed templates/nccl-test-mpijob.yaml.tmpl\nvar mpijobTemplate string\n\n// ---------------------------------------------------------------------------\n// Instance topology\n// ---------------------------------------------------------------------------\n\n// NvidiaInstanceTopology describes the GPU/EFA hardware topology for an NVIDIA instance family.\ntype NvidiaInstanceTopology struct {\n\tFamily string\n\tGPUsPerNode int // total GPUs per node (e.g. 8 for p5.48xlarge)\n\tAllGPUCount int // same as GPUsPerNode for \"All\" allocation mode\n\tRdmaType string // RDMA device type (e.g. \"efa\")\n\tRCTSubDir string // subdirectory under rcts/\n\tTestCaseSubDir string // subdirectory under testcases/\n}\n\nvar instanceTopologies = map[string]NvidiaInstanceTopology{\n\t\"p5\": {\n\t\tFamily: \"p5\",\n\t\tGPUsPerNode: 8,\n\t\tAllGPUCount: 8,\n\t\tRdmaType: \"efa\",\n\t\tRCTSubDir: \"p5\",\n\t\tTestCaseSubDir: \"p5\",\n\t},\n}\n\n// GetTopologyForNodeType returns the NvidiaInstanceTopology for a given node type\n// (e.g. \"p5.48xlarge\"). It extracts the family prefix before the first \".\"\n// and looks it up in the registry.\nfunc GetTopologyForNodeType(nodeType string) (*NvidiaInstanceTopology, error) {\n\tfamily := common.ExtractFamily(nodeType)\n\ttopo, ok := instanceTopologies[family]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"unsupported instance family %q (from %q); supported: %s\",\n\t\t\tfamily, nodeType, supportedFamilies())\n\t}\n\treturn &topo, nil\n}\n\nfunc supportedFamilies() string {\n\tfamilies := make([]string, 0, len(instanceTopologies))\n\tfor k := range instanceTopologies {\n\t\tfamilies = append(families, k)\n\t}\n\treturn strings.Join(families, \", \")\n}\n\n// ---------------------------------------------------------------------------\n// MPIJob rendering\n// ---------------------------------------------------------------------------\n\n// NvidiaMPIJobParams holds all template parameters for rendering the NCCL MPIJob YAML.\ntype NvidiaMPIJobParams struct {\n\tSlotsPerWorker int\n\tTotalProcesses int\n\tWorkerReplicas int\n\tContainerTestImage string\n\tResourceClaims []common.ResourceClaimRef\n}\n\n// RenderNvidiaMPIJobYAML renders the embedded NCCL MPIJob Go template with the given params\n// and returns the resulting YAML bytes.\nfunc RenderNvidiaMPIJobYAML(params NvidiaMPIJobParams) ([]byte, error) {\n\ttmpl, err := template.New(\"mpijob\").Parse(mpijobTemplate)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"parsing MPIJob template: %w\", err)\n\t}\n\tvar buf bytes.Buffer\n\tif err := tmpl.Execute(&buf, params); err != nil {\n\t\treturn nil, fmt.Errorf(\"rendering MPIJob template: %w\", err)\n\t}\n\treturn buf.Bytes(), nil\n}\n\n// ---------------------------------------------------------------------------\n// NVIDIA-specific helpers\n// ---------------------------------------------------------------------------\n\n// getGPUCount returns the GPU device count from an RCT.\n// For AllocationMode \"All\" it returns the topology's AllGPUCount;\n// otherwise it returns the explicit Count from the gpu.nvidia.com request.\nfunc getGPUCount(rct *common.ResourceClaimTemplateSpec, topo *NvidiaInstanceTopology) int {\n\tfor _, req := range rct.Spec.Spec.Devices.Requests {\n\t\tif req.DeviceClassName != \"gpu.nvidia.com\" {\n\t\t\tcontinue\n\t\t}\n\t\tif req.AllocationMode == \"All\" {\n\t\t\treturn topo.AllGPUCount\n\t\t}\n\t\tif req.Count <= 0 {\n\t\t\tlog.Printf(\"[WARN] gpu.nvidia.com request has non-positive count: %d\", req.Count)\n\t\t}\n\t\treturn req.Count\n\t}\n\tlog.Printf(\"[WARN] no gpu.nvidia.com device request found in RCT, returning GPU count 0\")\n\treturn 0\n}\n\n// ComputeNvidiaMPIJobParams computes MPIJob parameters from a test case spec.\n// It resolves each claim's resourceClaimTemplateName against the RCT index to\n// get the GPU count, then calculates SlotsPerWorker and TotalProcesses.\nfunc ComputeNvidiaMPIJobParams(tc *common.TestCaseSpec, rctIndex map[string]*common.ResourceClaimTemplateSpec, topo *NvidiaInstanceTopology, workerReplicas int, containerTestImage string) (*NvidiaMPIJobParams, error) {\n\tif topo == nil {\n\t\treturn nil, fmt.Errorf(\"instance topology is required\")\n\t}\n\tif workerReplicas <= 0 {\n\t\treturn nil, fmt.Errorf(\"workerReplicas must be positive, got %d\", workerReplicas)\n\t}\n\tif containerTestImage == \"\" {\n\t\treturn nil, fmt.Errorf(\"containerTestImage is required\")\n\t}\n\n\ttotalGPUs := 0\n\tvar claims []common.ResourceClaimRef\n\n\tfor _, tcClaim := range tc.ResourceClaims {\n\t\trct, ok := rctIndex[tcClaim.ResourceClaimTemplateName]\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"resource claim template %q not found in RCT index\", tcClaim.ResourceClaimTemplateName)\n\t\t}\n\n\t\ttotalGPUs += getGPUCount(rct, topo)\n\n\t\tclaims = append(claims, common.ResourceClaimRef{\n\t\t\tName: tcClaim.Name,\n\t\t\tTemplateName: tcClaim.ResourceClaimTemplateName,\n\t\t})\n\t}\n\n\tslotsPerWorker := totalGPUs\n\ttotalProcesses := slotsPerWorker * workerReplicas\n\n\treturn &NvidiaMPIJobParams{\n\t\tSlotsPerWorker: slotsPerWorker,\n\t\tTotalProcesses: totalProcesses,\n\t\tWorkerReplicas: workerReplicas,\n\t\tContainerTestImage: containerTestImage,\n\t\tResourceClaims: claims,\n\t}, nil\n}\n" }, { "path": "test/cases/nvidia-inference/bert_inference_test.go", "content": "//go:build e2e\n\npackage inference\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\n//go:embed manifests/bert-inference.yaml\nvar bertInferenceManifest []byte\n\nvar renderedBertInferenceManifest []byte\n\ntype bertInferenceManifestTplVars struct {\n\tBertInferenceImage string\n\tInferenceMode string\n\tGPUPerNode string\n}\n\nfunc TestBertInference(t *testing.T) {\n\tfeature := features.New(\"bert-inference\").\n\t\tWithLabel(\"suite\", \"nvidia\").\n\t\tWithLabel(\"hardware\", \"gpu\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tif testConfig.BertInferenceImage == \"\" {\n\t\t\t\tt.Fatalf(\"[ERROR] bertInferenceImage must be set\")\n\t\t\t}\n\n\t\t\tlog.Println(\"[INFO] Rendering BERT inference manifest...\")\n\t\t\tvar err error\n\t\t\trenderedBertInferenceManifest, err = fwext.RenderManifests(\n\t\t\t\tbertInferenceManifest,\n\t\t\t\tbertInferenceManifestTplVars{\n\t\t\t\t\tBertInferenceImage: testConfig.BertInferenceImage,\n\t\t\t\t\tInferenceMode: testConfig.InferenceMode,\n\t\t\t\t\tGPUPerNode: fmt.Sprintf(\"%d\", testConfig.GpuRequested),\n\t\t\t\t},\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"[ERROR] Failed to render BERT inference manifest: %v\", err)\n\t\t\t}\n\n\t\t\tlog.Println(\"[INFO] Applying BERT inference manifest...\")\n\t\t\tif applyErr := fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedBertInferenceManifest); applyErr != nil {\n\t\t\t\tt.Fatalf(\"[ERROR] Failed to apply BERT inference manifest: %v\", applyErr)\n\t\t\t}\n\t\t\tlog.Println(\"[INFO] BERT inference manifest applied successfully.\")\n\n\t\t\t// Record time after applying the manifest\n\t\t\tctx = context.WithValue(ctx, \"applyTime\", time.Now())\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"BERT inference Job succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog.Println(\"[INFO] Checking BERT inference job completion...\")\n\t\t\tjob := &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"bert-inference\", Namespace: \"default\"},\n\t\t\t}\n\t\t\tif err := wait.For(\n\t\t\t\tfwext.NewConditionExtension(cfg.Client().Resources()).JobSucceeded(job),\n\t\t\t\twait.WithTimeout(20*time.Minute),\n\t\t\t); err != nil {\n\t\t\t\tlog.Println(\"[ERROR] BERT inference job failed. Gathering logs...\")\n\t\t\t\tif err := printJobLogs(ctx, cfg, \"default\", \"bert-inference\"); err != nil {\n\t\t\t\t\tt.Logf(\"[WARNING] Failed to retrieve bert-inference job logs: %v\", err)\n\t\t\t\t}\n\t\t\t\tt.Fatalf(\"[ERROR] BERT inference job did not succeed: %v\", err)\n\t\t\t}\n\n\t\t\tlog.Println(\"[INFO] BERT inference job succeeded. Gathering logs...\")\n\t\t\t// Compute duration from manifest apply to job success\n\t\t\tstartVal := ctx.Value(\"applyTime\")\n\t\t\tif startVal != nil {\n\t\t\t\tif applyTime, ok := startVal.(time.Time); ok {\n\t\t\t\t\tduration := time.Since(applyTime)\n\t\t\t\t\tlog.Printf(\"[INFO] BERT inference job completed in %s\", duration)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Print logs (including node name) for the Pod\n\t\t\tif err := printJobLogs(ctx, cfg, \"default\", \"bert-inference\"); err != nil {\n\t\t\t\tt.Logf(\"[WARNING] Failed to retrieve BERT inference job logs: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog.Println(\"[INFO] Cleaning up BERT inference job resources...\")\n\t\t\tif err := fwext.DeleteManifests(cfg.Client().RESTConfig(), renderedBertInferenceManifest); err != nil {\n\t\t\t\tt.Fatalf(\"[ERROR] Failed to delete BERT inference manifest: %v\", err)\n\t\t\t}\n\t\t\tlog.Println(\"[INFO] BERT inference job resources cleaned up.\")\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, feature)\n}\n\nfunc printJobLogs(ctx context.Context, cfg *envconf.Config, namespace, jobName string) error {\n\tcs, err := getClientset(cfg.Client().RESTConfig())\n\tif err != nil {\n\t\treturn fmt.Errorf(\"[ERROR] Failed to create kubernetes clientset: %w\", err)\n\t}\n\n\tpods, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{\n\t\tLabelSelector: fmt.Sprintf(\"job-name=%s\", jobName),\n\t})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"[ERROR] Failed to list pods for job %s: %w\", jobName, err)\n\t}\n\tif len(pods.Items) == 0 {\n\t\treturn fmt.Errorf(\"[ERROR] No pods found for job %s\", jobName)\n\t}\n\n\tfor _, pod := range pods.Items {\n\t\tlog.Printf(\"[INFO] Pod %s is running on node %s\", pod.Name, pod.Spec.NodeName)\n\n\t\tlog.Printf(\"[INFO] Retrieving logs from pod %s...\", pod.Name)\n\t\tstream, err := cs.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{}).Stream(ctx)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"[ERROR] Failed to get logs from pod %s: %w\", pod.Name, err)\n\t\t}\n\t\tdefer stream.Close()\n\n\t\tbuf := make([]byte, 4096)\n\t\tfor {\n\t\t\tn, readErr := stream.Read(buf)\n\t\t\tif n > 0 {\n\t\t\t\tlog.Printf(\"[INFO] Logs from Pod %s:\\n%s\", pod.Name, string(buf[:n]))\n\t\t\t}\n\t\t\tif readErr == io.EOF {\n\t\t\t\tlog.Printf(\"[INFO] Completed log stream for pod %s.\", pod.Name)\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif readErr != nil {\n\t\t\t\treturn fmt.Errorf(\"[ERROR] Failed to read logs from pod %s: %w\", pod.Name, readErr)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc getClientset(restConfig *rest.Config) (*kubernetes.Clientset, error) {\n\tcs, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"[ERROR] Cannot create kubernetes clientset: %w\", err)\n\t}\n\treturn cs, nil\n}\n" }, { "path": "test/cases/nvidia-inference/main_test.go", "content": "//go:build e2e\n\npackage inference\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"slices\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\ntype TestConfig struct {\n\tcommon.MetricOps\n\tBertInferenceImage string `flag:\"bertInferenceImage\" desc:\"BERT inference container image\"`\n\tInferenceMode string `flag:\"inferenceMode\" desc:\"Inference mode for BERT (throughput or latency)\"`\n\tGpuRequested int `flag:\"gpuRequested\" desc:\"Number of GPUs required for inference\"`\n}\n\nvar (\n\ttestenv env.Environment\n\ttestConfig TestConfig\n)\n\nfunc TestMain(m *testing.M) {\n\t// Initialize testConfig with default values\n\ttestConfig = TestConfig{\n\t\tInferenceMode: \"throughput\",\n\t\tGpuRequested: 1,\n\t}\n\n\t_, err := common.ParseFlags(&testConfig)\n\tif err != nil {\n\t\tlog.Fatalf(\"[ERROR] Failed to parse flags: %v\", err)\n\t}\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"[ERROR] Failed to initialize test environment: %v\", err)\n\t}\n\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = env.NewWithConfig(cfg).WithContext(ctx)\n\n\tmanifestsList := [][]byte{\n\t\tmanifests.NvidiaDevicePluginManifest,\n\t}\n\n\tif len(testConfig.MetricDimensions) > 0 {\n\t\t// Render CloudWatch Agent manifest with dynamic dimensions\n\t\trenderedCloudWatchAgentManifest, err := manifests.RenderCloudWatchAgentManifest(testConfig.MetricDimensions)\n\t\tif err != nil {\n\t\t\tlog.Printf(\"Warning: Failed to render CloudWatch Agent manifest: %v\", err)\n\t\t}\n\t\tmanifestsList = append(manifestsList, manifests.DCGMExporterManifest, renderedCloudWatchAgentManifest)\n\t}\n\n\ttestenv.Setup(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"[INFO] Applying manifests.\")\n\t\t\terr := fwext.ApplyManifests(config.Client().RESTConfig(), manifestsList...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"[ERROR] Failed to apply manifests: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"[INFO] Successfully applied manifests.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t\tcommon.DeployDaemonSet(\"nvidia-device-plugin-daemonset\", \"kube-system\"),\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tif len(testConfig.MetricDimensions) > 0 {\n\t\t\t\tif ctx, err := common.DeployDaemonSet(\"dcgm-exporter\", \"kube-system\")(ctx, config); err != nil {\n\t\t\t\t\treturn ctx, err\n\t\t\t\t}\n\t\t\t\tif ctx, err := common.DeployDaemonSet(\"cwagent\", \"amazon-cloudwatch\")(ctx, config); err != nil {\n\t\t\t\t\treturn ctx, err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t},\n\t\tcheckGpuCapacity,\n\t)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"[INFO] Deleting manifests.\")\n\t\t\tslices.Reverse(manifestsList)\n\t\t\terr := fwext.DeleteManifests(config.Client().RESTConfig(), manifestsList...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"[ERROR] failed to delete manifests: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"[INFO] Successfully deleted manifests.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\texitCode := testenv.Run(m)\n\tlog.Printf(\"[INFO] Tests finished with exit code %d\", exitCode)\n\tos.Exit(exitCode)\n}\n\n// checkGpuCapacity ensures at least one node has >= the requested number of GPUs,\n// and logs each node's instance type.\nfunc checkGpuCapacity(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tlog.Printf(\"[INFO] Validating cluster has at least %d GPU(s).\", testConfig.GpuRequested)\n\n\tcs, err := kubernetes.NewForConfig(config.Client().RESTConfig())\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to create kubernetes client: %w\", err)\n\t}\n\n\terr = wait.For(func(ctx context.Context) (bool, error) {\n\t\tnodes, err := cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\t\tif err != nil {\n\t\t\treturn false, fmt.Errorf(\"failed to list nodes: %w\", err)\n\t\t} else if len(nodes.Items) == 0 {\n\t\t\treturn false, fmt.Errorf(\"no nodes found in the cluster\")\n\t\t}\n\t\tfor _, node := range nodes.Items {\n\t\t\tinstanceType := node.Labels[\"node.kubernetes.io/instance-type\"]\n\t\t\tgpuCap, ok := node.Status.Capacity[\"nvidia.com/gpu\"]\n\t\t\tif ok && int(gpuCap.Value()) >= testConfig.GpuRequested {\n\t\t\t\tlog.Printf(\"[INFO] Node %s (type: %s) meets the request of %d GPU(s).\",\n\t\t\t\t\tnode.Name, instanceType, testConfig.GpuRequested)\n\t\t\t\treturn true, nil\n\t\t\t}\n\t\t\tlog.Printf(\"[INFO] Node %s (type: %s) has no GPU capacity.\", node.Name, instanceType)\n\t\t}\n\t\tlog.Printf(\"[INFO] No node meets the GPU requirement. The GPU info might not be propagated yet. Retrying...\")\n\t\treturn false, nil\n\t}, wait.WithTimeout(5*time.Minute), wait.WithInterval(10*time.Second))\n\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"no node has >= %d GPU(s)\", testConfig.GpuRequested)\n\t}\n\n\tlog.Println(\"[INFO] GPU capacity check passed.\")\n\treturn ctx, nil\n}\n" }, { "path": "test/cases/nvidia-inference/manifests/bert-inference.yaml", "content": "# Single-node BERT inference job with GPU. Memory-backed volume for /dev/shm\napiVersion: batch/v1\nkind: Job\nmetadata:\n name: bert-inference\nspec:\n backoffLimit: 4\n template:\n spec:\n restartPolicy: OnFailure\n volumes:\n - name: dshm\n emptyDir:\n medium: Memory\n containers:\n - name: bert-inference\n image: {{.BertInferenceImage}}\n imagePullPolicy: Always\n command: [\"python\", \"infer.py\"]\n env:\n - name: INFERENCE_MODE\n value: \"{{.InferenceMode}}\"\n volumeMounts:\n - mountPath: /dev/shm\n name: dshm\n resources:\n requests:\n nvidia.com/gpu: {{.GPUPerNode}}\n limits:\n nvidia.com/gpu: {{.GPUPerNode}}\n" }, { "path": "test/cases/nvidia-training/bert_training_test.go", "content": "//go:build e2e\n\npackage training\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\n// Use the parameterized manifest\nvar (\n\t//go:embed manifests/bert-training.yaml\n\tbertTrainingManifest []byte\n)\n\nfunc TestBertTraining(t *testing.T) {\n\tif testConfig.BertTrainingImage == \"\" {\n\t\tt.Fatal(fmt.Errorf(\"bertTrainingImage must be set to run the test\"))\n\t}\n\n\tslotsPerWorker := gpuPerNode\n\tworkerReplicas := nodeCount\n\tnp := slotsPerWorker * workerReplicas\n\tefaRequested := 0\n\tif testConfig.EfaEnabled && efaPerNode > 0 {\n\t\tefaRequested = 1\n\t}\n\n\tbertTraining := features.New(\"bert-training\").\n\t\tWithLabel(\"suite\", \"nvidia\").\n\t\tWithLabel(\"hardware\", \"gpu\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\trenderVars := map[string]string{\n\t\t\t\t\"BertTrainingImage\": testConfig.BertTrainingImage,\n\t\t\t\t\"SlotsPerWorker\": fmt.Sprintf(\"%d\", slotsPerWorker),\n\t\t\t\t\"NP\": fmt.Sprintf(\"%d\", np),\n\t\t\t\t\"WorkerReplicas\": fmt.Sprintf(\"%d\", workerReplicas),\n\t\t\t\t\"GPUPerNode\": fmt.Sprintf(\"%d\", gpuPerNode),\n\t\t\t\t\"EFARequested\": fmt.Sprintf(\"%d\", efaRequested),\n\t\t\t}\n\n\t\t\trenderedManifest, err := fwext.RenderManifests(bertTrainingManifest, renderVars)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\terr = fwext.ApplyManifests(cfg.Client().RESTConfig(), renderedManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"BERT training Job succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tjob := &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"bert-training-launcher\", Namespace: \"default\"},\n\t\t\t}\n\t\t\tif err := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).JobSucceeded(job),\n\t\t\t\twait.WithTimeout(time.Minute*20),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t); err != nil {\n\t\t\t\tt.Logf(\"[ERROR] BERT training job failed. Gathering logs...\")\n\t\t\t\tif err = printJobLogs(ctx, cfg, \"default\", \"bert-training-launcher\"); err != nil {\n\t\t\t\t\tt.Logf(\"Warning: failed to retrieve bert-training job logs: %v\", err)\n\t\t\t\t}\n\t\t\t\tt.Fatalf(\"[ERROR] BERT training job did not succeed: %v\", err)\n\t\t\t}\n\t\t\tt.Logf(\"[INFO] BERT training job succeeded. Gathering logs...\")\n\n\t\t\terr := printJobLogs(ctx, cfg, \"default\", \"bert-training-launcher\")\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"Warning: failed to retrieve bert-training job logs: %v\", err)\n\t\t\t}\n\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, bertTraining)\n}\n\nfunc printJobLogs(ctx context.Context, cfg *envconf.Config, namespace, jobName string) error {\n\tclientset, err := getClientset(cfg.Client().RESTConfig())\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to create kubernetes clientset: %w\", err)\n\t}\n\n\tpodList, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{\n\t\tLabelSelector: fmt.Sprintf(\"job-name=%s\", jobName),\n\t})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to list pods for job %s: %w\", jobName, err)\n\t}\n\n\tif len(podList.Items) == 0 {\n\t\treturn fmt.Errorf(\"no pods found for job %s\", jobName)\n\t}\n\n\tfor _, pod := range podList.Items {\n\t\treq := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{})\n\t\tlogStream, err := req.Stream(ctx)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to get logs from pod %s: %w\", pod.Name, err)\n\t\t}\n\t\tdefer logStream.Close()\n\n\t\tbuf := make([]byte, 4096)\n\t\tfor {\n\t\t\tn, err := logStream.Read(buf)\n\t\t\tif n > 0 {\n\t\t\t\tfmt.Printf(\"Logs from Pod %s: \\n%s\\n\", pod.Name, string(buf[:n]))\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc getClientset(restConfig *rest.Config) (*kubernetes.Clientset, error) {\n\tclientset, err := kubernetes.NewForConfig(restConfig)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to create kubernetes clientset: %w\", err)\n\t}\n\treturn clientset, nil\n}\n" }, { "path": "test/cases/nvidia-training/main_test.go", "content": "//go:build e2e\n\npackage training\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"fmt\"\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"slices\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nfunc TestMain(m *testing.M) {\n\t_, err := common.ParseFlags(&testConfig)\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to parse flags: %v\", err)\n\t}\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = env.NewWithConfig(cfg).WithContext(ctx)\n\n\tmanifestsList := [][]byte{\n\t\tmanifests.NvidiaDevicePluginManifest,\n\t\tmanifests.MpiOperatorManifest,\n\t\tmanifests.EfaDevicePluginManifest,\n\t}\n\n\tif len(testConfig.MetricDimensions) > 0 {\n\t\t// Render CloudWatch Agent manifest with dynamic dimensions\n\t\trenderedCloudWatchAgentManifest, err := manifests.RenderCloudWatchAgentManifest(testConfig.MetricDimensions)\n\t\tif err != nil {\n\t\t\tlog.Printf(\"Warning: failed to render CloudWatch Agent manifest: %v\", err)\n\t\t}\n\t\tmanifestsList = append(manifestsList, manifests.DCGMExporterManifest, renderedCloudWatchAgentManifest)\n\t}\n\n\ttestenv.Setup(\n\t\t// Apply all manifests\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Applying manifests.\")\n\n\t\t\terr := fwext.ApplyManifests(config.Client().RESTConfig(), manifestsList...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to apply manifests: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"Successfully applied manifests.\")\n\t\t\treturn ctx, nil\n\t\t},\n\n\t\t// Wait for MPI Operator deployment\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Waiting for MPI Operator deployment to be available.\")\n\t\t\tdeployment := appsv1.Deployment{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"mpi-operator\", Namespace: \"mpi-operator\"},\n\t\t\t}\n\t\t\terr := wait.For(\n\t\t\t\tconditions.New(config.Client().Resources()).DeploymentConditionMatch(\n\t\t\t\t\t&deployment, appsv1.DeploymentAvailable, v1.ConditionTrue,\n\t\t\t\t),\n\t\t\t\twait.WithTimeout(time.Minute*5),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"MPI Operator deployment is not available: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"MPI Operator deployment is available.\")\n\t\t\treturn ctx, nil\n\t\t},\n\n\t\t// Wait for required DaemonSets\n\t\tcommon.DeployDaemonSet(\"nvidia-device-plugin-daemonset\", \"kube-system\"),\n\t\tcommon.DeployDaemonSet(\"aws-efa-k8s-device-plugin-daemonset\", \"kube-system\"),\n\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tif len(testConfig.MetricDimensions) > 0 {\n\t\t\t\tif ctx, err := common.DeployDaemonSet(\"dcgm-exporter\", \"kube-system\")(ctx, config); err != nil {\n\t\t\t\t\treturn ctx, err\n\t\t\t\t}\n\t\t\t\tif ctx, err := common.DeployDaemonSet(\"cwagent\", \"amazon-cloudwatch\")(ctx, config); err != nil {\n\t\t\t\t\treturn ctx, err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn ctx, nil\n\t\t}, // Deploy CloudWatch Agent + DCGM only if MetricDimensions are set\n\n\t\tcheckNodeTypes, // Dynamically check node types and capacities after device plugins are ready\n\t)\n\n\ttestenv.Finish(\n\t\tfunc(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\t\tlog.Println(\"Deleting NVIDIA device plugin, MPI operator, EFA device plugin DCGM Exporter and CloudWatch Agent manifests.\")\n\t\t\tslices.Reverse(manifestsList)\n\t\t\terr := fwext.DeleteManifests(config.Client().RESTConfig(), manifestsList...)\n\t\t\tif err != nil {\n\t\t\t\treturn ctx, fmt.Errorf(\"failed to delete manifests: %w\", err)\n\t\t\t}\n\t\t\tlog.Println(\"Successfully deleted NVIDIA device plugin, MPI operator, EFA device plugin, DCGM Exporter and CloudWatch Agent manifests.\")\n\t\t\treturn ctx, nil\n\t\t},\n\t)\n\n\tlog.Println(\"Starting tests...\")\n\texitCode := testenv.Run(m)\n\tlog.Printf(\"Tests finished with exit code %d\", exitCode)\n\tos.Exit(exitCode)\n}\n\nfunc checkNodeTypes(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\tclientset, err := kubernetes.NewForConfig(config.Client().RESTConfig())\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to create Kubernetes client: %w\", err)\n\t}\n\n\tnodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})\n\tif err != nil {\n\t\treturn ctx, fmt.Errorf(\"failed to list nodes: %w\", err)\n\t}\n\n\tif len(nodes.Items) == 0 {\n\t\treturn ctx, fmt.Errorf(\"no nodes found in the cluster\")\n\t}\n\n\tfor i := 1; i < len(nodes.Items); i++ {\n\t\tif nodes.Items[i].Labels[\"node.kubernetes.io/instance-type\"] != nodes.Items[i-1].Labels[\"node.kubernetes.io/instance-type\"] {\n\t\t\treturn ctx, fmt.Errorf(\"node types are not the same, all node types must be the same in the cluster\")\n\t\t}\n\t}\n\n\tif testConfig.NodeType != \"\" {\n\t\tcount := 0\n\t\tfor _, v := range nodes.Items {\n\t\t\tif v.Labels[\"node.kubernetes.io/instance-type\"] == testConfig.NodeType {\n\t\t\t\tcount++\n\t\t\t\tif gpuCap, ok := v.Status.Capacity[\"nvidia.com/gpu\"]; ok {\n\t\t\t\t\tgpuPerNode = int(gpuCap.Value())\n\t\t\t\t}\n\t\t\t\tif efaCap, ok := v.Status.Capacity[\"vpc.amazonaws.com/efa\"]; ok {\n\t\t\t\t\tefaPerNode = int(efaCap.Value())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif count == 0 {\n\t\t\treturn ctx, fmt.Errorf(\"no nodes match the specified nodeType: %s\", testConfig.NodeType)\n\t\t}\n\t\tnodeCount = count\n\t} else {\n\t\ttestConfig.NodeType = nodes.Items[0].Labels[\"node.kubernetes.io/instance-type\"]\n\t\tnodeCount = len(nodes.Items)\n\t\tif gpuCap, ok := nodes.Items[0].Status.Capacity[\"nvidia.com/gpu\"]; ok {\n\t\t\tgpuPerNode = int(gpuCap.Value())\n\t\t}\n\t\tif efaCap, ok := nodes.Items[0].Status.Capacity[\"vpc.amazonaws.com/efa\"]; ok {\n\t\t\tefaPerNode = int(efaCap.Value())\n\t\t}\n\t}\n\n\tlog.Printf(\"[INFO] Node Type: %s\", testConfig.NodeType)\n\tlog.Printf(\"[INFO] Node Count: %d\", nodeCount)\n\tlog.Printf(\"[INFO] GPU Per Node: %d\", gpuPerNode)\n\tlog.Printf(\"[INFO] EFA Per Node: %d\", efaPerNode)\n\n\treturn ctx, nil\n}\n" }, { "path": "test/cases/nvidia-training/manifests/bert-training.yaml", "content": "apiVersion: kubeflow.org/v2beta1\nkind: MPIJob\nmetadata:\n name: bert-training\nspec:\n slotsPerWorker: {{.SlotsPerWorker}}\n runPolicy:\n backoffLimit: 20\n cleanPodPolicy: Running\n mpiReplicaSpecs:\n Launcher:\n replicas: 1\n template:\n spec:\n restartPolicy: OnFailure\n containers:\n - image: {{.BertTrainingImage}}\n imagePullPolicy: Always\n name: bert-training\n env:\n - name: NCCL_DEBUG\n value: \"TRACE\"\n - name: MASTER_ADDR\n value: \"bert-training\"\n - name: MASTER_PORT\n value: \"12355\"\n command:\n - /opt/amazon/openmpi/bin/mpirun\n - --allow-run-as-root\n - --tag-output\n - -np\n - \"{{.NP}}\" # Number of processes derived from node/gpu calculations\n - -bind-to\n - none\n - -map-by\n - slot\n - -x\n - PATH\n - -x\n - LD_LIBRARY_PATH\n - -x\n - NCCL_DEBUG\n - -x\n - MASTER_ADDR\n - -x\n - MASTER_PORT\n - --mca \n - pml\n - \"^cm\"\n - --mca\n - routed\n - direct\n - --oversubscribe\n - --mca\n - orte_base_help_aggregate \n - \"0\"\n - python\n - train.py\n Worker:\n replicas: {{.WorkerReplicas}}\n template:\n spec:\n volumes:\n - name: dshm\n emptyDir:\n medium: Memory\n containers:\n - image: {{.BertTrainingImage}}\n imagePullPolicy: Always\n name: bert-training-worker\n volumeMounts:\n - mountPath: /dev/shm\n name: dshm\n resources:\n requests:\n nvidia.com/gpu: {{.GPUPerNode}}\n vpc.amazonaws.com/efa: {{.EFARequested}}\n limits:\n nvidia.com/gpu: {{.GPUPerNode}}\n vpc.amazonaws.com/efa: {{.EFARequested}}\n" }, { "path": "test/cases/nvidia-training/vars.go", "content": "//go:build e2e\n\npackage training\n\nimport (\n\t\"github.com/aws/aws-k8s-tester/test/common\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n)\n\ntype Config struct {\n\tcommon.MetricOps\n\tBertTrainingImage string `flag:\"bertTrainingImage\" desc:\"Docker image used for BERT training workload\"`\n\tEfaEnabled bool `flag:\"efaEnabled\" desc:\"Enable Elastic Fabric Adapter (EFA)\"`\n\tNodeType string `flag:\"nodeType\" desc:\"Instance type for cluster nodes\"`\n}\n\n// Shared global variables\nvar (\n\ttestenv env.Environment\n\ttestConfig Config\n\n\tnodeCount int\n\tgpuPerNode int\n\tefaPerNode int\n)\n" }, { "path": "test/cases/quick/io_uring_test.go", "content": "//go:build e2e\n\npackage quick\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\n\t\"sigs.k8s.io/e2e-framework/klient/k8s\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nfunc TestNpmInstallWithCPULimits(t *testing.T) {\n\tfeat := features.New(\"npm-install\").\n\t\tWithLabel(\"suite\", \"quick\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog.Println(\"[Setup] Verifying cluster nodes...\")\n\t\t\tvar nodeList corev1.NodeList\n\t\t\tif err := cfg.Client().Resources().List(ctx, &nodeList); err != nil {\n\t\t\t\tt.Fatalf(\"Failed to list nodes: %v\", err)\n\t\t\t}\n\n\t\t\t// Log node information\n\t\t\tfor _, node := range nodeList.Items {\n\t\t\t\tarch := node.Labels[\"kubernetes.io/arch\"]\n\t\t\t\tkernelVersion := node.Status.NodeInfo.KernelVersion\n\t\t\t\tt.Logf(\"Node: %s, Architecture: %s, Kernel: %s\", node.Name, arch, kernelVersion)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Pod can successfully run npm install\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tpodName := \"npm-install-test\"\n\t\t\tpodNS := \"default\"\n\n\t\t\tpod := &corev1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tName: podName,\n\t\t\t\t\tNamespace: podNS,\n\t\t\t\t\tLabels: map[string]string{\n\t\t\t\t\t\t\"app\": \"npm-install-test\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tSpec: corev1.PodSpec{\n\t\t\t\t\tContainers: []corev1.Container{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName: \"test-container\",\n\t\t\t\t\t\t\tImage: \"public.ecr.aws/ubuntu/ubuntu:noble\",\n\t\t\t\t\t\t\tCommand: []string{\"/bin/sh\", \"-c\"},\n\t\t\t\t\t\t\tArgs: []string{`\n\t\t\t\t\t\t\t\tset -x\n\t\t\t\t\t\t\t\techo \"[Test] Starting npm installation test...\"\n\t\t\t\t\t\t\t\tmkdir asd && \n\t\t\t\t\t\t\t\tcd asd && \n\t\t\t\t\t\t\t\tapt-get update && \n\t\t\t\t\t\t\t\tapt-get install -y npm nodejs && \n\t\t\t\t\t\t\t\techo \"[Test] Starting npm install webpack...\"\n\t\t\t\t\t\t\t\tnpm install webpack --loglevel verbose || exit 1\n\t\t\t\t\t\t\t\techo \"[Test] npm install completed successfully\"\n\t\t\t\t\t\t\t`},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tRestartPolicy: corev1.RestartPolicyNever,\n\t\t\t\t},\n\t\t\t}\n\n\t\t\tif err := cfg.Client().Resources().Create(ctx, pod); err != nil {\n\t\t\t\tt.Fatalf(\"[Assess] Failed to create pod: %v\", err)\n\t\t\t}\n\n\t\t\tlog.Printf(\"[Assess] Waiting up to 10 minutes for pod %s to complete...\", podName)\n\t\t\terr := wait.For(\n\t\t\t\te2e.NewConditionExtension(cfg.Client().Resources()).ResourceMatch(pod, func(object k8s.Object) bool {\n\t\t\t\t\tpod := object.(*corev1.Pod)\n\t\t\t\t\treturn pod.Status.Phase == corev1.PodSucceeded\n\t\t\t\t}),\n\t\t\t\twait.WithTimeout(10*time.Minute),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Logf(\"[Assess] Pod did not complete successfully: %v\", err)\n\t\t\t\te2e.PrintDaemonSetPodLogs(t, ctx, cfg.Client().RESTConfig(), podNS, \"app=npm-install-test\")\n\t\t\t\tt.Fatal(\"Pod did not complete within 10 minutes - possible io_uring hang detected\")\n\t\t\t}\n\n\t\t\tlog.Printf(\"[Assess] Pod %s completed successfully\", podName)\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tpodName := \"npm-install-test\"\n\t\t\tpodNS := \"default\"\n\n\t\t\tt.Logf(\"[Teardown] Cleaning up pod %s/%s...\", podNS, podName)\n\t\t\tpod := &corev1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tName: podName,\n\t\t\t\t\tNamespace: podNS,\n\t\t\t\t},\n\t\t\t}\n\t\t\tif err := cfg.Client().Resources().Delete(ctx, pod); err != nil {\n\t\t\t\tt.Logf(\"[Teardown] Failed to delete pod: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, feat)\n}\n" }, { "path": "test/cases/quick/limit_test.go", "content": "//go:build e2e\n\npackage quick\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t_ \"embed\"\n\t\"io\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/k8s\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nvar (\n\t//go:embed manifests/ulimit.yaml\n\tulimitManifest []byte\n\n\texpectedResourceLimit = map[string]string{\n\t\t\"-R\": \"unlimited\",\n\t\t\"-c\": \"unlimited\",\n\t\t\"-d\": \"unlimited\",\n\t\t\"-e\": \"0\",\n\t\t\"-f\": \"unlimited\",\n\t\t\"-i\": \"30446\",\n\t\t\"-l\": \"unlimited\",\n\t\t\"-m\": \"unlimited\",\n\t\t\"-n\": \"1048576\",\n\t\t\"-p\": \"8\",\n\t\t\"-q\": \"819200\",\n\t\t\"-r\": \"0\",\n\t\t\"-s\": \"10240\",\n\t\t\"-t\": \"unlimited\",\n\t\t\"-u\": \"unlimited\",\n\t\t\"-v\": \"unlimited\",\n\t\t\"-x\": \"unlimited\",\n\t}\n)\n\nfunc TestUserLimits(t *testing.T) {\n\tf1 := features.New(\"ulimit pod\").\n\t\tWithLabel(\"type\", \"ulimit\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\terr := fwext.ApplyManifests(cfg.Client().RESTConfig(), ulimitManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"failed to apply manifests: %v\", err)\n\t\t\t}\n\t\t\tpod := &corev1.Pod{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: \"ulimit\", Namespace: \"default\"},\n\t\t\t}\n\t\t\terr = wait.For(conditions.New(cfg.Client().Resources()).ResourceMatch(pod, containerTerminated),\n\t\t\t\twait.WithTimeout(time.Minute*5))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"encounter error when waiting for container finished running commands: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Use default resources limit\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tclient, err := kubernetes.NewForConfig(cfg.Client().RESTConfig())\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\ttailLine := int64(10000)\n\t\t\tpodLogOptions := corev1.PodLogOptions{\n\t\t\t\tContainer: \"al2023\",\n\t\t\t\tTailLines: &tailLine,\n\t\t\t}\n\t\t\treq := client.CoreV1().Pods(\"default\").GetLogs(\"ulimit\", &podLogOptions)\n\t\t\tlogs, err := req.Stream(ctx)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"error in opening stream: %v\", err)\n\t\t\t}\n\t\t\tdefer logs.Close()\n\t\t\tcompareResourceLimitsWithExpectedValues(t, logs)\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\terr := fwext.DeleteManifests(cfg.Client().RESTConfig(), ulimitManifest)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"failed to delete manifests: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).Feature()\n\n\t// test feature\n\ttestenv.Test(t, f1)\n}\n\nfunc compareResourceLimitsWithExpectedValues(t *testing.T, logs io.ReadCloser) {\n\tbuf := new(bytes.Buffer)\n\t_, err := io.Copy(buf, logs)\n\tif err != nil {\n\t\tt.Fatalf(\"error in copy information from podLogs to buf: %v\", err)\n\t}\n\tstr := buf.String()\n\n\tlines := strings.Split(str, \"\\n\")\n\tfor _, line := range lines[:len(lines)-1] {\n\t\tinfo := strings.Split(line, \" \")\n\t\tmarker := getMarker(info[len(info)-2])\n\t\tvalue := info[len(info)-1]\n\t\tif expectedResourceLimit[marker] != value {\n\t\t\tt.Errorf(\"resource limit doesn't match with the default value, limit we get %v, but default value is %v\", line, expectedResourceLimit[marker])\n\t\t} else {\n\t\t\tt.Logf(\"resrouce limit fetched from ulimit: %v. Equal to the default value %v\", line, expectedResourceLimit[marker])\n\t\t}\n\t}\n}\n\nfunc containerTerminated(obj k8s.Object) bool {\n\tj := obj.(*corev1.Pod)\n\tcontainerTerminatedState := j.Status.ContainerStatuses[0].State.Terminated\n\treturn containerTerminatedState.Reason == \"Completed\"\n}\n\nfunc getMarker(str string) string {\n\tstartIndex := 0\n\tif str[:1] == \"(\" {\n\t\tstartIndex = 1\n\t}\n\treturn str[startIndex : len(str)-1]\n}\n" }, { "path": "test/cases/quick/main_test.go", "content": "//go:build e2e\n\npackage quick\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"testing\"\n\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nvar (\n\ttestenv env.Environment\n)\n\nfunc TestMain(m *testing.M) {\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\ttestenv = env.NewWithConfig(cfg)\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = testenv.WithContext(ctx)\n\n\ttestenv.Setup(func(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\tlog.Println(\"Starting quick test suite...\")\n\t\treturn ctx, nil\n\t})\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/quick/manifests/ulimit.yaml", "content": "apiVersion: v1\nkind: Pod\nmetadata:\n name: ulimit\nspec:\n restartPolicy: Never\n containers:\n - name: al2023\n image: public.ecr.aws/amazonlinux/amazonlinux:2023\n command: [\"ulimit\"]\n args:\n - -a\n" }, { "path": "test/cases/quick/node_topology_test.go", "content": "//go:build e2e\n\npackage quick\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv1 \"k8s.io/api/core/v1\"\n\tcloudprovider \"k8s.io/cloud-provider-aws/pkg/providers/v1\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nfunc TestNodeTopology(t *testing.T) {\n\ttopology := features.New(\"node-topology\").\n\t\tWithLabel(\"suite\", \"node-topology\").\n\t\tAssess(\"Nodes have correct network topology labels\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\n\t\t\tvar nodes v1.NodeList\n\t\t\tcfg.Client().Resources().List(ctx, &nodes)\n\n\t\t\tif len(nodes.Items) == 0 {\n\t\t\t\tt.Fatal(\"no nodes found in the cluster\")\n\t\t\t}\n\n\t\t\tnodeMap := make(map[string]v1.Node)\n\t\t\tvar instanceIDs []string\n\t\t\tec2Client := e2e.NewEC2Client()\n\t\t\tfor _, node := range nodes.Items {\n\t\t\t\tproviderIDParts := strings.Split(node.Spec.ProviderID, \"/\")\n\t\t\t\tinstanceID := providerIDParts[len(providerIDParts)-1]\n\t\t\t\tinstanceIDs = append(instanceIDs, instanceID)\n\t\t\t\tnodeMap[instanceID] = node\n\t\t\t}\n\n\t\t\tnodeTopologies, err := ec2Client.DescribeInstanceTopology(instanceIDs)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"could not describe instance topologies: %v\", err)\n\t\t\t}\n\n\t\t\tt.Logf(\"checking instance topologies for %d node(s) (out of %d node(s) in the cluster)\", len(nodeTopologies), len(instanceIDs))\n\n\t\t\tfor _, nodeTopology := range nodeTopologies {\n\t\t\t\tnode := nodeMap[aws.ToString(nodeTopology.InstanceId)]\n\t\t\t\tinstanceType := node.Labels[\"node.kubernetes.io/instance-type\"]\n\n\t\t\t\tt.Logf(\"verifying instance topology for node %s (type: %s)\", node.Name, instanceType)\n\n\t\t\t\tfor i, networkNode := range nodeTopology.NetworkNodes {\n\t\t\t\t\t// https://github.com/kubernetes/cloud-provider-aws/blob/b47d2cf2a33ae655cd353ec42ea43362b804c397/pkg/providers/v1/well_known_labels.go#L26\n\t\t\t\t\texpectedLabel := cloudprovider.LabelNetworkNodePrefix + strconv.Itoa(i+1)\n\t\t\t\t\tif actualValue, ok := node.Labels[expectedLabel]; !ok {\n\t\t\t\t\t\tt.Errorf(\"node %s (type: %s) does not have expected network label %s\", node.Name, instanceType, expectedLabel)\n\t\t\t\t\t} else if actualValue != networkNode {\n\t\t\t\t\t\tt.Errorf(\"node %s (type: %s) has incorrect value for label %s: expected %s, got %s\", node.Name, instanceType, expectedLabel, networkNode, actualValue)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// https://github.com/kubernetes/cloud-provider-aws/blob/b47d2cf2a33ae655cd353ec42ea43362b804c397/pkg/providers/v1/well_known_labels.go#L22C2-L22C13\n\t\t\t\tif aws.ToString(nodeTopology.ZoneId) != node.Labels[cloudprovider.LabelZoneID] {\n\t\t\t\t\tt.Logf(\"node %s (type: %s) has incorrect value for label %s: expected %s, got %s\", node.Name, instanceType, cloudprovider.LabelZoneID, aws.ToString(nodeTopology.ZoneId), node.Labels[cloudprovider.LabelZoneID])\n\t\t\t\t\tt.Fail()\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn ctx\n\t\t}).Feature()\n\n\ttestenv.Test(t, topology)\n}\n" }, { "path": "test/cases/workload/main_test.go", "content": "//go:build e2e\n\npackage workload\n\nimport (\n\t\"context\"\n\t\"flag\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"testing\"\n\t\"time\"\n\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\nconst (\n\tdefaultWorkloadTestTimeout = 10 * time.Minute\n)\n\nvar (\n\ttestenv env.Environment\n\tworkloadTestCommand *string\n\tworkloadTestImage *string\n\tworkloadTestName *string\n\tworkloadTestResources *string\n\tworkloadTestTimeout *time.Duration\n)\n\nfunc TestMain(m *testing.M) {\n\tworkloadTestCommand = flag.String(\"workloadTestCommand\", \"\", \"command for workload test\")\n\tworkloadTestImage = flag.String(\"workloadTestImage\", \"\", \"image for workload test\")\n\tworkloadTestName = flag.String(\"workloadTestName\", \"workload-test\", \"name for workload test\")\n\tworkloadTestResources = flag.String(\"workloadTestResources\", \"\", \"JSON map of resources for workload test (e.g., '{\\\"nvidia.com/gpu\\\": \\\"1\\\"}')\")\n\tworkloadTestTimeout = flag.Duration(\"workloadTestTimeout\", defaultWorkloadTestTimeout, fmt.Sprintf(\"timeout for workload test (default: %s)\", defaultWorkloadTestTimeout))\n\tcfg, err := envconf.NewFromFlags()\n\tif err != nil {\n\t\tlog.Fatalf(\"failed to initialize test environment: %v\", err)\n\t}\n\ttestenv = env.NewWithConfig(cfg)\n\tctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)\n\tdefer cancel()\n\ttestenv = testenv.WithContext(ctx)\n\n\ttestenv.Setup(func(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\tlog.Println(\"Starting workload test suite...\")\n\t\treturn ctx, nil\n\t})\n\n\tos.Exit(testenv.Run(m))\n}\n" }, { "path": "test/cases/workload/workload_test.go", "content": "//go:build e2e\n\npackage workload\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/smithy-go/ptr\"\n\tbatchv1 \"k8s.io/api/batch/v1\"\n\tcorev1 \"k8s.io/api/core/v1\"\n\t\"k8s.io/apimachinery/pkg/api/resource\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nfunc createWorkloadJob(name, image, command string, resources map[string]string, timeout time.Duration) *batchv1.Job {\n\tcontainer := corev1.Container{\n\t\tName: name,\n\t\tImage: image,\n\t\tImagePullPolicy: corev1.PullAlways,\n\t\tResources: buildResourceRequirements(resources),\n\t}\n\n\t// Override entrypoint if command is provided\n\tif command != \"\" {\n\t\tcontainer.Command = strings.Fields(command)\n\t}\n\n\tpodSpec := corev1.PodSpec{\n\t\tRestartPolicy: corev1.RestartPolicyNever,\n\t\tActiveDeadlineSeconds: ptr.Int64(int64(timeout.Seconds())),\n\t\tContainers: []corev1.Container{container},\n\t}\n\n\tjob := &batchv1.Job{\n\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\tName: name,\n\t\t\tNamespace: corev1.NamespaceDefault,\n\t\t\tLabels: map[string]string{\"app\": name},\n\t\t},\n\t\tSpec: batchv1.JobSpec{\n\t\t\tBackoffLimit: ptr.Int32(4),\n\t\t\tTemplate: corev1.PodTemplateSpec{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{\n\t\t\t\t\tLabels: map[string]string{\"app\": name},\n\t\t\t\t},\n\t\t\t\tSpec: podSpec,\n\t\t\t},\n\t\t},\n\t}\n\n\treturn job\n}\n\nfunc buildResourceRequirements(resources map[string]string) corev1.ResourceRequirements {\n\tif len(resources) == 0 {\n\t\treturn corev1.ResourceRequirements{}\n\t}\n\trl := make(corev1.ResourceList)\n\tfor name, qty := range resources {\n\t\trl[corev1.ResourceName(name)] = resource.MustParse(qty)\n\t}\n\treturn corev1.ResourceRequirements{Limits: rl, Requests: rl}\n}\n\nfunc parseResources(resourcesJSON string) (map[string]string, error) {\n\tif resourcesJSON == \"\" {\n\t\treturn nil, nil\n\t}\n\tvar resources map[string]string\n\tif err := json.Unmarshal([]byte(resourcesJSON), &resources); err != nil {\n\t\treturn nil, err\n\t}\n\tfor name, qty := range resources {\n\t\tif q, err := resource.ParseQuantity(qty); err != nil || q.IsZero() {\n\t\t\tdelete(resources, name)\n\t\t}\n\t}\n\treturn resources, nil\n}\n\nfunc TestWorkload(t *testing.T) {\n\tname := ptr.ToString(workloadTestName)\n\timage := ptr.ToString(workloadTestImage)\n\tcommand := ptr.ToString(workloadTestCommand)\n\ttimeout := ptr.ToDuration(workloadTestTimeout)\n\n\tif name == \"\" {\n\t\tt.Fatal(\"workloadTestName must be set to run the test\")\n\t}\n\tif image == \"\" {\n\t\tt.Fatal(\"workloadTestImage must be set to run the test\")\n\t}\n\n\tresources, err := parseResources(ptr.ToString(workloadTestResources))\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to parse workloadTestResources: %v\", err)\n\t}\n\n\tfeature := features.New(name).WithLabel(\"suite\", \"workload\")\n\tif _, ok := resources[\"aws.amazon.com/neuron\"]; ok {\n\t\tfeature = feature.WithLabel(\"hardware\", \"neuron\")\n\t}\n\tif _, ok := resources[\"nvidia.com/gpu\"]; ok {\n\t\tfeature = feature.WithLabel(\"hardware\", \"gpu\")\n\t}\n\n\tworkload := feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\tjob := createWorkloadJob(name, image, command, resources, timeout)\n\t\tif len(resources) > 0 {\n\t\t\tt.Logf(\"Creating %s job with resources: %v\", name, resources)\n\t\t} else {\n\t\t\tt.Logf(\"Creating %s job\", name)\n\t\t}\n\t\tif err := cfg.Client().Resources().Create(ctx, job); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tt.Logf(\"%s job created successfully\", name)\n\t\treturn ctx\n\t}).\n\t\tAssess(\"Job succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tjob := &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: name, Namespace: corev1.NamespaceDefault},\n\t\t\t}\n\t\t\tt.Logf(\"Waiting for %s job to complete\", name)\n\t\t\terr := wait.For(fwext.NewConditionExtension(cfg.Client().Resources()).JobSucceeded(job),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(timeout),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tlog, err := fwext.GetJobLogs(cfg.Client().RESTConfig(), &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: name, Namespace: corev1.NamespaceDefault},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\tt.Logf(\"Test log for %s:\", name)\n\t\t\tt.Log(log)\n\t\t\tjob := &batchv1.Job{\n\t\t\t\tObjectMeta: metav1.ObjectMeta{Name: name, Namespace: corev1.NamespaceDefault},\n\t\t\t}\n\t\t\tif err := cfg.Client().Resources().Delete(ctx, job, func(do *metav1.DeleteOptions) {\n\t\t\t\tpolicy := metav1.DeletePropagationBackground\n\t\t\t\tdo.PropagationPolicy = &policy\n\t\t\t}); err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n\n\ttestenv.Test(t, workload)\n}\n" }, { "path": "test/common/dra.go", "content": "//go:build e2e\n\npackage common\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/test/manifests\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tv1 \"k8s.io/api/core/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\n// DeployDranet renders the dranet manifest template with the given image,\n// applies it to the cluster, and waits for the dranet DaemonSet to be ready.\n// Returns the rendered manifest bytes for later cleanup.\nfunc DeployDranet(ctx context.Context, config *envconf.Config, rdmaDeviceDraDriverImage string) (renderedManifest []byte, err error) {\n\trenderedManifest, err = fwext.RenderManifests(manifests.DranetManifest, struct {\n\t\tRdmaDeviceDraDriverImage string\n\t}{\n\t\tRdmaDeviceDraDriverImage: rdmaDeviceDraDriverImage,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to render dranet manifest: %w\", err)\n\t}\n\tif err := fwext.ApplyManifests(config.Client().RESTConfig(), renderedManifest); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to apply dranet manifest: %w\", err)\n\t}\n\tds := appsv1.DaemonSet{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"dranet-aws-dranet\", Namespace: \"kube-system\"},\n\t}\n\terr = wait.For(\n\t\tfwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&ds),\n\t\twait.WithTimeout(5*time.Minute),\n\t\twait.WithContext(ctx),\n\t)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"dranet daemonset is not ready: %w\", err)\n\t}\n\tlog.Println(\"dranet daemonset is ready.\")\n\treturn renderedManifest, nil\n}\n\n// CountNodesByType lists cluster nodes and returns the count of nodes matching\n// the given node.kubernetes.io/instance-type label. Returns an error if the\n// count is 0.\nfunc CountNodesByType(ctx context.Context, clientset kubernetes.Interface, nodeType string) (int, error) {\n\tnodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{\n\t\tLabelSelector: \"node.kubernetes.io/instance-type=\" + nodeType,\n\t})\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"failed to list nodes: %w\", err)\n\t}\n\tcount := len(nodes.Items)\n\tif count == 0 {\n\t\treturn 0, fmt.Errorf(\"no nodes of type %q found\", nodeType)\n\t}\n\tlog.Printf(\"[INFO] Found %d node(s) of type %s\", count, nodeType)\n\treturn count, nil\n}\n\n// DeployMPIOperator applies the MPI operator manifest and waits for the\n// mpi-operator Deployment to become available.\nfunc DeployMPIOperator(ctx context.Context, config *envconf.Config) error {\n\tif err := fwext.ApplyManifests(config.Client().RESTConfig(), manifests.MpiOperatorManifest); err != nil {\n\t\treturn fmt.Errorf(\"failed to apply mpi-operator manifest: %w\", err)\n\t}\n\tdep := appsv1.Deployment{\n\t\tObjectMeta: metav1.ObjectMeta{Name: \"mpi-operator\", Namespace: \"mpi-operator\"},\n\t}\n\terr := wait.For(conditions.New(config.Client().Resources()).DeploymentConditionMatch(&dep, appsv1.DeploymentAvailable, v1.ConditionTrue),\n\t\twait.WithContext(ctx))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"failed to deploy mpi-operator: %w\", err)\n\t}\n\tlog.Println(\"mpi-operator deployment is available.\")\n\treturn nil\n}\n" }, { "path": "test/common/dra_features.go", "content": "//go:build e2e\n\npackage common\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io/fs\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\t\"github.com/aws/aws-k8s-tester/internal/e2e/mpijobs\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait/conditions\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n\t\"sigs.k8s.io/e2e-framework/pkg/features\"\n)\n\nconst (\n\t// NegativeTestTimeout is the duration to wait before checking that a\n\t// negative test case's worker pods are still Pending.\n\tNegativeTestTimeout = 1 * time.Minute\n\t// NegativeTestStabilizationTimeout is the duration to wait after pods\n\t// are first observed as Pending before re-checking they remain Pending.\n\tNegativeTestStabilizationTimeout = 2 * time.Minute\n\t// PositiveTestTimeout is the duration to wait for an MPIJob to succeed.\n\tPositiveTestTimeout = 20 * time.Minute\n)\n\n// ComputeAndRenderFunc is a callback that computes MPIJob parameters and renders\n// the MPIJob YAML for a given test case. Each package provides its own implementation\n// that calls its package-specific ComputeMPIJobParams and RenderMPIJobYAML functions.\ntype ComputeAndRenderFunc func(tc *TestCaseSpec, rctIndex map[string]*ResourceClaimTemplateSpec) (renderedYAML []byte, err error)\n\n// BuildPositiveFeature constructs an e2e-framework Feature for a positive DRA\n// test case. It applies the manifest, waits for the MPIJob to succeed, retrieves\n// logs, and cleans up.\nfunc BuildPositiveFeature(name, suiteName, mpiJobName string, manifest []byte) features.Feature {\n\treturn features.New(name).\n\t\tWithLabel(\"suite\", suiteName).\n\t\tWithLabel(\"type\", \"positive\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tt.Logf(\"Applying MPIJob manifest for %s\", name)\n\t\t\tif err := fwext.ApplyManifests(cfg.Client().RESTConfig(), manifest); err != nil {\n\t\t\t\tt.Fatalf(\"applying MPIJob manifest: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"MPIJob succeeds\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tmpiJob := mpijobs.NewUnstructured(mpiJobName, \"default\")\n\t\t\tt.Log(\"Waiting for MPIJob to complete\")\n\t\t\terr := wait.For(\n\t\t\t\tconditions.New(cfg.Client().Resources()).ResourceMatch(mpiJob, mpijobs.MPIJobSucceeded),\n\t\t\t\twait.WithContext(ctx),\n\t\t\t\twait.WithTimeout(PositiveTestTimeout),\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"MPIJob did not succeed: %v\", err)\n\t\t\t}\n\n\t\t\tlog, err := fwext.GetJobLogs(cfg.Client().RESTConfig(), mpiJob)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"failed to get job logs: %v\", err)\n\t\t\t} else {\n\t\t\t\tt.Logf(\"Test log for %s:\", name)\n\t\t\t\tt.Log(log)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tif err := fwext.DeleteManifests(cfg.Client().RESTConfig(), manifest); err != nil {\n\t\t\t\tt.Errorf(\"deleting MPIJob manifest: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n}\n\n// BuildNegativeFeature constructs an e2e-framework Feature for a negative DRA\n// test case. It applies the manifest, waits for a timeout, verifies worker pods\n// remain Pending, and cleans up.\nfunc BuildNegativeFeature(name, suiteName, mpiJobName string, manifest []byte, expectedPendingCount int, clientset kubernetes.Interface) features.Feature {\n\treturn features.New(name).\n\t\tWithLabel(\"suite\", suiteName).\n\t\tWithLabel(\"type\", \"negative\").\n\t\tSetup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tt.Logf(\"Applying MPIJob manifest for negative test %s\", name)\n\t\t\tif err := fwext.ApplyManifests(cfg.Client().RESTConfig(), manifest); err != nil {\n\t\t\t\tt.Fatalf(\"applying MPIJob manifest: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tAssess(\"Worker pods remain Pending\", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tt.Log(\"Waiting for worker pods to be Pending...\")\n\t\t\tselector := fmt.Sprintf(\"training.kubeflow.org/job-name=%s,training.kubeflow.org/job-role=worker\", mpiJobName)\n\t\t\tlistOpts := metav1.ListOptions{\n\t\t\t\tLabelSelector: selector,\n\t\t\t\tFieldSelector: \"status.phase=Pending\",\n\t\t\t}\n\t\t\terr := wait.For(func(ctx context.Context) (bool, error) {\n\t\t\t\tpods, err := clientset.CoreV1().Pods(\"default\").List(ctx, listOpts)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn false, nil\n\t\t\t\t}\n\t\t\t\treturn len(pods.Items) >= expectedPendingCount, nil\n\t\t\t}, wait.WithContext(ctx), wait.WithTimeout(NegativeTestTimeout))\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"expected %d worker pods in Pending state: %v\", expectedPendingCount, err)\n\t\t\t}\n\t\t\tt.Logf(\"Found %d Pending worker pods, waiting %v to confirm they remain unschedulable...\", expectedPendingCount, NegativeTestStabilizationTimeout)\n\t\t\ttime.Sleep(NegativeTestStabilizationTimeout)\n\t\t\tpods, err := clientset.CoreV1().Pods(\"default\").List(ctx, listOpts)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"re-checking Pending pods: %v\", err)\n\t\t\t}\n\t\t\tif len(pods.Items) < expectedPendingCount {\n\t\t\t\tt.Fatalf(\"expected %d Pending worker pods after stabilization, but found %d\", expectedPendingCount, len(pods.Items))\n\t\t\t}\n\t\t\tt.Logf(\"All %d worker pods are still Pending after stabilization (scheduling failure confirmed)\", expectedPendingCount)\n\t\t\treturn ctx\n\t\t}).\n\t\tTeardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {\n\t\t\tif err := fwext.DeleteManifests(cfg.Client().RESTConfig(), manifest); err != nil {\n\t\t\t\tt.Errorf(\"deleting MPIJob manifest: %v\", err)\n\t\t\t}\n\t\t\treturn ctx\n\t\t}).\n\t\tFeature()\n}\n\n// DiscoverAndBuildFeatures encapsulates the common test discovery loop:\n// 1. Reads test case YAML files from testCasesFS at testCaseDir\n// 2. Parses each via ParseTestCaseSpec\n// 3. Invokes computeAndRender to get the rendered MPIJob YAML\n// 4. Builds positive or negative features based on ExpectFailure\nfunc DiscoverAndBuildFeatures(\n\ttestCasesFS fs.FS,\n\ttestCaseDir string,\n\trctIndex map[string]*ResourceClaimTemplateSpec,\n\tsuiteName string,\n\tmpiJobName string,\n\tnodeCount int,\n\tcomputeAndRender ComputeAndRenderFunc,\n\tclientset kubernetes.Interface,\n) ([]features.Feature, error) {\n\tentries, err := fs.ReadDir(testCasesFS, testCaseDir)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"reading test case directory %s: %w\", testCaseDir, err)\n\t}\n\n\tvar featureList []features.Feature\n\tfor _, entry := range entries {\n\t\tif entry.IsDir() || !IsYAMLFile(entry.Name()) {\n\t\t\tcontinue\n\t\t}\n\n\t\ttcName := strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name()))\n\t\ttcPath := filepath.Join(testCaseDir, entry.Name())\n\n\t\ttcData, err := fs.ReadFile(testCasesFS, tcPath)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"reading test case %s: %w\", tcPath, err)\n\t\t}\n\n\t\ttc, err := ParseTestCaseSpec(tcData)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"parsing test case %s: %w\", tcPath, err)\n\t\t}\n\n\t\trenderedYAML, err := computeAndRender(tc, rctIndex)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"computing/rendering MPIJob for %s: %w\", tcName, err)\n\t\t}\n\n\t\tif tc.ExpectFailure {\n\t\t\tfeatureList = append(featureList, BuildNegativeFeature(tcName, suiteName, mpiJobName, renderedYAML, nodeCount, clientset))\n\t\t} else {\n\t\t\tfeatureList = append(featureList, BuildPositiveFeature(tcName, suiteName, mpiJobName, renderedYAML))\n\t\t}\n\t}\n\treturn featureList, nil\n}\n" }, { "path": "test/common/dra_types.go", "content": "package common\n\nimport (\n\t\"fmt\"\n\t\"io/fs\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\tyaml \"gopkg.in/yaml.v2\"\n)\n\n// ---------------------------------------------------------------------------\n// Test case spec — what the user authors per test\n// ---------------------------------------------------------------------------\n\n// TestCaseClaimRef is a single entry in a test case YAML file.\ntype TestCaseClaimRef struct {\n\tName string `yaml:\"name\"`\n\tResourceClaimTemplateName string `yaml:\"resourceClaimTemplateName\"`\n}\n\n// TestCaseSpec is the structure of a test case YAML file.\n// Each file defines the resourceClaims that a single MPIJob test should use.\n// When ExpectFailure is true, the test runner treats the case as a negative test —\n// it expects the MPIJob's worker pods to remain Pending (unschedulable).\ntype TestCaseSpec struct {\n\tExpectFailure bool `yaml:\"expectFailure\"`\n\tResourceClaims []TestCaseClaimRef `yaml:\"resourceClaims\"`\n}\n\n// ---------------------------------------------------------------------------\n// ResourceClaimTemplate parsing\n// ---------------------------------------------------------------------------\n\n// ResourceClaimTemplateSpec mirrors the relevant parts of a ResourceClaimTemplate YAML.\ntype ResourceClaimTemplateSpec struct {\n\tMetadata struct {\n\t\tName string `yaml:\"name\"`\n\t} `yaml:\"metadata\"`\n\tSpec struct {\n\t\tSpec struct {\n\t\t\tDevices struct {\n\t\t\t\tRequests []struct {\n\t\t\t\t\tName string `yaml:\"name\"`\n\t\t\t\t\tDeviceClassName string `yaml:\"deviceClassName\"`\n\t\t\t\t\tAllocationMode string `yaml:\"allocationMode\"`\n\t\t\t\t\tCount int `yaml:\"count\"`\n\t\t\t\t} `yaml:\"requests\"`\n\t\t\t} `yaml:\"devices\"`\n\t\t} `yaml:\"spec\"`\n\t} `yaml:\"spec\"`\n}\n\n// ---------------------------------------------------------------------------\n// MPIJob rendering helpers\n// ---------------------------------------------------------------------------\n\n// ResourceClaimRef holds the name and template name for a single resource claim\n// in the rendered MPIJob.\ntype ResourceClaimRef struct {\n\tName string\n\tTemplateName string\n}\n\n// ---------------------------------------------------------------------------\n// Parsing helpers\n// ---------------------------------------------------------------------------\n\n// ParseTestCaseSpec parses YAML bytes into a TestCaseSpec.\n// It returns an error if the YAML is invalid or contains no resourceClaims.\nfunc ParseTestCaseSpec(data []byte) (*TestCaseSpec, error) {\n\tvar tc TestCaseSpec\n\tif err := yaml.Unmarshal(data, &tc); err != nil {\n\t\treturn nil, fmt.Errorf(\"parsing test case YAML: %w\", err)\n\t}\n\tif len(tc.ResourceClaims) == 0 {\n\t\treturn nil, fmt.Errorf(\"test case has no resourceClaims\")\n\t}\n\treturn &tc, nil\n}\n\n// IsYAMLFile reports whether the given filename has a .yaml or .yml extension.\nfunc IsYAMLFile(name string) bool {\n\text := filepath.Ext(name)\n\treturn ext == \".yaml\" || ext == \".yml\"\n}\n\n// LoadRCTIndex scans a directory of RCT YAML files from the given fs.FS and\n// returns a map of metadata.name → parsed spec.\nfunc LoadRCTIndex(fsys fs.FS, dir string) (map[string]*ResourceClaimTemplateSpec, error) {\n\tentries, err := fs.ReadDir(fsys, dir)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"reading RCT directory %s: %w\", dir, err)\n\t}\n\tindex := make(map[string]*ResourceClaimTemplateSpec)\n\tfor _, entry := range entries {\n\t\tif entry.IsDir() || !IsYAMLFile(entry.Name()) {\n\t\t\tcontinue\n\t\t}\n\t\tdata, err := fs.ReadFile(fsys, filepath.Join(dir, entry.Name()))\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"reading %s: %w\", entry.Name(), err)\n\t\t}\n\t\tvar rct ResourceClaimTemplateSpec\n\t\tif err := yaml.Unmarshal(data, &rct); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"parsing %s: %w\", entry.Name(), err)\n\t\t}\n\t\tindex[rct.Metadata.Name] = &rct\n\t}\n\treturn index, nil\n}\n\n// ExtractFamily extracts the instance family prefix from a node type string\n// (before the first \".\"). For example, \"trn1.32xlarge\" returns \"trn1\".\nfunc ExtractFamily(nodeType string) string {\n\tif idx := strings.Index(nodeType, \".\"); idx > 0 {\n\t\treturn nodeType[:idx]\n\t}\n\treturn nodeType\n}\n\n// ---------------------------------------------------------------------------\n// Runtime helpers\n// ---------------------------------------------------------------------------\n\n// SplitImageRepoTag splits a container image reference on the last \":\" into\n// repository and tag. If there is no \":\", the entire string is treated as the\n// repository and the tag defaults to \"latest\".\nfunc SplitImageRepoTag(image string) (repo, tag string) {\n\tidx := strings.LastIndex(image, \":\")\n\tif idx < 0 {\n\t\treturn image, \"latest\"\n\t}\n\treturn image[:idx], image[idx+1:]\n}\n\n// ValidateRequiredFlags validates that all flag values in the provided map are\n// non-empty. Returns a descriptive error for the first missing flag, or nil if\n// all flags are present.\nfunc ValidateRequiredFlags(flags map[string]string) error {\n\tfor name, value := range flags {\n\t\tif value == \"\" {\n\t\t\treturn fmt.Errorf(\"-%s is required and must be non-empty\", name)\n\t\t}\n\t}\n\treturn nil\n}\n\n// LoadRCTManifests reads all YAML files from the given RCT subdirectory in an\n// embedded filesystem and returns them as raw byte slices suitable for\n// fwext.ApplyManifests.\nfunc LoadRCTManifests(fsys fs.FS, rctSubDir string) ([][]byte, error) {\n\tentries, err := fs.ReadDir(fsys, rctSubDir)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"reading RCT directory %s: %w\", rctSubDir, err)\n\t}\n\tvar manifests [][]byte\n\tfor _, entry := range entries {\n\t\tif entry.IsDir() || !IsYAMLFile(entry.Name()) {\n\t\t\tcontinue\n\t\t}\n\t\tdata, err := fs.ReadFile(fsys, filepath.Join(rctSubDir, entry.Name()))\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"reading %s: %w\", entry.Name(), err)\n\t\t}\n\t\tmanifests = append(manifests, data)\n\t}\n\treturn manifests, nil\n}\n" }, { "path": "test/common/flags.go", "content": "//go:build e2e\n\npackage common\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"github.com/urfave/sflags/gen/gpflag\"\n\t\"github.com/spf13/pflag\"\n\t\"reflect\"\n)\n\n// For CloudWatch metric dimension flag\ntype MetricOps struct {\n\t// gpflag supports map[string]string but with a different non-standard parsing format (key:val) that doesn't match\n\t// what the project wants (comma separated key=value pairs). So, we force it to skip parsing under gpflag.Parse.\n\tMetricDimensions map[string]string `flag:\"-\"`\n}\n\nfunc ParseFlags(config interface{}) (*pflag.FlagSet, error) {\n\tflags, err := gpflag.Parse(config)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse flags: %w\", err)\n\t}\n\n\t// gpflag supports map[string]string but with a different non-standard parsing format (key:val) that doesn't\n\t// match what the project wants (key=val,key=val). So, we handle MetricDimensions separately here to accept\n\t// comma separated key=value pairs.\n\tif _, hasField := reflect.TypeOf(config).Elem().FieldByName(\"MetricDimensions\"); hasField {\n\t\tfield := reflect.ValueOf(config).Elem().FieldByName(\"MetricDimensions\")\n\t\tmetricDims := field.Addr().Interface().(*map[string]string)\n\t\tflags.StringToStringVar(metricDims, \"metricDimensions\", nil, \"CloudWatch metric dimensions as comma-separated key=value pairs\")\n\t}\n\n\tflags.VisitAll(func(pf *pflag.Flag) {\n\t\tflag.CommandLine.Var(pf.Value, pf.Name, pf.Usage)\n\t})\n\n\treturn flags, nil\n}\n" }, { "path": "test/common/resources.go", "content": "//go:build e2e\n\npackage common\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"time\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n\tappsv1 \"k8s.io/api/apps/v1\"\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"sigs.k8s.io/e2e-framework/klient/wait\"\n\t\"sigs.k8s.io/e2e-framework/pkg/env\"\n\t\"sigs.k8s.io/e2e-framework/pkg/envconf\"\n)\n\n// DeployDaemonSet returns a function to deploy and wait for a DaemonSet to be ready\nfunc DeployDaemonSet(name, namespace string) env.Func {\n\treturn func(ctx context.Context, config *envconf.Config) (context.Context, error) {\n\t\tlog.Printf(\"Waiting for %s daemonset to be ready.\", name)\n\t\tdaemonset := appsv1.DaemonSet{\n\t\t\tObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},\n\t\t}\n\t\terr := wait.For(\n\t\t\tfwext.NewConditionExtension(config.Client().Resources()).DaemonSetReady(&daemonset),\n\t\t\twait.WithTimeout(5*time.Minute),\n\t\t\twait.WithContext(ctx),\n\t\t)\n\t\tif err != nil {\n\t\t\treturn ctx, fmt.Errorf(\"%s daemonset is not ready: %w\", name, err)\n\t\t}\n\t\tlog.Printf(\"%s daemonset is ready.\", name)\n\t\treturn ctx, nil\n\t}\n}\n" }, { "path": "test/images/efa/Dockerfile", "content": "FROM public.ecr.aws/amazonlinux/amazonlinux:2023\n\nARG EFA_BIN_PATH=\"/opt/amazon/efa/bin\"\n\nRUN dnf -y swap gnupg2-minimal gnupg2 && \\\n dnf install -y \\\n gcc gcc-c++ make \\ \n ca-certificates \\\n cmake \\\n emacs \\\n git \\\n jq \\\n wget \\\n unzip \\\n vim \\\n zlib-devel \\ \n openssl \\\n openssl-devel \\ \n sqlite-devel \\ \n gdbm-devel \\ \n glibc-devel \\ \n bzip2-devel \\ \n ncurses-devel \\ \n tk-devel \\ \n libffi-devel \\ \n libcap-devel \\ \n tar \\\n gnupg2 \n\nENV PATH=\"$PATH:$EFA_BIN_PATH\"\n\nRUN cd $HOME \\\n && curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz \\\n && wget https://efa-installer.amazonaws.com/aws-efa-installer.key && gpg --import aws-efa-installer.key \\\n && cat aws-efa-installer.key | gpg --fingerprint \\\n && wget https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz.sig && gpg --verify ./aws-efa-installer-latest.tar.gz.sig \\\n && tar -xf aws-efa-installer-latest.tar.gz \\\n && cd aws-efa-installer \\\n && ./efa_installer.sh -y -d --skip-kmod --skip-limit-conf --no-verify \\\n # TODO: remove this in favor of letting the efa installer add it if that ever becomes an option.\n # At the moment, this is only installed if omitting --no-verify, which would require\n # building in a context with EFA available\n && install -T -m 0755 efa_test.sh \"${EFA_BIN_PATH}/efa_test.sh\" \\\n && cd $HOME \\\n && rm -rf aws-efa-installer\n\nRUN dnf clean all\n\nRUN INSTALL_DIR=$(mktemp -d) && \\\n cd $INSTALL_DIR && \\\n curl \"https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip\" -o \"awscliv2.zip\" && \\\n unzip awscliv2.zip && \\\n ./aws/install && \\\n cd && \\ \n rm -rf $INSTALL_DIR\n\nCOPY test/images/efa/scripts ./scripts\n\nRUN chmod -R +x ./scripts" }, { "path": "test/images/efa/scripts/unit-test.sh", "content": "#!/usr/bin/env bash\n\nset -eu\n\nget_instance_type()\n{\n\n local token=$(curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\" 2>/dev/null)\n\n if [ -n \"$token\" ]; then\n curl -H \"X-aws-ec2-metadata-token: $token\" http://169.254.169.254/latest/meta-data/instance-type\n else\n curl http://169.254.169.254/latest/meta-data/instance-type\n fi\n}\n\nget_expected_efa_device_count() \n{\n aws ec2 describe-instance-types --instance-type=\"$EC2_INSTANCE_TYPE\" | jq -r '.InstanceTypes[].NetworkInfo.EfaInfo.MaximumEfaInterfaces'\n}\n\nEC2_INSTANCE_TYPE=${EC2_INSTANCE_TYPE:-$(get_instance_type)}\nEXPECTED_EFA_DEVICE_COUNT=${EXPECTED_EFA_DEVICE_COUNT:-$(get_expected_efa_device_count)}\n\necho \"Running test on a $EC2_INSTANCE_TYPE\"\n\nfi_info -p efa\nDGRAM_ENDPOINT_COUNT=$(fi_info -p efa | grep 'type:\\sFI_EP_DGRAM$' | wc -l)\nif ! test $EXPECTED_EFA_DEVICE_COUNT -le $DGRAM_ENDPOINT_COUNT; then\n echo \"Expected at least $EXPECTED_EFA_DEVICE_COUNT DGRAM endpoint(s) but found $DGRAM_ENDPOINT_COUNT\"\n exit 1\nelse\n echo \"Verified at least $EXPECTED_EFA_DEVICE_COUNT DGRAM endpoint(s) are available (found $DGRAM_ENDPOINT_COUNT)\"\nfi\n\nRDM_ENDPOINT_COUNT=$(fi_info -p efa | grep 'type:\\sFI_EP_RDM$' | wc -l)\nif ! test $EXPECTED_EFA_DEVICE_COUNT -le $RDM_ENDPOINT_COUNT; then\n echo \"Expected at least $EXPECTED_EFA_DEVICE_COUNT RDM endpoint(s) but found $RDM_ENDPOINT_COUNT\"\n exit 1\nelse\n echo \"Verified at least $EXPECTED_EFA_DEVICE_COUNT RDM endpoint(s) are available (found $RDM_ENDPOINT_COUNT)\"\nfi\n\n\necho \"Running single-node efa test\"\n\n# Run efa_test.sh, a utility added during the build while installing EFA\nefa_test.sh\n\necho \"Success!\"" }, { "path": "test/images/neuron/Dockerfile", "content": "FROM public.ecr.aws/docker/library/ubuntu:22.04\n\n# Neuron SDK components version numbers\n# https://github.com/aws-neuron/deep-learning-containers/blob/main/docker/pytorch/training/2.5.1/Dockerfile.neuronx\nARG NEURONX_DISTRIBUTED_VERSION=0.16.25997+f431c02e\nARG NEURONX_CC_VERSION=2.22.12471.0+b4a00d10\nARG NEURONX_FRAMEWORK_VERSION=2.9.0.2.11.19912+e48cd891\nARG NEURONX_COLLECTIVES_LIB_VERSION=2.29.41.0-681fef5f5\nARG NEURONX_RUNTIME_LIB_VERSION=2.29.40.0-f954cd7a5\nARG NEURONX_TOOLS_VERSION=2.27.33.0-5d9c0b901\n\nARG PYTHON=python3.10\nARG PYTHON_VERSION=3.10.12\nARG PIP=pip3\nARG OMPI_VERSION=4.1.5\n\n# This arg required to stop docker build waiting for region configuration while installing tz data from ubuntu 20\nARG DEBIAN_FRONTEND=noninteractive\n\n# Python won’t try to write .pyc or .pyo files on the import of source modules\n# Force stdin, stdout and stderr to be totally unbuffered. Good for logging\nENV PYTHONDONTWRITEBYTECODE=1\nENV PYTHONUNBUFFERED=1\nENV PYTHONIOENCODING=UTF-8\nENV LANG=C.UTF-8\nENV LC_ALL=C.UTF-8\nENV LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/opt/aws/neuron/lib\"\nENV LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/opt/amazon/efa/lib\"\nENV LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/opt/amazon/efa/lib64\"\nENV LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/opt/amazon/openmpi/lib64\"\nENV LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/usr/local/lib\"\nENV PATH /opt/aws/neuron/bin/:$PATH\n# ENV SAGEMAKER_TRAINING_MODULE=sagemaker_pytorch_container.training:main\nENV DGLBACKEND=pytorch\n\nRUN apt-get update \\\n && apt-get upgrade -y \\\n && apt-get install -y --no-install-recommends \\\n build-essential \\\n ca-certificates \\\n cmake \\\n curl \\\n emacs \\\n git \\\n jq \\\n libopencv-dev \\\n software-properties-common \\\n wget \\\n unzip \\\n vim \\\n zlib1g-dev \\\n openssl \\\n libssl-dev \\\n libsqlite3-dev \\\n libgdbm-dev \\\n libc6-dev \\\n libbz2-dev \\\n libncurses-dev \\\n tk-dev \\\n libffi-dev \\\n libcap-dev \\\n gnupg2 \\\n gpg-agent \\\n libarchive13 \\\n && rm -rf /var/lib/apt/lists/* \\\n && apt-get clean\n\nRUN apt update\nRUN apt install -y openssh-server openssh-client wget gnupg2 sudo\n\n# Install Neuron packages\nRUN . /etc/os-release\nRUN echo \"deb https://apt.repos.neuron.amazonaws.com focal main\" > /etc/apt/sources.list.d/neuron.list\nRUN wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | apt-key add -\n\nRUN apt-get update \\\n && apt-get install -y \\\n aws-neuronx-tools=$NEURONX_TOOLS_VERSION \\\n aws-neuronx-collectives=$NEURONX_COLLECTIVES_LIB_VERSION \\\n aws-neuronx-runtime-lib=$NEURONX_RUNTIME_LIB_VERSION \\\n && rm -rf /var/lib/apt/lists/* \\\n && rm -rf /tmp/tmp* \\\n && apt-get clean\n\n# Install Open MPI\nRUN mkdir -p /tmp/openmpi \\\n && cd /tmp/openmpi \\\n && wget --quiet https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OMPI_VERSION}.tar.gz \\\n && tar zxf openmpi-${OMPI_VERSION}.tar.gz \\\n && cd openmpi-${OMPI_VERSION} \\\n && ./configure --enable-orterun-prefix-by-default \\\n && make -j $(nproc) all \\\n && make install \\\n && ldconfig \\\n && rm -rf /tmp/openmpi\n\n# install Python\nRUN wget -q https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \\\n && tar -xzf Python-$PYTHON_VERSION.tgz \\\n && cd Python-$PYTHON_VERSION \\\n && ./configure --enable-shared --prefix=/usr/local \\\n && make -j $(nproc) && make install \\\n && cd .. && rm -rf ../Python-$PYTHON_VERSION* \\\n && ln -s /usr/local/bin/pip3 /usr/bin/pip \\\n && ln -s /usr/local/bin/$PYTHON /usr/local/bin/python \\\n && ${PIP} --no-cache-dir install --upgrade \\\n pip \\\n setuptools\n\nWORKDIR /\n\n# The ENV variables declared below are changed in the previous section\n# Grouping these ENV variables in the first section causes\n# ompi_info to fail. This is only observed in CPU containers\nENV PATH=\"$PATH:/home/.openmpi/bin\"\nENV LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/home/.openmpi/lib/\"\nRUN ompi_info --parsable --all | grep mpi_built_with_cuda_support:value\n\nRUN ${PIP} install --no-cache-dir -U \\\n \"bokeh>=2.3,<3\" \\\n \"awscli<2\" \\\n scipy \\\n click \\\n \"cryptography\" \\\n psutil==5.6.7 \\\n dataset \\\n tenacity \\\n transformers==4.36.2 \\\n Pillow\n\nRUN mkdir -p /etc/pki/tls/certs && cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt\nRUN ${PIP} config set global.extra-index-url https://pip.repos.neuron.amazonaws.com \\\n && ${PIP} install --force-reinstall torch-neuronx==$NEURONX_FRAMEWORK_VERSION --extra-index-url https://pip.repos.neuron.amazonaws.com \\\n && ${PIP} install --force-reinstall neuronx-cc==$NEURONX_CC_VERSION --extra-index-url https://pip.repos.neuron.amazonaws.com \\\n && ${PIP} install --force-reinstall --no-deps neuronx_distributed==$NEURONX_DISTRIBUTED_VERSION --extra-index-url https://pip.repos.neuron.amazonaws.com\n\n# attrs, neuronx-cc required: >=19.2.0, sagemaker <24,>=23.1.0\n# protobuf neuronx-cc<4, sagemaker-training >=3.9.2,<=3.20.3\n# awscli 1.25.47 has requirement docutils<0.17,>=0.10\n# etcd for kubernetes installation\n# awscli 1.27.127 has requirement rsa<4.8,>=3.1.2, but you have rsa 4.9.\n# awscli 1.27.127 requires urllib3 < 1.27, python-etcd requires urllib3 >= 1.7, latest urllib3 release is 2.0.2\nRUN ${PIP} install --no-cache-dir -U \\\n \"attrs<24,>=23.1.0\" \\\n \"protobuf>=3.18.3,<=3.20.3\" \\\n \"docutils>=0.10,<0.17\" \\\n \"rsa<4.8,>=3.1.2\" \\\n \"urllib3>=1.26.0,<1.27\"\n\n# EFA Installer does apt get. Make sure to run apt update before that\nRUN apt-get update\nRUN cd $HOME \\\n && curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz \\\n && wget https://efa-installer.amazonaws.com/aws-efa-installer.key && gpg --import aws-efa-installer.key \\\n && cat aws-efa-installer.key | gpg --fingerprint \\\n && wget https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz.sig && gpg --verify ./aws-efa-installer-latest.tar.gz.sig \\\n && tar -xf aws-efa-installer-latest.tar.gz \\\n && cd aws-efa-installer \\\n && ./efa_installer.sh -y -g -d --skip-kmod --skip-limit-conf --no-verify \\\n && cd $HOME\n\n# Clean up after apt update\nRUN rm -rf /var/lib/apt/lists/* \\\n && rm -rf /tmp/tmp* \\\n && apt-get clean\n\n# Install some common packages used by training scripts\n# torchvision needed for MLP. since it depends on torch and torch neuron/torch\n# is already installed install it with nodeps\nRUN pip3 install --no-cache-dir --no-deps -U \\\n torchvision==0.16.*\n\nRUN HOME_DIR=/root \\\n && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \\\n && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \\\n && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \\\n && chmod +x /usr/local/bin/testOSSCompliance \\\n && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \\\n && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \\\n && rm -rf ${HOME_DIR}/oss_compliance* \\\n && rm -rf /tmp/tmp*\n\nRUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/pytorch-2.1/license.txt\n\nRUN mkdir -p /var/run/sshd && \\\n sed -i 's/[ #]\\(.*StrictHostKeyChecking \\).*/ \\1no/g' /etc/ssh/ssh_config && \\\n echo \" UserKnownHostsFile /dev/null\" >> /etc/ssh/ssh_config && \\\n sed -i 's/#\\(StrictModes \\).*/\\1no/g' /etc/ssh/sshd_config && \\\n sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config && \\\n sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config\n\nCOPY test/images/neuron/hack/install-test-resources.sh ./hack/install-test-resources.sh\nRUN chmod +x ./hack/install-test-resources.sh && \\\n ./hack/install-test-resources.sh /home/ubuntu && \\\n rm -f ./hack/install-test-resources.sh\n\nRUN useradd -ms /bin/bash ubuntu\nRUN echo 'ubuntu:password' | chpasswd\nRUN usermod -aG sudo ubuntu &&\\\n chown -R ubuntu /home/ubuntu\n\nWORKDIR /home/ubuntu\nUSER ubuntu\n\nRUN mkdir -p /home/ubuntu/.ssh && \\ \n ssh-keygen -t rsa -f /home/ubuntu/.ssh/id_rsa -N '' && \\\n cp /home/ubuntu/.ssh/id_rsa.pub /home/ubuntu/.ssh/authorized_keys\n\nCOPY test/images/neuron/tests ./tests\n" }, { "path": "test/images/neuron/hack/install-test-resources.sh", "content": "#!/bin/bash\n\nset -o pipefail\nset -o nounset\nset -o errexit\n\nUSER_DIR=${1:-\"/root\"}\n# attempt to cache dataset to avoid runtime download. \n# needs to match https://github.com/pytorch/vision/blob/c0331c5e2933c621db9a44623f4f3981fe2342e0/torchvision/datasets/mnist.py#L42\nMNIST_RESOURCES=(\"train-images-idx3-ubyte.gz\" \"train-labels-idx1-ubyte.gz\" \"t10k-images-idx3-ubyte.gz\" \"t10k-labels-idx1-ubyte.gz\")\nfor i in {0..1}; do \n # we need to populate data for each rank, and we currently always run with 2\n DOWNLOAD_DIR=\"${USER_DIR}/MNIST_DATA_train/${i}/MNIST/raw\"\n mkdir -p \"$DOWNLOAD_DIR\"\n for RESOURCE in ${MNIST_RESOURCES[@]}; do\n DEST_FILE=\"${DOWNLOAD_DIR}/${RESOURCE}\"\n SOURCE_URL=\"https://ossci-datasets.s3.amazonaws.com/mnist/${RESOURCE}\"\n echo \"Downloading ${SOURCE_URL} to ${DEST_FILE}\"\n curl -o \"$DEST_FILE\" \"$SOURCE_URL\"\n done \ndone" }, { "path": "test/images/neuron/tests/singleNodeTest.sh", "content": "#!/usr/bin/env bash\n\nset -e\n\ntorchrun --nproc_per_node=2 --nnodes=1 tests/testNeuronSingleAllReduce.py\ntorchrun --nproc_per_node=2 --nnodes=1 tests/testNeuronParallelState.py\ntorchrun --nproc_per_node=2 --nnodes=1 tests/testNeuronMlp.py" }, { "path": "test/images/neuron/tests/testNeuronMlp.py", "content": "# Source: https://github.com/aws/deep-learning-containers/blob/master/test/dlc_tests/container_tests/bin/pytorch_tests/testNeuronMlp\nimport os\nimport time\nimport torch\n\nfrom torchvision.datasets import mnist\nfrom torch.utils.data import DataLoader\nfrom torchvision.transforms import ToTensor\n\n# XLA imports\nimport torch_xla.core.xla_model as xm\nimport torch_xla.runtime as xr\n\n# XLA imports for parallel loader and multi-processing\nimport torch_xla.distributed.parallel_loader as pl\nfrom torch.utils.data.distributed import DistributedSampler\n\n# Initialize XLA process group for torchrun\nimport torch_xla.distributed.xla_backend\nimport torch.nn as nn\nimport torch.nn.functional as F\n\ntorch.distributed.init_process_group('xla')\n\n# Global constants\nEPOCHS = 4\nWARMUP_STEPS = 2\nBATCH_SIZE = 32\n\n# Load MNIST train dataset\ntrain_dataset = mnist.MNIST(root=os.path.join(os.path.expanduser(\"~\") + '/MNIST_DATA_train', str(xr.global_ordinal())),\n train=True, download=True, transform=ToTensor())\n\n# Declare 3-layer MLP for MNIST dataset\nclass MLP(nn.Module):\n def __init__(self, input_size = 28 * 28, output_size = 10, layers = [120, 84]):\n super(MLP, self).__init__()\n self.fc1 = nn.Linear(input_size, layers[0])\n self.fc2 = nn.Linear(layers[0], layers[1])\n self.fc3 = nn.Linear(layers[1], output_size)\n\n def forward(self, x):\n x = F.relu(self.fc1(x))\n x = F.relu(self.fc2(x))\n x = self.fc3(x)\n return F.log_softmax(x, dim=1)\n\n\ndef main():\n # XLA MP: get world size\n world_size = xr.world_size()\n # multi-processing: ensure each worker has same initial weights\n torch.manual_seed(0)\n\n # Move model to device and declare optimizer and loss function\n device = 'xla'\n model = MLP().to(device)\n # For multiprocessing, scale up learning rate\n optimizer = torch.optim.SGD(model.parameters(), lr=0.01 * world_size)\n loss_fn = torch.nn.NLLLoss()\n\n # Prepare data loader\n train_sampler = None\n if world_size > 1:\n train_sampler = DistributedSampler(train_dataset,\n num_replicas=world_size,\n rank=xr.global_ordinal(),\n shuffle=True)\n train_loader = DataLoader(train_dataset,\n batch_size=BATCH_SIZE,\n sampler=train_sampler,\n shuffle=False if train_sampler else True)\n # XLA MP: use MpDeviceLoader from torch_xla.distributed\n train_device_loader = pl.MpDeviceLoader(train_loader, device)\n\n # Run the training loop\n print('----------Training ---------------')\n model.train()\n for epoch in range(EPOCHS):\n start = time.time()\n for idx, (train_x, train_label) in enumerate(train_device_loader):\n optimizer.zero_grad()\n train_x = train_x.view(train_x.size(0), -1)\n output = model(train_x)\n loss = loss_fn(output, train_label)\n loss.backward()\n xm.optimizer_step(optimizer) # XLA MP: performs grad allreduce and optimizer step\n if idx < WARMUP_STEPS: # skip warmup iterations\n start = time.time()\n\n # Compute statistics for the last epoch\n interval = idx - WARMUP_STEPS # skip warmup iterations\n throughput = interval / (time.time() - start)\n print(\"Train throughput (iter/sec): {}\".format(throughput))\n print(\"Final loss is {:0.4f}\".format(loss.detach().to('cpu')))\n\n # Save checkpoint for evaluation (xm.save ensures only one process save)\n os.makedirs(os.path.expanduser(\"~\") + \"/checkpoints\", exist_ok=True)\n checkpoint = {'state_dict': model.state_dict()}\n xm.save(checkpoint, os.path.expanduser(\"~\") + '/checkpoints/checkpoint.pt')\n\n print('----------End Training ---------------')\n\nif __name__ == '__main__':\n main()\n" }, { "path": "test/images/neuron/tests/testNeuronParallelState.py", "content": "# Source: https://github.com/aws/deep-learning-containers/blob/master/test/dlc_tests/container_tests/bin/pytorch_tests/testNeuronParallelState\nimport argparse\nimport atexit\nimport json\nimport os\nimport traceback\nfrom datetime import datetime\n\nimport torch\nimport torch_xla.core.xla_model as xm\nimport torch_xla.debug.metrics as met\nimport torch_xla.runtime as xr\n\nfrom neuronx_distributed.parallel_layers import parallel_state\nfrom neuronx_distributed.parallel_layers.utils import is_pjrt_device\n\ndatetime_str = str(datetime.now())\n\n\nresults = {\"inference_success\": 1}\n\n\ndef test_initialize_model_parallel(tensor_model_parallel_size):\n def _test_initialize_model_parallel():\n if torch.distributed.get_rank() == 0:\n print(\"testing initialize_model_parallel with size {}\".format(tensor_model_parallel_size))\n tensor_model_parallel_size_ = min(tensor_model_parallel_size, torch.distributed.get_world_size())\n assert not parallel_state.model_parallel_is_initialized()\n parallel_state.initialize_model_parallel(tensor_model_parallel_size=tensor_model_parallel_size_)\n assert parallel_state.model_parallel_is_initialized()\n\n # Checks.\n def check(group, world_size, rank):\n assert world_size == torch.distributed.get_world_size(group=group)\n assert rank == torch.distributed.get_rank(group=group)\n\n # Model parallel.\n world_size = tensor_model_parallel_size_\n rank = torch.distributed.get_rank() % tensor_model_parallel_size_\n assert world_size == parallel_state.get_tensor_model_parallel_size()\n assert rank == parallel_state.get_tensor_model_parallel_rank()\n check(parallel_state.get_tensor_model_parallel_group(), world_size, rank)\n\n # Data parallel.\n world_size = torch.distributed.get_world_size() // tensor_model_parallel_size_\n rank = torch.distributed.get_rank() // tensor_model_parallel_size\n assert world_size == parallel_state.get_data_parallel_size()\n assert rank == parallel_state.get_data_parallel_rank()\n check(parallel_state.get_data_parallel_group(), world_size, rank)\n\n # Reset groups\n parallel_state.destroy_model_parallel()\n\n torch.distributed.barrier()\n if torch.distributed.get_rank() == 0:\n print(\"test passed\")\n\n global results\n try:\n _test_initialize_model_parallel()\n except:\n results[\"inference_success\"] = 0\n print(traceback.format_exc())\n raise\n\n\ndef test_get_tensor_model_parallel_src_rank(tensor_model_parallel_size_):\n def _test_get_tensor_model_parallel_src_rank():\n if torch.distributed.get_rank() == 0:\n print(\"testing get_tensor_model_parallel_src_rank with size {}\".format(tensor_model_parallel_size_))\n tensor_model_parallel_size = min(tensor_model_parallel_size_, torch.distributed.get_world_size())\n assert not parallel_state.model_parallel_is_initialized()\n parallel_state.initialize_model_parallel(tensor_model_parallel_size)\n assert parallel_state.model_parallel_is_initialized()\n\n # Checks\n src_rank = torch.distributed.get_rank() - parallel_state.get_tensor_model_parallel_rank()\n assert parallel_state.get_tensor_model_parallel_src_rank() == src_rank\n\n # Reset groups\n parallel_state.destroy_model_parallel()\n\n torch.distributed.barrier()\n if torch.distributed.get_rank() == 0:\n print(\"test passed\")\n\n global results\n try:\n _test_get_tensor_model_parallel_src_rank()\n except:\n results[\"inference_success\"] = 0\n print(traceback.format_exc())\n raise\n\n\nif __name__ == \"__main__\":\n if is_pjrt_device():\n import torch_xla.experimental.pjrt_backend\n torch.distributed.init_process_group(\"xla\", init_method=\"pjrt://\")\n else:\n torch.distributed.init_process_group(\"xla\")\n world_size = xr.world_size()\n tensor_model_parallel_size = 1\n while tensor_model_parallel_size <= world_size:\n test_initialize_model_parallel(tensor_model_parallel_size)\n test_get_tensor_model_parallel_src_rank(tensor_model_parallel_size)\n tensor_model_parallel_size *= 2\n" }, { "path": "test/images/neuron/tests/testNeuronSingleAllReduce.py", "content": "# Source: https://github.com/aws/deep-learning-containers/blob/master/test/dlc_tests/container_tests/bin/pytorch_tests/testNeuronSingleAllReduce\nimport os\nimport torch\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_backend\nimport torch_xla.runtime as xr\ntorch.distributed.init_process_group('xla')\nimport torch_xla.distributed.xla_multiprocessing as xmp\nos.environ[\"NEURON_RT_EXEC_TIMEOUT\"] = \"20\"\nos.environ[\"NCCL_DEBUG\"] = \"WARN\"\nos.environ[\"NCCL_DEBUG_SUBSYS\"] = \"ALL\"\ndef _mp_fn():\n world_size = xr.world_size()\n device = xm.xla_device()\n rank = xr.global_ordinal()\n ones = torch.ones((2, 3))\n xones = ones.to(device)\n if world_size > 0:\n print(\"running all reduce\")\n for i in range(0, 5):\n print(f'at iteration {i}, with local rank {rank}', flush=True)\n result = xm.all_reduce(xm.REDUCE_SUM, xones)\n result_cpu = result.cpu()\n #xm.mark_step()\n print(result_cpu, flush = True)\n expected = torch.ones((2,3))*world_size\n assert expected.allclose(result_cpu)\n print('PASS')\nif __name__ == '__main__':\n _mp_fn()\n #xmp.spawn(_mp_fn, args=(),nprocs=2, join=True)\n" }, { "path": "test/images/neuron-inference/Dockerfile", "content": "###############################################################################\n# 0) Base image, arguments, and environment\n###############################################################################\nFROM public.ecr.aws/docker/library/ubuntu:22.04\n\n# Disable interactive prompts\nENV DEBIAN_FRONTEND=noninteractive\n\n# Ensure Python prints are unbuffered so we see logs in real time\nENV PYTHONUNBUFFERED=1\n\n# Neuron SDK components version numbers\n# https://github.com/aws-neuron/deep-learning-containers/blob/main/docker/pytorch/inference/2.5.1/Dockerfile.neuronx\nARG NEURONX_CC_VERSION=2.22.12471.0+b4a00d10\nARG NEURONX_FRAMEWORK_VERSION=2.9.0.2.11.19912+e48cd891\nARG NEURONX_COLLECTIVES_LIB_VERSION=2.29.41.0-681fef5f5\nARG NEURONX_RUNTIME_LIB_VERSION=2.29.40.0-f954cd7a5\nARG NEURONX_TOOLS_VERSION=2.27.33.0-5d9c0b901\n\n# Python\nARG PYTHON=python3.10\nARG PYTHON_VERSION=3.10.12\n\nENV PYTHONDONTWRITEBYTECODE=1 \\\n PYTHONUNBUFFERED=1 \\\n PYTHONIOENCODING=UTF-8 \\\n LANG=C.UTF-8 \\\n LC_ALL=C.UTF-8\n\n# Extend library paths for Neuron\nENV LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/opt/aws/neuron/lib\"\nENV LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/usr/local/lib\"\nENV PATH=\"/opt/aws/neuron/bin:${PATH}\"\n\n###############################################################################\n# 1) Base system packages, user setup\n###############################################################################\nRUN apt-get update \\\n && apt-get upgrade -y \\\n && apt-get install -y --no-install-recommends \\\n build-essential \\\n ca-certificates \\\n curl \\\n git \\\n jq \\\n wget \\\n unzip \\\n vim \\\n zlib1g-dev \\\n openssl \\\n libssl-dev \\\n libsqlite3-dev \\\n libgdbm-dev \\\n libc6-dev \\\n libbz2-dev \\\n libncurses-dev \\\n tk-dev \\\n libffi-dev \\\n gnupg2 \\\n gpg-agent \\\n libarchive13 \\\n openssh-server \\\n sudo \\\n && rm -rf /var/lib/apt/lists/* \\\n && apt-get clean\n\n###############################################################################\n# 2) Neuron SDK\n###############################################################################\nRUN . /etc/os-release \\\n && echo \"deb https://apt.repos.neuron.amazonaws.com focal main\" > /etc/apt/sources.list.d/neuron.list \\\n && wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | apt-key add - \\\n && apt-get update -y \\\n && apt-get install -y \\\n aws-neuronx-tools=${NEURONX_TOOLS_VERSION} \\\n aws-neuronx-collectives=${NEURONX_COLLECTIVES_LIB_VERSION} \\\n aws-neuronx-runtime-lib=${NEURONX_RUNTIME_LIB_VERSION} \\\n && rm -rf /var/lib/apt/lists/* \\\n && apt-get clean\n\n###############################################################################\n# 3) Python 3.10 from source\n###############################################################################\nRUN wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \\\n && tar -xzf Python-${PYTHON_VERSION}.tgz \\\n && cd Python-${PYTHON_VERSION} \\\n && ./configure --enable-shared --prefix=/usr/local \\\n && make -j $(nproc) && make install \\\n && cd .. && rm -rf Python-${PYTHON_VERSION}* \\\n && ln -s /usr/local/bin/pip3 /usr/bin/pip \\\n && ln -s /usr/local/bin/${PYTHON} /usr/local/bin/python \\\n && pip --no-cache-dir install --upgrade pip setuptools wheel\n\n###############################################################################\n# 4) Install PyTorch Neuron, Transformers Neuron, etc. via pip\n###############################################################################\nRUN pip config set global.extra-index-url https://pip.repos.neuron.amazonaws.com \\\n && pip install --force-reinstall \\\n \"torch-neuronx==${NEURONX_FRAMEWORK_VERSION}\" \\\n \"neuronx-cc==${NEURONX_CC_VERSION}\" \\\n \"transformers==4.36.2\"\n\n###############################################################################\n# 5) Application files and Python dependencies\n###############################################################################\nWORKDIR /app\nCOPY infer.py /app/\n" }, { "path": "test/images/neuron-inference/infer.py", "content": "import logging\nimport os\nimport sys\nimport time\nimport json\nimport subprocess\nimport random\nimport concurrent.futures\nimport numpy as np\nfrom copy import deepcopy\n\nimport torch\nimport torch_neuronx\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom transformers import BertForPreTraining, BertTokenizer\n\nlogging.basicConfig(\n level=logging.INFO,\n format='[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s',\n handlers=[logging.StreamHandler(sys.stdout)]\n)\nlogger = logging.getLogger(\"BERTNeuronInference\")\nlogger.setLevel(logging.INFO) \n\ndef get_neuron_monitor_stats():\n \"\"\"\n Runs neuron-monitor command and returns the first JSON output as a dictionary.\n Also validates if the environment has Inferentia1/2 device and proper device count.\n \n Returns:\n dict: Parsed JSON output containing neuron monitor statistics\n \n Raises:\n RuntimeError: If neuron-monitor command is not found or fails to execute\n RuntimeError: If environment doesn't have proper Neuron support\n json.JSONDecodeError: If the output cannot be parsed as valid JSON\n \"\"\"\n try:\n # Run neuron-monitor with timeout to get first output\n process = subprocess.Popen(\n ['neuron-monitor'], \n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True\n )\n \n # Wait for first line of output\n output = process.stdout.readline()\n \n # Terminate the process since we only need first output\n process.terminate()\n process.wait()\n \n if not output:\n raise RuntimeError(\"No output received from neuron-monitor\")\n \n # Parse JSON output\n stats = json.loads(output)\n \n # Check for Neuron hardware support\n hardware_info = stats.get('neuron_hardware_info', {})\n device_type = hardware_info.get('neuron_device_type', '').lower()\n neuroncore_per_device_count = hardware_info.get('neuroncore_per_device_count', 0)\n \n if neuroncore_per_device_count <= 0:\n raise RuntimeError(f\"No Neuron devices found (neuroncore_per_device_count: {neuroncore_per_device_count})\")\n \n return neuroncore_per_device_count\n \n except FileNotFoundError:\n raise RuntimeError(\"neuron-monitor command not found\")\n except json.JSONDecodeError as e:\n raise RuntimeError(f\"Failed to parse JSON output: {e}\")\n except Exception as e:\n raise RuntimeError(f\"Error running neuron-monitor: {e}\")\n\n\ndef print_info(msg: str):\n \"\"\"Helper function to prefix all info messages uniformly.\"\"\"\n logger.info(f\"[INFO] {msg}\")\n\n\ndef print_warning(msg: str):\n \"\"\"Helper function for warnings.\"\"\"\n logger.warning(f\"[WARNING] {msg}\")\n\n\ndef print_error(msg: str):\n \"\"\"Helper function for errors.\"\"\"\n logger.error(f\"[ERROR] {msg}\")\n\n\ndef create_dummy_data(tokenizer, batch_size, num_samples=10000, max_length=128, seed=42):\n \"\"\"\n Creates a realistic Next Sentence Prediction (NSP) dataset for BERT model testing.\n\n Args:\n tokenizer (BertTokenizer): instance used to tokenize the input sentences\n batch_size (int): specifying the size of each batch\n num_samples (int): specifying total number of samples to generate (default: 100)\n max_length (int): specifying maximum sequence length for tokenization (default: 128)\n seed (int): for random seed to ensure reproducibility (default: 42)\n\n Returns:\n TensorDataset containing:\n - input_ids (torcTensor): of tokenized input sequences\n - attention_mask: of attention masks\n - nsp_labels: Tensor of NSP labels (0 for random next sentence, 1 for actual next sentence)\n\n Notes:\n - Automatically adjusts num_samples to be a multiple of batch_size\n - Creates balanced dataset with 50% true next sentences and 50% random sentences\n - Uses a predefined set of sample sentences for generating pairs\n \"\"\"\n\n random.seed(seed)\n\n if num_samples % batch_size != 0:\n adjusted = (num_samples // batch_size) * batch_size\n print_info(\n f\"Adjusting num_samples from {num_samples} to {adjusted} \"\n \"to ensure full batches.\"\n )\n num_samples = adjusted\n\n sample_sentences = [\n \"The dog loves playing fetch in the park.\",\n \"Artificial intelligence is reshaping the future.\",\n \"Movies with complex storylines can be very engaging.\",\n \"This restaurant serves an amazing brunch on weekends.\",\n \"Many researchers are exploring neural network architectures.\",\n \"A day at the beach can reduce stress and improve well-being.\",\n \"ChatGPT is a popular large language model by OpenAI.\",\n \"The annual developer conference showcased innovative technologies.\",\n \"Hiking in the mountains offers both challenge and relaxation.\",\n \"Robotics and automation are revolutionizing many industries.\",\n ]\n\n sentences_a = []\n sentences_b = []\n nsp_labels = []\n\n for _ in range(num_samples):\n idx_a = random.randint(0, len(sample_sentences) - 1)\n if random.random() < 0.5:\n # “True” next sentence\n idx_b = (idx_a + 1) % len(sample_sentences)\n nsp_labels.append(1)\n else:\n # Random sentence\n idx_b = random.randint(0, len(sample_sentences) - 1)\n nsp_labels.append(0)\n\n sentences_a.append(sample_sentences[idx_a])\n sentences_b.append(sample_sentences[idx_b])\n\n inputs = tokenizer(\n sentences_a,\n sentences_b,\n max_length=max_length,\n padding=\"max_length\",\n truncation=True,\n return_tensors=\"pt\",\n )\n\n return TensorDataset(\n inputs.input_ids,\n inputs.attention_mask,\n torch.tensor(nsp_labels, dtype=torch.long)\n )\n\n\ndef run_inference(model, tokenizer, batch_size, mode, n_models=2, n_threads=2):\n \"\"\"\n Runs BERT model inference using Neuron runtime with dummy NSP (Next Sentence Prediction) data.\n\n Args:\n model (BertForPreTraining): model instance to be used for inference\n tokenizer (BertTokenizer): instance for processing input text\n batch_size (int): specifying batch size (8 for throughput mode, 1 for latency mode)\n mode (str): indicating inference mode ('throughput' or 'latency')\n n_models (int): number of models to spawn\n n_threads (int): number of threads for inference\n\n Returns:\n None, but prints performance metrics including:\n - Duration of the job\n - Average time per batch\n - Throughput (samples per second)\n - P50, P95, P99 latency \n - Batch Size\n - Total Batches Processed\n - Total Inferences\n\n Notes:\n - Performance metrics are logged with prefix [BERT_INFERENCE_NEURON_METRICS]\n - Uses torch_neuronx for model compilation\n - Handles both throughput and latency testing modes\n - Runs inference with no gradient computation (torch.no_grad) \n \"\"\"\n \n print_info(\"About to create dummy data...\")\n try:\n dataset = create_dummy_data(tokenizer, batch_size=batch_size)\n except Exception as e:\n print_error(f\"Failed to create dummy data: {e}\")\n raise\n\n print_info(\"Dummy data creation completed.\")\n\n dataloader = DataLoader(\n dataset, \n batch_size=batch_size\n )\n \n # First compile the model for Neuron: \n # Since we run inference in batches, we must first\n # split the dataset into the size of input expected in a\n # single batch. This input signature would then be used\n # to call the .trace() method and compile the Bert model to Neuron.\n _input_ids, _attention_masks, _output_ids = dataset.tensors\n _split_input_ids = torch.split(_input_ids, batch_size)[0]\n _split_attention_masks = torch.split(_attention_masks, batch_size)[0]\n batch_input = (_split_input_ids, _split_attention_masks)\n try:\n # Use multicore context for automatic core allocation\n with torch_neuronx.experimental.multicore_context():\n model_neuron = torch_neuronx.trace(model, batch_input)\n except Exception as e:\n logger.exception(f\"[ERROR] Failed to trace BERT model. Failed with error: {e}\")\n raise e\n\n latencies = []\n rows_processed = 0\n\n print_info(f\"Starting Neuron inference ...\")\n begin = time.time()\n \n with torch.no_grad():\n for batch in dataloader:\n batch_input_tensor, batch_attention_tensor, _ = batch \n batch_input = (batch_input_tensor, batch_attention_tensor)\n start = time.time()\n _ = model_neuron(*batch_input)\n finish = time.time()\n \n latencies.append((finish - start) * 1000)\n rows_processed += len(batch_input_tensor)\n\n end = time.time()\n\n # Compute metrics\n boundaries = [50, 95, 99]\n percentiles = {}\n\n for boundary in boundaries:\n name = f'latency_p{boundary}'\n percentiles[name] = np.percentile(latencies, boundary)\n \n duration = end - begin\n inferences = rows_processed\n throughput = inferences / duration\n avg_time_per_batch = np.mean(latencies)\n\n # Print metrics\n print_info(\"Neuron inference completed.\")\n\n # Print metrics to support old logging format\n print_info(\n \"[BERT_INFERENCE_NEURON_METRICS] \"\n f\"mode={mode} \"\n f\"avg_time_per_batch={avg_time_per_batch:.6f} \"\n f\"throughput_samples_per_sec={throughput:.6f}\"\n )\n\n # performance metrics\n print_info(f\"[BERT_INFERENCE_NEURON_METRICS] mode={mode}\")\n print_info(f\"[BERT_INFERENCE_NEURON_METRICS] duration={duration:.6f}\")\n print_info(f\"[BERT_INFERENCE_NEURON_METRICS] avg_time_per_batch={avg_time_per_batch:.6f}\")\n print_info(f\"[BERT_INFERENCE_NEURON_METRICS] throughput_samples_per_sec={throughput:.6f}\")\n\n # latency metrics\n for name, value in percentiles.items():\n print_info(f\"[BERT_INFERENCE_NEURON_METRICS] {name}={value:.6f}\")\n \n print_info(f\"[BERT_INFERENCE_NEURON_METRICS] batch_size={batch_size}\")\n print_info(f\"[BERT_INFERENCE_NEURON_METRICS] total_batches_processed={len(latencies)}\")\n print_info(f\"[BERT_INFERENCE_NEURON_METRICS] total_inferences={inferences}\")\n\n\ndef main():\n \"\"\"Main entry\"\"\"\n print_info(\"Starting main()...\")\n try:\n neuroncore_per_device_count = get_neuron_monitor_stats()\n print_info(f\"Spawing a total of {neuroncore_per_device_count} models\")\n except RuntimeError as e:\n print_error(f\"Neuron environment not detected. Failed with error: {e}\")\n sys.exit(1)\n\n mode = os.environ.get(\"INFERENCE_MODE\", \"throughput\").lower()\n if mode not in [\"throughput\", \"latency\"]:\n print_warning(\n f\"Unrecognized INFERENCE_MODE '{mode}'. \"\n \"Falling back to 'throughput'.\"\n )\n mode = \"throughput\"\n\n batch_size = 1 if mode == \"latency\" else 8\n print_info(f\"Running Neuron inference in {mode} mode with batch size {batch_size}.\")\n\n print_info(\"Loading tokenizer and model...\")\n try:\n model_name = \"bert-base-uncased\"\n tokenizer = BertTokenizer.from_pretrained(model_name)\n model = BertForPreTraining.from_pretrained(model_name, torchscript=True)\n\n except Exception as e:\n print_error(f\"Failed to load model/tokenizer: {e}\")\n sys.exit(1)\n print_info(\"Model and tokenizer loaded successfully.\")\n\n run_inference(model, tokenizer, batch_size, mode, n_models=neuroncore_per_device_count)\n print_info(\"main() completed all steps successfully.\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "test/images/neuron-training/Dockerfile", "content": "FROM public.ecr.aws/docker/library/ubuntu:22.04\n\n###############################################################################\n# 0) Arguments and environment\n###############################################################################\nARG DEBIAN_FRONTEND=noninteractive\n\n# Neuron SDK component versions (pin these precisely)\n# https://github.com/aws-neuron/deep-learning-containers/blob/main/docker/pytorch/training/2.5.1/Dockerfile.neuronx\nARG NEURONX_CC_VERSION=2.22.12471.0+b4a00d10\nARG NEURONX_FRAMEWORK_VERSION=2.9.0.2.11.19912+e48cd891\nARG NEURONX_COLLECTIVES_LIB_VERSION=2.29.41.0-681fef5f5\nARG NEURONX_RUNTIME_LIB_VERSION=2.29.40.0-f954cd7a5\nARG NEURONX_TOOLS_VERSION=2.27.33.0-5d9c0b901\n\n# Python\nARG PYTHON=python3.10\nARG PYTHON_VERSION=3.10.12\n\nENV PYTHONDONTWRITEBYTECODE=1 \\\n PYTHONUNBUFFERED=1 \\\n PYTHONIOENCODING=UTF-8 \\\n LANG=C.UTF-8 \\\n LC_ALL=C.UTF-8\n\n# Extend library paths for Neuron & EFA\nENV LD_LIBRARY_PATH=\"/opt/aws/neuron/lib:/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/local/lib:${LD_LIBRARY_PATH}\"\nENV PATH=\"/opt/aws/neuron/bin:${PATH}\"\n\n###############################################################################\n# 1) Base system packages, user setup\n###############################################################################\nRUN apt-get update \\\n && apt-get upgrade -y \\\n && apt-get install -y --no-install-recommends \\\n build-essential \\\n ca-certificates \\\n curl \\\n git \\\n jq \\\n wget \\\n unzip \\\n vim \\\n lcov \\\n pkg-config \\\n zlib1g-dev \\\n openssl \\\n libssl-dev \\\n libsqlite3-dev \\\n libgdbm-dev \\\n libc6-dev \\\n libbz2-dev \\\n libncurses-dev \\\n tk-dev \\\n libffi-dev \\\n gnupg2 \\\n gpg-agent \\\n libarchive13 \\\n openssh-server \\ \n openssh-client \\\n sudo \\\n && rm -rf /var/lib/apt/lists/* \\\n && apt-get clean\n\n###############################################################################\n# 2) Neuron SDK\n###############################################################################\nRUN . /etc/os-release \\\n && echo \"deb https://apt.repos.neuron.amazonaws.com focal main\" > /etc/apt/sources.list.d/neuron.list \\\n && wget -qO - https://apt.repos.neuron.amazonaws.com/GPG-PUB-KEY-AMAZON-AWS-NEURON.PUB | apt-key add - \\\n && apt-get update \\\n && apt-get install -y \\\n aws-neuronx-tools=${NEURONX_TOOLS_VERSION} \\\n aws-neuronx-collectives=${NEURONX_COLLECTIVES_LIB_VERSION} \\\n aws-neuronx-runtime-lib=${NEURONX_RUNTIME_LIB_VERSION} \\\n && rm -rf /var/lib/apt/lists/* \\\n && apt-get clean\n\n###############################################################################\n# 3) EFA installer (for MPI-based jobs)\n###############################################################################\nRUN apt-get update \\\n && cd /tmp \\\n && curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz \\\n && wget https://efa-installer.amazonaws.com/aws-efa-installer.key \\\n && gpg --import aws-efa-installer.key \\\n && cat aws-efa-installer.key | gpg --fingerprint \\\n && wget https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz.sig \\\n && gpg --verify ./aws-efa-installer-latest.tar.gz.sig \\\n && tar -xf aws-efa-installer-latest.tar.gz \\\n && cd aws-efa-installer \\\n && ./efa_installer.sh -y -g -d --skip-kmod --skip-limit-conf --no-verify \\\n && cd /tmp \\\n && rm -rf aws-efa-installer* \\\n && rm -rf /var/lib/apt/lists/* \\\n && apt-get clean\n\nENV PATH=\"/opt/amazon/openmpi/bin:${PATH}\"\nENV LD_LIBRARY_PATH=\"/opt/amazon/openmpi/lib64:${LD_LIBRARY_PATH}\"\n\n###############################################################################\n# 4) Python 3.10 from source\n###############################################################################\nRUN wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \\\n && tar -xzf Python-${PYTHON_VERSION}.tgz \\\n && cd Python-${PYTHON_VERSION} \\\n && ./configure --enable-shared --prefix=/usr/local \\\n && make -j $(nproc) && make install \\\n && cd .. && rm -rf Python-${PYTHON_VERSION}* \\\n && ln -s /usr/local/bin/pip3 /usr/bin/pip \\\n && ln -s /usr/local/bin/${PYTHON} /usr/local/bin/python \\\n && pip --no-cache-dir install --upgrade pip setuptools wheel\n\n###############################################################################\n# 5) Install pinned Python packages\n###############################################################################\nRUN pip config set global.extra-index-url https://pip.repos.neuron.amazonaws.com \\\n && pip install --force-reinstall \\\n \"torch-neuronx==${NEURONX_FRAMEWORK_VERSION}\" \\\n \"neuronx-cc==${NEURONX_CC_VERSION}\" \\\n \"transformers==4.36.2\"\n\n###############################################################################\n# 6) SSH and finalize\n###############################################################################\n# Configure SSH (auto-accept new host keys)\nRUN mkdir -p /var/run/sshd && \\\n sed -i 's/[ #]\\(.*StrictHostKeyChecking \\).*/ \\1no/g' /etc/ssh/ssh_config && \\\n echo \" UserKnownHostsFile /dev/null\" >> /etc/ssh/ssh_config && \\\n sed -i 's/#\\(StrictModes \\).*/\\1no/g' /etc/ssh/sshd_config\n\n\nWORKDIR /app/\nCOPY train.py /app/\n" }, { "path": "test/images/neuron-training/train.py", "content": "import os\nimport time\nimport random\n\nimport torch\nimport torch.distributed as dist\n\n# === torch_xla imports for device and parallel loader ===\nimport torch_xla.core.xla_model as xm\nimport torch_xla.runtime as xr\nimport torch_xla.distributed.xla_backend\nimport torch_xla.distributed.parallel_loader as pl\n\nfrom torch.utils.data import DataLoader, TensorDataset, DistributedSampler\nfrom transformers import BertForPreTraining, BertTokenizer\n\nRANK = int(os.environ.get(\"RANK\", 0))\nWORLD_SIZE = int(os.environ.get(\"WORLD_SIZE\", 1))\n\ndef create_dummy_data(tokenizer, num_samples=100, max_length=128):\n \"\"\"\n Creates dummy BERT pretraining data (MLM + NSP).\n \"\"\"\n print(f\"Creating dummy data: {num_samples} samples, max_length={max_length}\")\n sentences = [f\"This is a dummy sentence number {i}\" for i in range(num_samples)]\n encodings = tokenizer(\n sentences,\n max_length=max_length,\n padding=\"max_length\",\n truncation=True,\n return_tensors=\"pt\",\n )\n labels = encodings.input_ids.detach().clone()\n\n # Randomly mask some tokens for MLM\n mlm_probability = 0.15\n input_ids, labels = mask_tokens(encodings.input_ids, tokenizer, mlm_probability)\n\n # Dummy next-sentence prediction labels\n next_sentence_labels = torch.randint(0, 2, (num_samples,))\n\n return TensorDataset(input_ids, encodings.attention_mask, labels, next_sentence_labels)\n\n\ndef mask_tokens(inputs, tokenizer, mlm_probability):\n \"\"\"\n Randomly mask tokens for MLM. Unmasked tokens => label = -100\n so we don't compute loss on them.\n \"\"\"\n labels = inputs.clone()\n probability_matrix = torch.full(labels.shape, mlm_probability)\n special_tokens_mask = [\n tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True)\n for val in labels.tolist()\n ]\n probability_matrix.masked_fill_(\n torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0\n )\n masked_indices = torch.bernoulli(probability_matrix).bool()\n labels[~masked_indices] = -100\n inputs[masked_indices] = tokenizer.convert_tokens_to_ids(tokenizer.mask_token)\n\n return inputs, labels\n\ndef complete_epoch(epoch, optimizer, parallel_loader, model):\n\n for step_idx, batch in enumerate(parallel_loader, start=1):\n optimizer.zero_grad()\n input_ids, attention_mask, mlm_labels, next_sentence_labels = batch\n\n outputs = model(\n input_ids=input_ids,\n attention_mask=attention_mask,\n labels=mlm_labels,\n next_sentence_label=next_sentence_labels,\n )\n loss = outputs.loss\n loss.backward()\n\n xm.optimizer_step(optimizer)\n\n if step_idx % 10 == 0:\n print(f\"[Rank {RANK}] - Epoch {epoch}, Step {step_idx}, Loss={loss.item():.4f}\")\n\ndef main():\n dist.init_process_group(\n \"xla\",\n init_method=\"xla://\"\n )\n\n # print info with xla runtime functions to sanity check run context correctly propagates to backend\n print(f\"Starting train.py with rank={xr.global_ordinal()}, world_size={xr.world_size()}\")\n\n # Seed everything for reproducibility\n SEED = 42\n random.seed(SEED)\n torch.manual_seed(SEED)\n\n device = xm.xla_device()\n\n # Preload model + tokenizer\n tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n model = BertForPreTraining.from_pretrained(\"bert-base-uncased\")\n print(f\"[Rank {RANK}]: Model & tokenizer loaded.\")\n\n # Create dummy dataset\n dataset = create_dummy_data(tokenizer, num_samples=1000, max_length=128)\n\n # Shard dataset for each RANK\n sampler = DistributedSampler(\n dataset,\n num_replicas=WORLD_SIZE,\n rank=RANK,\n shuffle=True,\n drop_last=False,\n )\n train_loader = DataLoader(dataset, batch_size=512, sampler=sampler)\n\n # XLA parallel data loader\n parallel_loader = pl.MpDeviceLoader(train_loader, device)\n\n # Move model to XLA device\n model = model.to(device)\n\n optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)\n\n # Let's do 5 epochs\n epochs = 5\n\n model.train()\n\n # TODO: precompile the model. This warmup is arbitrary based on observed behavior\n # neuronx-cc seems to recompile for the first 2 runs for some reason tbd\n print(f\"[Rank {RANK}] - Starting warmup (2 repetitions of epoch 0)\")\n warmup_start = time.time()\n complete_epoch(0, optimizer, parallel_loader, model)\n complete_epoch(0, optimizer, parallel_loader, model)\n warump_time = time.time() - warmup_start\n print(f\"[Rank {RANK}] - Finished warmup in {warump_time:.2f}s\")\n\n print(f\"[Rank {RANK}] - Starting training for {epochs} epochs...\")\n\n start_time = time.time()\n epoch_times = []\n\n for epoch in range(1, epochs + 1):\n epoch_start_time = time.time()\n print(f\"[Rank {RANK}] - Epoch {epoch}/{epochs}\")\n\n complete_epoch(epoch, optimizer, parallel_loader, model)\n\n epoch_time = time.time() - epoch_start_time\n epoch_times.append(epoch_time)\n\n print(f\"[Rank {RANK}] - Epoch {epoch} done in {epoch_time:.2f}s\")\n\n # Total training time\n total_time = time.time() - start_time\n print(f\"[Rank {RANK}] - All epochs complete in {total_time:.2f}s\")\n\n # Each rank processes (dataset_size / WORLD_SIZE) * epochs samples\n local_samples = (len(dataset) / WORLD_SIZE) * epochs\n local_throughput = local_samples / total_time\n\n # Average epoch time (local)\n if epoch_times:\n avg_epoch_time = sum(epoch_times) / len(epoch_times)\n else:\n avg_epoch_time = 0.0\n\n print(\n f\"[Rank {RANK}] - local_samples={local_samples:.1f}, total_time={total_time:.2f}s, \"\n f\"local_throughput={local_throughput:.2f} samples/s, local_avg_epoch_time={avg_epoch_time:.2f}s\"\n )\n\n print(f\"[Rank {RANK}] training complete. Exiting main().\")\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "test/images/nvidia/Dockerfile", "content": "ARG CUDA_MAJOR_VERSION=12\nARG CUDA_MINOR_VERSION=8\n\n# Start with the NVIDIA CUDA base image\nFROM nvidia/cuda:$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION.0-devel-ubuntu22.04\n\nARG CUDA_MAJOR_VERSION\nARG CUDA_MINOR_VERSION\n\nENV DEBIAN_FRONTEND=noninteractive\n\n# Install necessary dependencies\nRUN apt update -y \\\n && apt upgrade -y \\\n && apt remove -y --allow-change-held-packages \\\n libmlx5-1 \\\n ibverbs-utils \\\n libibverbs-dev \\\n libibverbs1 \\\n libnccl2 \\\n libnccl-dev \\\n && rm -rf /opt/hpcx \\\n && rm -rf /usr/local/mpi \\\n && rm -rf /usr/local/ucx \\\n && rm -f /etc/ld.so.conf.d/hpcx.conf \\\n && apt install -y \\\n git \\\n gcc \\\n openssh-client \\\n openssh-server \\\n build-essential \\\n curl \\\n autoconf \\\n libtool \\\n automake \\\n cmake \\\n apt-utils \\\n libhwloc-dev \\\n freeglut3-dev \\\n libglu1-mesa-dev \\\n datacenter-gpu-manager-4-cuda12 \\\n datacenter-gpu-manager-4-cuda13\n\nRUN ldconfig\n\nENV LD_LIBRARY_PATH /opt/amazon/openmpi/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/cuda/lib:/usr/local/lib/:/usr/lib64:/usr/lib/x86_64-linux-gnu/:/usr/lib/aarch64-linux-gnu/:$LD_LIBRARY_PATH\nENV PATH /usr/local/cuda/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/sbin:/usr/bin:/usr/local/bin:$PATH\n\nRUN mkdir -p /var/run/sshd \\\n && sed -i 's/[ #]\\(.*StrictHostKeyChecking \\).*/ \\1no/g' /etc/ssh/ssh_config \\\n && echo \" UserKnownHostsFile /dev/null\" >> /etc/ssh/ssh_config \\\n && sed -i 's/#\\(StrictModes \\).*/\\1no/g' /etc/ssh/sshd_config\n\n# Build CUDA Samples\nRUN git clone https://github.com/NVIDIA/cuda-samples.git /tmp/cuda-samples \\\n --branch v$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION \\\n && cd /tmp/cuda-samples/Samples/0_Introduction/vectorAdd && cmake . && make -j$(nproc) && cp vectorAdd /usr/bin \\\n && cd /tmp/cuda-samples/Samples/1_Utilities/deviceQuery && cmake . && make -j$(nproc) && cp deviceQuery /usr/bin \\\n && cd && rm -rf /tmp/cuda-samples\n\n# Install EFA\nARG EFA_INSTALLER_VERSION=latest\nRUN curl -sL https://efa-installer.amazonaws.com/aws-efa-installer-$EFA_INSTALLER_VERSION.tar.gz | tar xvz -C /tmp \\\n && cd /tmp/aws-efa-installer \\\n && ./efa_installer.sh --yes --enable-gdr --skip-kmod --skip-limit-conf --no-verify --mpi openmpi4 \\\n && cd && rm -rf /tmp/aws-efa-installer\n\n# Build nvbandwidth\nARG NVBANDWIDTH_VERSION=v0.8\nRUN apt install -y libboost-program-options-dev\nRUN git clone https://github.com/NVIDIA/nvbandwidth.git --branch $NVBANDWIDTH_VERSION /tmp/nvbandwidth \\\n && cd /tmp/nvbandwidth \\\n && cmake -DMULTINODE=1 . && make && cp nvbandwidth /usr/bin \\\n && cd && rm -rf /tmp/cuda-samples\n\n# Install NCCL\nARG LIBNCCL_VERSION=2.28.7-1\nRUN git clone https://github.com/NVIDIA/nccl.git --branch v$LIBNCCL_VERSION /tmp/nccl \\\n && cd /tmp/nccl \\\n && make -j $(nproc) \\\n && make install \\\n && cd && rm -rf /tmp/nccl\n\n# Install AWS-OFI-NCCL plugin\nARG AWS_OFI_NCCL_VERSION=1.17.2\nRUN curl -sL https://github.com/aws/aws-ofi-nccl/releases/download/v$AWS_OFI_NCCL_VERSION/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION.tar.gz | tar xvz -C /tmp \\\n && cd /tmp/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION \\\n && ./configure \\\n --prefix=/opt/aws-ofi-nccl/install \\\n --with-mpi=/opt/amazon/openmpi \\\n --with-libfabric=/opt/amazon/efa \\\n --with-cuda=/usr/local/cuda \\\n --enable-platform-aws \\\n --disable-tests \\\n && make -j $(nproc) \\\n && make install \\\n && cd && rm -rf /tmp/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION\n\n# Install NCCL Tests\n# TODO: automate pin with version bump\nRUN git clone https://github.com/NVIDIA/nccl-tests /tmp/nccl-tests \\\n && cd /tmp/nccl-tests \\\n && make \\\n MPI=1 \\\n MPI_HOME=/opt/amazon/openmpi/ \\\n CUDA_HOME=/usr/local/cuda \\\n NCCL_HOME=/usr/local/lib \\\n && mkdir -p /opt/nccl-tests \\\n && mv build /opt/nccl-tests/build \\\n && cd && rm -rf /tmp/nccl-tests\n\n# Set a default command for debugging or modify as per requirements\nENV NCCL_PROTO simple\n# see https://linux.die.net/man/8/ld.so for usage. replaces LD_PRELOAD env.\nRUN echo \"/usr/local/lib/libnccl.so\" >> /etc/ld.so.preload\n\nRUN rm -rf /var/lib/apt/lists/*\n\nWORKDIR /app\n\nCOPY test/images/nvidia/gpu_unit_tests ./gpu_unit_tests\nRUN chmod +x ./gpu_unit_tests/unit_test\n" }, { "path": "test/images/nvidia/gpu_unit_tests/README.md", "content": "# What\n\ngpu_unit_tests is the unit tests for gpu enabled platforms. Idea is to create compact\nset of tests which will cover most of performance critical aspects for gpu\nplatforms. Test designed to run on single instance.\n# Usage\n\n```\n# Run tests\n./unit_test\n```\n\n**Generate test data for new instance type**\n\nStep 1: Copy the `gpu_unit_tests` folder to the EC2 instance where you want to generate the data.\n\nStep 2: Execute the following command in the `gpu_unit_tests` directory on the EC2 instance:\n```\nGENERATE_DATA=1 ./unit_test\n```\nStep 3:\nCopy the files from `tests/test_sysinfo.sh.data` (e.g., `tests/test_sysinfo.sh.data/p3.2xlarge`) to your local repository.\n\nStep 4:\nCreate PR with the new `tests/test_sysinfo.sh.data/xxx`\n\n# Test list\n\n- test_sysinfo.sh :: Validate basic system configuration by comparing it with test config\n - test_numa_topo_topo :: check cpu/numa topology\n - test_nvidia_gpu_count :: fail if one of GPUs is broken or is not visiable\n - test_nvidia_fabric_status :: fail if fabric manager is not active\n - test_nvidia_smi_topo :: fail if nvidia-smi topology is differ\n - test_nvidia_persistence_status :: validate persistence state\n - test_nvidia_gpu_unused :: Check that no other process are using GPUs, fail is a signal system misconfiguration.\n\n\n- 10_test_basic_cuda.sh :: Execute trivial cuda binaries, fail if cuda subsys is not healthy\n Use demo-suite binaries https://docs.nvidia.com/cuda/demo-suite/index.html and DCGM Diagnostics https://docs.nvidia.com/datacenter/dcgm/latest/user-guide/dcgm-diagnostics.html#run-levels-and-tests \n If this test suite fail this is a sign that cuda subsystem is not usable at all.\n Usually this is side effect of system misconfiguration (driver or fabric manager is not loaded)\n - test_01_device_query\n - test_02_vector_add\n - test_03_nvbandwidth\n - test_04_dcgm_diagnostics\n\n\n" }, { "path": "test/images/nvidia/gpu_unit_tests/bash_unit", "content": "#!/usr/bin/env bash\n#\n# bash unit testing enterprise edition framework for professionals\n# Copyright (C) 2011-2016 Pascal Grange\n# This program is free software; you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation; either version 3 of the License, or\n# (at your option) any later version.\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n# You should have received a copy of the GNU General Public License\n# along with this program; if not, write to the Free Software Foundation,\n# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA\n#\n# https://github.com/pgrange/bash_unit\n\nVERSION=v2.1.0\n\nESCAPE=$(printf \"\\033\")\nNOCOLOR=\"${ESCAPE}[0m\"\nRED=\"${ESCAPE}[91m\"\nGREEN=\"${ESCAPE}[92m\"\nYELLOW=\"${ESCAPE}[93m\"\nBLUE=\"${ESCAPE}[94m\"\n\n# Make bash_unit immune to some basic unix commands faking\nCAT=\"$(which cat)\"\nSED=\"$(which sed)\"\nGREP=\"$(which grep)\"\nRM=\"$(which rm)\"\nSHUF=\"$(which shuf)\"\n\nfail() {\n local message=${1:-}\n local stdout=${2:-}\n local stderr=${3:-}\n\n notify_test_failed \"$__bash_unit_current_test__\" \"$message\"\n [[ ! -z $stdout ]] && [ -s \"$stdout\" ] && notify_stdout < \"$stdout\"\n [[ ! -z $stderr ]] && [ -s \"$stderr\" ] && notify_stderr < \"$stderr\"\n\n stacktrace | notify_stack\n exit 1\n}\n\nskip() {\n local message=${1:-}\n notify_test_skipped \"$__bash_unit_current_test__\" \"$message\"\n echo \"skipped $message\" > $__bash_unit_test_skipped__\n exit 0\n}\n\n_notify_trace() {\n local caller_shift=$1\n local message=${2}\n local stdout=${3:-}\n local stderr=${4:-}\n\n [ -z $trace_file ] && return\n\n caller_hdr=\"\"\n cl=$((caller_shift + 2))\n\n if [ -n ${BASH_SOURCE[$cl]} ]\n then\n caller_hdr=\"${BASH_SOURCE[$cl]}:${BASH_LINENO[$((cl-1))]}\"\n fi\n echo \"trace:${caller_hdr}> $message\" >> $trace_file\n [[ ! -z $stdout ]] && [ -s \"$stdout\" ] && \"$SED\" 's:^:trace-out> :' < \"$stdout\" >> $trace_file\n [[ ! -z $stderr ]] && [ -s \"$stderr\" ] && \"$SED\" 's:^:trace-err> :' < \"$stderr\" >> $trace_file\n}\n\nnotify_trace_dbg() {\n _notify_trace 0 \"$1\"\n}\n\nnotify_trace_info() {\n [ -z $trace_file ] && return\n\n local message=${1:-}\n echo \"info> $message\" >> $trace_file\n}\n\nassert() {\n local assertion=$1\n local message=${2:-}\n\n _assert_expression \\\n \"$assertion\" \\\n \"[ \\$status == 0 ]\" \\\n \"\\\"$message\\\"\"\n}\n\nassert_fails() {\n local assertion=$1\n local message=${2:-}\n\n _assert_expression \\\n \"$assertion\" \\\n \"[ \\$status != 0 ]\" \\\n \"\\\"$message\\\"\"\n}\n\nassert_fail() {\n #deprecated, use assert_fails instead\n assert_fails \"$@\"\n}\n\nassert_status_code() {\n local expected_status=$1\n local assertion=\"$2\"\n local message=\"${3:-}\"\n\n _assert_expression \\\n \"$assertion\" \\\n \"[ \\$status == $expected_status ]\" \\\n \"\\\"$message\\\" expected status code $expected_status but was \\$status\"\n}\n\n_assert_expression() {\n local assertion=$1\n local condition=$2\n local message=$3\n (\n local stdout=$(mktemp)\n local stderr=$(mktemp)\n trap \"$RM -f \\\"$stdout\\\" \\\"$stderr\\\"\" EXIT\n\n local status\n eval \"($assertion)\" >\"$stdout\" 2>\"$stderr\" && status=$? || status=$?\n _notify_trace 1 \"assert_expression: exp: '$assertion', cond: '$condition', status: '$status'\" \"$stdout\" \"$stderr\"\n\n if ! eval \"$condition\"\n then\n fail \"$(eval echo $message)\" \"$stdout\" \"$stderr\"\n fi\n ) || exit $?\n}\n\nassert_equals() {\n local expected=$1\n local actual=$2\n local message=${3:-}\n [[ -z $message ]] || message=\"$message\\n\"\n\n notify_trace_dbg \"assert_equals '$expected' == '$actual'\"\n if [ \"$expected\" != \"$actual\" ]\n then\n fail \"$message expected [$expected] but was [$actual]\"\n fi\n}\n\nassert_not_equals() {\n local unexpected=\"$1\"\n local actual=\"$2\"\n local message=${3:-}\n [[ -z $message ]] || message=\"$message\\n\"\n\n notify_trace_dbg \"assert_not_equals: '$unexpected' != '$actual'\"\n [ \"$unexpected\" != \"$actual\" ] || \\\n fail \"$message expected different value than [$unexpected] but was the same\"\n}\n\nassert_matches() {\n local expected=$1\n local actual=$2\n local message=${3:-}\n [[ -z $message ]] || message=\"$message\\n\"\n\n notify_trace_dbg \"assert_matches: '$actual' =~ '$expected'\"\n if [[ ! \"${actual}\" =~ ${expected} ]]; then\n fail \"$message expected regex [$expected] to match [$actual]\"\n fi\n}\n\nassert_not_matches() {\n local unexpected=$1\n local actual=$2\n local message=${3:-}\n [[ -z $message ]] || message=\"$message\\n\"\n\n _notify_trace 0 \"assert_not_matches: ! '$actual' =~ '$unexpected'\"\n if [[ \"${actual}\" =~ ${unexpected} ]]; then\n fail \"$message expected regex [$unexpected] should not match but matched [$actual]\"\n fi\n}\n\nassert_within_delta() {\n function abs() {\n local value=$1\n local sign=$(( value < 0 ? -1 : 1 ))\n echo $((value * sign))\n }\n function is_number() {\n local value=$1\n test $value -eq $value 2>/dev/null\n }\n local expected=$1\n local actual=$2\n local max_delta=$3\n assert \"is_number $expected\" \"$message expected value [$expected] is not a number\"\n assert \"is_number $actual\" \"$message actual value [$actual] is not a number\"\n assert \"is_number $max_delta\" \"$message max_delta [$max_delta] is not a number\"\n local message=${4:-}\n [[ -z $message ]] || message=\"$message\\n\"\n\n local actual_delta=\"$(abs $(($expected - $actual)))\"\n\n if (( $actual_delta > $max_delta )); then\n fail \"$message expected value [$expected] to match [$actual] with a maximum delta of [$max_delta]\"\n fi\n}\n\nassert_no_diff() {\n local expected=$1\n local actual=$2\n local message=${3:-}\n [[ -z $message ]] || message=\"$message\\n\"\n\n assert 'diff '\"${expected}\"' '\"${actual}\" \\\n \"$message expected '\"${actual}\"' to be identical to '\"${expected}\"' but was different\"\n}\n\nfake() {\n local command=$1\n shift\n if [ $# -gt 0 ]\n then\n eval \"function $command() { export FAKE_PARAMS=(\\\"\\$@\\\") ; $@ ; }\"\n else\n eval \"function $command() { echo \\\"$($CAT)\\\" ; }\"\n fi\n export -f $command\n}\n\nstacktrace() {\n local i=1\n while ! [ -z \"${BASH_SOURCE[$i]:-}\" ]\n do\n echo ${BASH_SOURCE[$i]}:${BASH_LINENO[$((i-1))]}:${FUNCNAME[$i]}\\(\\)\n i=$((i + 1))\n done | \"$GREP\" -v \"^$BASH_SOURCE\"\n}\n\nrun_test_suite() {\n local failure=0\n\n if run_setup_suite\n then\n run_tests || failure=$?\n else\n failure=$?\n fi\n run_teardown_suite\n\n return $failure\n}\n\nrun_setup_suite() {\n if declare -F | \"$GREP\" ' setup_suite$' >/dev/null\n then\n setup_suite\n fi\n}\n\nmaybe_shuffle() {\n ((randomise)) && $SHUF || $CAT\n}\n\nrun_tests() {\n local failure=0\n\n for pending_test in $(set | \"$GREP\" -E '^(pending|todo).* \\(\\)' | \"$GREP\" -E \"$test_pattern\" | \"$SED\" -e 's: .*::')\n do\n notify_test_starting \"$pending_test\"\n notify_test_pending \"$pending_test\"\n done\n\n\n for test in $(set | \"$GREP\" -E '^test.* \\(\\)' | \"$GREP\" -E \"$test_pattern\" | \"$SED\" -e 's: .*::' | maybe_shuffle)\n do\n (\n local status=0\n declare -F | \"$GREP\" ' setup$' >/dev/null && setup\n __bash_unit_test_skipped__=$(mktemp)\n trap \"$RM -f \\\"$stdout\\\" \\\"$stderr\\\"\" EXIT\n if [[ -n \"$skip_pattern\" && (\"$test\" =~ $skip_pattern) ]]; then\n skip \"$test as specified in skip pattern: $skip_pattern\"\n fi\n (__bash_unit_current_test__=\"$test\" run_test) || status=$?\n test -s $__bash_unit_test_skipped__ && status=0\n declare -F | \"$GREP\" ' teardown$' >/dev/null && teardown\n exit $status\n )\n failure=$(( $? || failure))\n done\n return $failure\n}\n\nrun_test() {\n set -e\n notify_test_starting \"$__bash_unit_current_test__\"\n \"$__bash_unit_current_test__\" && notify_test_succeeded \"$__bash_unit_current_test__\"\n}\n\nrun_teardown_suite() {\n if declare -F | \"$GREP\" ' teardown_suite$' >/dev/null\n then\n teardown_suite\n fi\n}\n\nusage() {\n echo \"$1\" >&2\n echo \"$0 [-f ] [-p ] [-p ] [-s ] [-r] ... ...\" >&2\n echo >&2\n echo \"Runs tests in test files that match s\" >&2\n echo \"Skip tests in test files that match s\" >&2\n echo \" is optional only supported value is tap\" >&2\n echo \"-r to execute test cases in random order\" >&2\n echo \"-v to get current version information\" >&2\n echo \"See https://github.com/pgrange/bash_unit\" >&2\n exit 1\n}\n\n# Formating\n\npretty_success() {\n pretty_format \"$GREEN\" \"\\u2713\" \"${1:-}\"\n}\n\npretty_warning() {\n pretty_format \"$YELLOW\" \"\\u2717\" \"$1\"\n}\n\npretty_failure() {\n pretty_format \"$RED\" \"\\u2717\" \"${1:-}\"\n}\n\npretty_format() {\n local color=\"$1\"\n local pretty_symbol=\"$2\"\n local alt_symbol=\"${3:-}\"\n local term_utf8=false\n#env\n if is_terminal && [[ \"${LANG:-}\" =~ .*UTF-8.* ]]\n then\n term_utf8=true\n fi\n (\n $CAT\n if $term_utf8\n then\n echo -en \" $pretty_symbol \"\n else\n [[ ! -z \"$alt_symbol\" ]] && echo -en \" $alt_symbol \"\n fi\n ) | color \"$color\"\n}\n\ncolor() {\n _start_color() {\n if is_terminal ; then echo -en \"$color\" ; fi\n }\n _stop_color() {\n if is_terminal ; then echo -en \"$NOCOLOR\" ; fi\n }\n local color=$1\n shift\n _start_color\n if [ $# -gt 0 ]\n then\n echo $*\n else\n $CAT\n fi\n _stop_color\n}\n\nis_terminal() {\n [ -t 1 ] || [[ \"${FORCE_COLOR:-}\" == true ]]\n}\n\ntrace_suite_starting() {\n local test_file=\"$1\"\n notify_trace_info \"Running tests in $test_file\"\n }\ntrace_test_starting() {\n local test=\"$1\"\n notify_trace_info \"Running $test\"\n}\ntrace_test_pending() {\n local test=\"$1\"\n notify_trace_info \"Pending $test\"\n}\n\ntrace_test_skipped() {\n local test=\"$1\"\n local message=\"$2\"\n notify_trace_info \"Skip $test message: $message\"\n}\n\ntrace_test_succeeded() {\n local test=\"$1\"\n notify_trace_info \"Success $test\"\n}\ntrace_test_failed() {\n local test=\"$1\"\n local message=\"$2\"\n notify_trace_info \"$test with message: $message\"\n}\ntrace_suites_succeded() {\n notify_trace_info \"Overall result: SUCCESS\"\n}\ntrace_suites_failed() {\n notify_trace_info \"Overall result: FAILURE\"\n}\n\ntext_format() {\n notify_suite_starting() {\n local test_file=\"$1\"\n trace_suite_starting $test_file\n echo \"Running tests in $test_file\"\n }\n notify_test_starting() {\n local test=\"$1\"\n trace_test_starting $test\n echo -e -n \"\\tRunning $test ... \" | color \"$BLUE\"\n }\n notify_test_pending() {\n local test=\"$1\"\n trace_test_pending \"$test\"\n echo -n \"PENDING\" | pretty_warning\n echo\n }\n notify_test_skipped() {\n local test=\"$1\"\n local message=\"$2\"\n trace_test_skipped \"$test\" \"$message\"\n echo -n \"SKIPPED\" | pretty_warning\n [[ -z $message ]] || printf -- \"$message\\n\"\n echo\n }\n\n notify_test_succeeded() {\n local test=\"$1\"\n trace_test_succeeded \"$test\"\n echo -n \"SUCCESS\" | pretty_success\n echo\n }\n notify_test_failed() {\n local test=\"$1\"\n local message=\"$2\"\n trace_test_failed \"$test\" \"$message\"\n echo -n \"FAILURE\" | pretty_failure\n echo\n [[ -z $message ]] || printf -- \"$message\\n\"\n }\n notify_stdout() {\n \"$SED\" 's:^:out> :' | color \"$GREEN\"\n }\n notify_stderr() {\n \"$SED\" 's:^:err> :' | color \"$RED\"\n }\n notify_stack() {\n color \"$YELLOW\"\n }\n notify_suites_succeded() {\n trace_suites_succeded\n echo -n \"Overall result: SUCCESS\" | pretty_success\n echo\n }\n notify_suites_failed() {\n trace_suites_failed\n echo -n \"Overall result: FAILURE\" | pretty_failure\n echo\n }\n}\n\ntap_format() {\n notify_suite_starting() {\n local test_file=\"$1\"\n trace_suite_starting\n echo \"# Running tests in $test_file\"\n }\n notify_test_starting() {\n trace_test_starting $1\n }\n notify_test_pending() {\n local test=\"$1\"\n trace_test_pending \"$test\"\n echo -n \"ok\" | pretty_warning -\n echo -n \"$test\" | color \"$BLUE\"\n echo \" # skip test to be written\" | color \"$YELLOW\"\n }\n notify_test_skipped() {\n local test=\"$1\"\n local message=\"$2\"\n trace_test_skipped \"$test\" \"$message\"\n echo -n \"ok\" | pretty_warning -\n echo -n \"$test\" | color \"$BLUE\"\n echo \" # skip ${message}\" | color \"$YELLOW\"\n }\n\n notify_test_succeeded() {\n local test=\"$1\"\n trace_test_succeeded \"$test\"\n echo -n \"ok\" | pretty_success -\n echo \"$test\" | color \"$BLUE\"\n }\n notify_test_failed() {\n local test=\"$1\"\n local message=\"$2\"\n trace_test_failed \"$test\" \"$message\"\n echo -n \"not ok\" | pretty_failure -\n echo \"$test\" | color \"$BLUE\"\n [[ -z $message ]] || printf -- \"$message\\n\" | \"$SED\" -u -e 's/^/# /'\n }\n notify_stdout() {\n \"$SED\" 's:^:# out> :' | color \"$GREEN\"\n }\n notify_stderr() {\n \"$SED\" 's:^:# err> :' | color \"$RED\"\n }\n notify_stack() {\n \"$SED\" 's:^:# :' | color \"$YELLOW\"\n }\n notify_suites_succeded() {\n trace_suites_succeded\n }\n notify_suites_failed() {\n trace_suites_failed\n }\n}\n\noutput_format=text\ntest_pattern=\"\"\nskip_pattern=\"\"\ntrace_file=\"\"\nseparator=\"\"\nrandomise=0\nwhile getopts \"vp:t:f:r:s:\" option\ndo\n case \"$option\" in\n p)\n test_pattern=\"${test_pattern}${separator}${OPTARG}\"\n separator=\"|\"\n ;;\n s)\n skip_pattern=\"${skip_pattern}${separator}${OPTARG}\"\n separator=\"|\"\n ;;\n t)\n trace_file=\"$(realpath ${OPTARG})\"\n truncate -s0 \"$trace_file\"\n ;;\n f)\n output_format=\"${OPTARG}\"\n ;;\n r)\n randomise=1\n ;;\n v)\n echo \"bash_unit $VERSION\"\n exit\n ;;\n ?|:)\n usage\n ;;\n esac\ndone\nshift $((OPTIND-1))\n\nfor test_file in \"$@\"\ndo\n test -e \"$test_file\" || usage \"file does not exist: $test_file\"\n test -r \"$test_file\" || usage \"can not read file: $test_file\"\ndone\n\ncase \"$output_format\" in\n text)\n text_format\n ;;\n tap)\n tap_format\n ;;\n *)\n usage \"unsupported output format: $output_format\"\n ;;\nesac\n\n#run tests received as parameters\nfailure=0\nfor test_file in \"$@\"\ndo\n notify_suite_starting \"$test_file\"\n (\n set -e # Ensure bash_unit will exit with failure\n # in case of syntax error.\n if [[ \"${STICK_TO_CWD}\" != true ]]\n then\n cd \"$(dirname \"$test_file\")\"\n source \"$(basename \"$test_file\")\"\n else\n source \"$test_file\"\n fi\n set +e\n run_test_suite\n )\n failure=$(( $? || failure))\ndone\n\nif ((failure))\nthen\n notify_suites_failed\nelse\n notify_suites_succeded\nfi\n\nexit $failure\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/common.sh", "content": "#!/bin/bash\n\nget_instance_type()\n{\n # Retrieve instance metadata: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html#instance-metadata-retrieval-examples\n [ -n \"$FORCE_INSTANCE_TYPE\" ] && echo $FORCE_INSTANCE_TYPE\n\n local token=$(curl -X PUT \"http://169.254.169.254/latest/api/token\" -H \"X-aws-ec2-metadata-token-ttl-seconds: 21600\" 2>/dev/null)\n\n if [ -n \"$token\" ]; then\n curl -H \"X-aws-ec2-metadata-token: $token\" http://169.254.169.254/latest/meta-data/instance-type\n else\n curl http://169.254.169.254/latest/meta-data/instance-type\n fi\n}\n\nassert_gpu_unused()\n{\n cmd=\"nvidia-smi --query-compute-apps timestamp,gpu_bus_id,gpu_uuid,pid,name,used_memory --format csv,noheader\"\n assert_equals \"\" \"`$cmd`\" \"gpu is busy by other task, system misconfig?\"\n}\n\n_assert_data()\n{\n local expected=\"$1\"\n local cmd=\"$2\"\n local message=\"${3:-}\"\n local cmd_out=\"$ACTUAL_RESULTS/$(basename $expected)\"\n [[ -z $message ]] || message=\"$message\\n\"\n\n eval \"$cmd\" > $cmd_out\n diff_cmd=\"diff -up $expected $cmd_out\"\n diff_out=\"`$diff_cmd`\"\n\n notify_trace_dbg \"_assert_data $diff_cmd, out: $diff_out\"\n if [ -n \"$diff_out\" ]\n then\n\tfail \"$message test data value diff:\\n$diff_out\"\n fi\n}\n\nassert_data() {\n _assert_data \"$1\" \"$2\" \"$3\"\n}\n\ngenerate_data()\n{\n local expected=\"$1\"\n local cmd=\"$2\"\n local msg=\"$3\"\n local cmd_out=\"$ACTUAL_RESULTS/$(basename $expected)\"\n\n eval \"$cmd\" > $expected\n _assert_data \"$expected\" \"$cmd\" \"$msg\"\n}\n\nfunction is_vgpu()\n{\n local instance_type=${EC2_INSTANCE_TYPE:-$(get_instance_type)}\n case \"${instance_type}\" in\n g6f.*|gr6f.*) return ;;\n *) return 1 ;; # Not supported\n esac\n}\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_basic.sh", "content": "# Trivial cuda tests to validate that GPU it functional\n# Use demu-suite binaries https://docs.nvidia.com/cuda/demo-suite/index.html \n# and DCGM Diagnostics https://docs.nvidia.com/datacenter/dcgm/latest/user-guide/dcgm-diagnostics.html#run-levels-and-tests\n\nsetup_suite()\n{\n source common.sh\n assert_gpu_unused\n DEMO_SUITE_DIR=${DEMO_SUITE_DIR:-$(realpath /usr/local/cuda/extras/demo_suite)}\n}\n\nteardown_suite()\n{\n assert_gpu_unused\n}\n\ntest_01_device_query()\n{\n assert_status_code 0 \"$DEMO_SUITE_DIR/deviceQuery\"\n}\n\ntest_02_vector_add()\n{\n assert_status_code 0 \"$DEMO_SUITE_DIR/vectorAdd\"\n}\n\ntest_03_nvbandwidth()\n{\n assert_status_code 0 \"$DEMO_SUITE_DIR/nvbandwidth\"\n}\n\ntest_04_dcgm_diagnostics()\n{\n # This test is not applicable for vGPU instance types.\n if is_vgpu; then\n skip \"This test does not apply to vGPU instances (g6f.*, gr6f.*)\"\n fi\n\n # https://docs.nvidia.com/datacenter/dcgm/latest/user-guide/dcgm-diagnostics.html#run-levels-and-tests\n assert_status_code 0 \"dcgmi diag -r 2\"\n}\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh", "content": "# Validate basic system configuration by comparing with expected config\n#\nsetup_suite()\n{\n source common.sh\n\n EC2_INSTANCE_TYPE=${EC2_INSTANCE_TYPE:-$(get_instance_type)}\n data=test_sysinfo.sh.data/$EC2_INSTANCE_TYPE\n ACTUAL_RESULTS=`mktemp -t -d test_sysinfo.sh.actual-data.XXX`\n assert_not_equals \"\" \"$ACTUAL_RESULTS\"\n notify_trace_info \"ACTUAL_RESULTS: $ACTUAL_RESULTS\"\n\n if [ -n \"$GENERATE_DATA\" ]\n then\n\techo \"GENERATE_DATA is enabled...\"\n\tmkdir -p $data\n\tfunction assert_data() {\n\t generate_data \"$@\"\n\t}\n fi\n}\n\nteardown_suite()\n{\n assert \"test -z \\\"$GENERATE_DATA\\\"\" \"GENERATE_DATA was enabled, fail full suite\"\n assert_gpu_unused\n}\n\n\ntest_numa_topo_topo()\n{\n assert_data $data/numa_topo.txt \"grep . /sys/devices/system/node/node*/{cpulist,distance}\" \"Unexpected cpu topology\"\n}\n\ntest_nvidia_gpu_count()\n{\n #Just for logging purposesclear\n assert_status_code 0 \"nvidia-smi -q\"\n assert_data $data/gpu_count.txt \"nvidia-smi --query-gpu=name,index,pci.bus_id --format csv\" \"Unexpected gpu count\"\n}\n\n\ntest_nvidia_smi_topo()\n{\n assert_data $data/nvidia_smi_topo.txt \"nvidia-smi topo -m | grep GPU | cut -f 1-11\" \\\n\t\t\"Unexpected gpu topology, likely broken nvlinks\"\n}\n\n\ntest_nvidia_persistence_status()\n{\n assert_data $data/nvidia_persistence_status.txt \"nvidia-smi --query-gpu=name,pci.bus_id,persistence_mode --format=csv\" \\\n\t\t \"Unexpected perfistance status, likely system configuration issue\"\n}\n\ntest_nvidia_gpu_unused()\n{\n assert_gpu_unused\n}\n\ntest_nvidia_gpu_throttled()\n{\n\n # vGPU instances don't support GPU clock throttling detection.\n # This test is not applicable for vGPU instance types.\n if is_vgpu; then\n skip \"This test does not apply to vGPU instances (g6f.*, gr6f.*)\"\n fi\n # https://docs.nvidia.com/deploy/nvml-api/group__nvmlClocksEventReasons.html#group__nvmlClocksEventReasons\n # The only bit allowed is nvmlClocksEventReasonGpuIdle 0x0000000000000001LL\n filter=\"egrep -v -e '(0x0000000000000000|0x0000000000000001|0x0000000000000004)'\"\n cmd=\"nvidia-smi --query-gpu index,gpu_bus_id,gpu_uuid,clocks_throttle_reasons.active --format=csv,noheader\"\n assert_status_code 1 \"$cmd | $filter\" \"Throttled gpu detected\"\n}\n\n\ntest_nvidia_vgpu_license_status()\n{\n if ! is_vgpu; then\n skip \"This test only applies to vGPU instances (g6f.*, gr6f.*)\"\n fi\n\n assert_data $data/nvidia_vgpu_license_status.txt \\\n \"nvidia-smi -q | grep 'vGPU Software' -A 2\" \\\n \"vGPU license status validation failed\"\n}" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5.48xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA A10G, 0, 00000000:00:16.0\nNVIDIA A10G, 1, 00000000:00:17.0\nNVIDIA A10G, 2, 00000000:00:18.0\nNVIDIA A10G, 3, 00000000:00:19.0\nNVIDIA A10G, 4, 00000000:00:1A.0\nNVIDIA A10G, 5, 00000000:00:1B.0\nNVIDIA A10G, 6, 00000000:00:1C.0\nNVIDIA A10G, 7, 00000000:00:1D.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5.48xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-47,96-143\n/sys/devices/system/node/node1/cpulist:48-95,144-191\n/sys/devices/system/node/node0/distance:10 32\n/sys/devices/system/node/node1/distance:32 10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5.48xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA A10G, 00000000:00:16.0, Enabled\nNVIDIA A10G, 00000000:00:17.0, Enabled\nNVIDIA A10G, 00000000:00:18.0, Enabled\nNVIDIA A10G, 00000000:00:19.0, Enabled\nNVIDIA A10G, 00000000:00:1A.0, Enabled\nNVIDIA A10G, 00000000:00:1B.0, Enabled\nNVIDIA A10G, 00000000:00:1C.0, Enabled\nNVIDIA A10G, 00000000:00:1D.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5.48xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tCPU Affinity\tNUMA Affinity\nGPU0\t X \tPHB\tPHB\tPHB\tPHB\tPHB\tPHB\tPHB\t0-191\t0-1\nGPU1\tPHB\t X \tPHB\tPHB\tPHB\tPHB\tPHB\tPHB\t0-191\t0-1\nGPU2\tPHB\tPHB\t X \tPHB\tPHB\tPHB\tPHB\tPHB\t0-191\t0-1\nGPU3\tPHB\tPHB\tPHB\t X \tPHB\tPHB\tPHB\tPHB\t0-191\t0-1\nGPU4\tPHB\tPHB\tPHB\tPHB\t X \tPHB\tPHB\tPHB\t0-191\t0-1\nGPU5\tPHB\tPHB\tPHB\tPHB\tPHB\t X \tPHB\tPHB\t0-191\t0-1\nGPU6\tPHB\tPHB\tPHB\tPHB\tPHB\tPHB\t X \tPHB\t0-191\t0-1\nGPU7\tPHB\tPHB\tPHB\tPHB\tPHB\tPHB\tPHB\t X \t0-191\t0-1\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5.8xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA A10G, 0, 00000000:00:1E.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5.8xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-31\n/sys/devices/system/node/node0/distance:10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5.8xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA A10G, 00000000:00:1E.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5.8xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \t0-31\t0\t\tN/A\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5g.2xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA T4G, 0, 00000000:00:1F.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5g.2xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-7\n/sys/devices/system/node/node0/distance:10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5g.2xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA T4G, 00000000:00:1F.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g5g.2xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \t0-7\t0\t\tN/A\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.2xlarge/efa_count.txt", "content": "0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.2xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA L4-6Q, 0, 00000000:31:00.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.2xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-7\n/sys/devices/system/node/node0/distance:10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.2xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA L4-6Q, 00000000:31:00.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.2xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \t0-7\t0\t\tN/A\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.2xlarge/nvidia_vgpu_license_status.txt", "content": " vGPU Software Licensed Product\n Product Name : NVIDIA RTX Virtual Workstation\n License Status : Licensed (Expiry: N/A)\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.4xlarge/efa_count.txt", "content": "0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.4xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA L4-12Q, 0, 00000000:35:00.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.4xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-15\n/sys/devices/system/node/node0/distance:10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.4xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA L4-12Q, 00000000:35:00.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.4xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \t0-15\t0\t\tN/A\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.4xlarge/nvidia_vgpu_license_status.txt", "content": " vGPU Software Licensed Product\n Product Name : NVIDIA RTX Virtual Workstation\n License Status : Licensed (Expiry: N/A)\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.large/efa_count.txt", "content": "0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.large/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA L4-3Q, 0, 00000000:31:00.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.large/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-1\n/sys/devices/system/node/node0/distance:10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.large/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA L4-3Q, 00000000:31:00.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.large/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \t0-1\t0\t\tN/A\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.large/nvidia_vgpu_license_status.txt", "content": " vGPU Software Licensed Product\n Product Name : NVIDIA RTX Virtual Workstation\n License Status : Licensed (Expiry: N/A)\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.xlarge/efa_count.txt", "content": "0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA L4-3Q, 0, 00000000:31:00.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-3\n/sys/devices/system/node/node0/distance:10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA L4-3Q, 00000000:31:00.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \t0-3\t0\t\tN/A\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/g6f.xlarge/nvidia_vgpu_license_status.txt", "content": " vGPU Software Licensed Product\n Product Name : NVIDIA RTX Virtual Workstation\n License Status : Licensed (Expiry: N/A)\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p3.16xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nTesla V100-SXM2-16GB, 0, 00000000:00:17.0\nTesla V100-SXM2-16GB, 1, 00000000:00:18.0\nTesla V100-SXM2-16GB, 2, 00000000:00:19.0\nTesla V100-SXM2-16GB, 3, 00000000:00:1A.0\nTesla V100-SXM2-16GB, 4, 00000000:00:1B.0\nTesla V100-SXM2-16GB, 5, 00000000:00:1C.0\nTesla V100-SXM2-16GB, 6, 00000000:00:1D.0\nTesla V100-SXM2-16GB, 7, 00000000:00:1E.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p3.16xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-15,32-47\n/sys/devices/system/node/node1/cpulist:16-31,48-63\n/sys/devices/system/node/node0/distance:10 21\n/sys/devices/system/node/node1/distance:21 10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p3.16xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nTesla V100-SXM2-16GB, 00000000:00:17.0, Enabled\nTesla V100-SXM2-16GB, 00000000:00:18.0, Enabled\nTesla V100-SXM2-16GB, 00000000:00:19.0, Enabled\nTesla V100-SXM2-16GB, 00000000:00:1A.0, Enabled\nTesla V100-SXM2-16GB, 00000000:00:1B.0, Enabled\nTesla V100-SXM2-16GB, 00000000:00:1C.0, Enabled\nTesla V100-SXM2-16GB, 00000000:00:1D.0, Enabled\nTesla V100-SXM2-16GB, 00000000:00:1E.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p3.16xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tCPU Affinity\tNUMA Affinity\nGPU0\t X \tNV1\tNV1\tNV2\tNV2\tPHB\tPHB\tPHB\t0-63\t0-1\nGPU1\tNV1\t X \tNV2\tNV1\tPHB\tNV2\tPHB\tPHB\t0-63\t0-1\nGPU2\tNV1\tNV2\t X \tNV2\tPHB\tPHB\tNV1\tPHB\t0-63\t0-1\nGPU3\tNV2\tNV1\tNV2\t X \tPHB\tPHB\tPHB\tNV1\t0-63\t0-1\nGPU4\tNV2\tPHB\tPHB\tPHB\t X \tNV1\tNV1\tNV2\t0-63\t0-1\nGPU5\tPHB\tNV2\tPHB\tPHB\tNV1\t X \tNV2\tNV1\t0-63\t0-1\nGPU6\tPHB\tPHB\tNV1\tPHB\tNV1\tNV2\t X \tNV2\t0-63\t0-1\nGPU7\tPHB\tPHB\tPHB\tNV1\tNV2\tNV1\tNV2\t X \t0-63\t0-1\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p3.2xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nTesla V100-SXM2-16GB, 0, 00000000:00:1E.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p3.2xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-7\n/sys/devices/system/node/node0/distance:10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p3.2xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nTesla V100-SXM2-16GB, 00000000:00:1E.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p3.2xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \t0-7\t0\t\tN/A\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p4d.24xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA A100-SXM4-40GB, 0, 00000000:10:1C.0\nNVIDIA A100-SXM4-40GB, 1, 00000000:10:1D.0\nNVIDIA A100-SXM4-40GB, 2, 00000000:20:1C.0\nNVIDIA A100-SXM4-40GB, 3, 00000000:20:1D.0\nNVIDIA A100-SXM4-40GB, 4, 00000000:90:1C.0\nNVIDIA A100-SXM4-40GB, 5, 00000000:90:1D.0\nNVIDIA A100-SXM4-40GB, 6, 00000000:A0:1C.0\nNVIDIA A100-SXM4-40GB, 7, 00000000:A0:1D.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p4d.24xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-23,48-71\n/sys/devices/system/node/node1/cpulist:24-47,72-95\n/sys/devices/system/node/node0/distance:10 21\n/sys/devices/system/node/node1/distance:21 10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p4d.24xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA A100-SXM4-40GB, 00000000:10:1C.0, Enabled\nNVIDIA A100-SXM4-40GB, 00000000:10:1D.0, Enabled\nNVIDIA A100-SXM4-40GB, 00000000:20:1C.0, Enabled\nNVIDIA A100-SXM4-40GB, 00000000:20:1D.0, Enabled\nNVIDIA A100-SXM4-40GB, 00000000:90:1C.0, Enabled\nNVIDIA A100-SXM4-40GB, 00000000:90:1D.0, Enabled\nNVIDIA A100-SXM4-40GB, 00000000:A0:1C.0, Enabled\nNVIDIA A100-SXM4-40GB, 00000000:A0:1D.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p4d.24xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tCPU Affinity\tNUMA Affinity\nGPU0\t X \tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\t0-23,48-71\t0\nGPU1\tNV12\t X \tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\t0-23,48-71\t0\nGPU2\tNV12\tNV12\t X \tNV12\tNV12\tNV12\tNV12\tNV12\t0-23,48-71\t0\nGPU3\tNV12\tNV12\tNV12\t X \tNV12\tNV12\tNV12\tNV12\t0-23,48-71\t0\nGPU4\tNV12\tNV12\tNV12\tNV12\t X \tNV12\tNV12\tNV12\t24-47,72-95\t1\nGPU5\tNV12\tNV12\tNV12\tNV12\tNV12\t X \tNV12\tNV12\t24-47,72-95\t1\nGPU6\tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\t X \tNV12\t24-47,72-95\t1\nGPU7\tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\t X \t24-47,72-95\t1\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p4de.24xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA A100-SXM4-80GB, 0, 00000000:10:1C.0\nNVIDIA A100-SXM4-80GB, 1, 00000000:10:1D.0\nNVIDIA A100-SXM4-80GB, 2, 00000000:20:1C.0\nNVIDIA A100-SXM4-80GB, 3, 00000000:20:1D.0\nNVIDIA A100-SXM4-80GB, 4, 00000000:90:1C.0\nNVIDIA A100-SXM4-80GB, 5, 00000000:90:1D.0\nNVIDIA A100-SXM4-80GB, 6, 00000000:A0:1C.0\nNVIDIA A100-SXM4-80GB, 7, 00000000:A0:1D.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p4de.24xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-23,48-71\n/sys/devices/system/node/node1/cpulist:24-47,72-95\n/sys/devices/system/node/node0/distance:10 21\n/sys/devices/system/node/node1/distance:21 10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p4de.24xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA A100-SXM4-80GB, 00000000:10:1C.0, Enabled\nNVIDIA A100-SXM4-80GB, 00000000:10:1D.0, Enabled\nNVIDIA A100-SXM4-80GB, 00000000:20:1C.0, Enabled\nNVIDIA A100-SXM4-80GB, 00000000:20:1D.0, Enabled\nNVIDIA A100-SXM4-80GB, 00000000:90:1C.0, Enabled\nNVIDIA A100-SXM4-80GB, 00000000:90:1D.0, Enabled\nNVIDIA A100-SXM4-80GB, 00000000:A0:1C.0, Enabled\nNVIDIA A100-SXM4-80GB, 00000000:A0:1D.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p4de.24xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tCPU Affinity\tNUMA Affinity\nGPU0\t X \tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\t0-23,48-71\t0\nGPU1\tNV12\t X \tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\t0-23,48-71\t0\nGPU2\tNV12\tNV12\t X \tNV12\tNV12\tNV12\tNV12\tNV12\t0-23,48-71\t0\nGPU3\tNV12\tNV12\tNV12\t X \tNV12\tNV12\tNV12\tNV12\t0-23,48-71\t0\nGPU4\tNV12\tNV12\tNV12\tNV12\t X \tNV12\tNV12\tNV12\t24-47,72-95\t1\nGPU5\tNV12\tNV12\tNV12\tNV12\tNV12\t X \tNV12\tNV12\t24-47,72-95\t1\nGPU6\tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\t X \tNV12\t24-47,72-95\t1\nGPU7\tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\tNV12\t X \t24-47,72-95\t1\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p5.48xlarge/gpu_count.txt", "content": "name, index, pci.bus_id\nNVIDIA H100 80GB HBM3, 0, 00000000:53:00.0\nNVIDIA H100 80GB HBM3, 1, 00000000:64:00.0\nNVIDIA H100 80GB HBM3, 2, 00000000:75:00.0\nNVIDIA H100 80GB HBM3, 3, 00000000:86:00.0\nNVIDIA H100 80GB HBM3, 4, 00000000:97:00.0\nNVIDIA H100 80GB HBM3, 5, 00000000:A8:00.0\nNVIDIA H100 80GB HBM3, 6, 00000000:B9:00.0\nNVIDIA H100 80GB HBM3, 7, 00000000:CA:00.0\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p5.48xlarge/numa_topo.txt", "content": "/sys/devices/system/node/node0/cpulist:0-47,96-143\n/sys/devices/system/node/node1/cpulist:48-95,144-191\n/sys/devices/system/node/node0/distance:10 32\n/sys/devices/system/node/node1/distance:32 10\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p5.48xlarge/nvidia_persistence_status.txt", "content": "name, pci.bus_id, persistence_mode\nNVIDIA H100 80GB HBM3, 00000000:53:00.0, Enabled\nNVIDIA H100 80GB HBM3, 00000000:64:00.0, Enabled\nNVIDIA H100 80GB HBM3, 00000000:75:00.0, Enabled\nNVIDIA H100 80GB HBM3, 00000000:86:00.0, Enabled\nNVIDIA H100 80GB HBM3, 00000000:97:00.0, Enabled\nNVIDIA H100 80GB HBM3, 00000000:A8:00.0, Enabled\nNVIDIA H100 80GB HBM3, 00000000:B9:00.0, Enabled\nNVIDIA H100 80GB HBM3, 00000000:CA:00.0, Enabled\n" }, { "path": "test/images/nvidia/gpu_unit_tests/tests/test_sysinfo.sh.data/p5.48xlarge/nvidia_smi_topo.txt", "content": "\t\u001b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tCPU Affinity\tNUMA Affinity\nGPU0\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t0-47,96-143\t0\nGPU1\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t0-47,96-143\t0\nGPU2\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\t0-47,96-143\t0\nGPU3\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\t0-47,96-143\t0\nGPU4\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\t48-95,144-191\t1\nGPU5\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\t48-95,144-191\t1\nGPU6\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\t48-95,144-191\t1\nGPU7\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \t48-95,144-191\t1\n" }, { "path": "test/images/nvidia/gpu_unit_tests/unit_test", "content": "#!/usr/bin/env bash\n\nset -o errexit\nset -o nounset\nset -o pipefail\n\nTRACE_LOG=trace.log\nTEST_TIMEOUT=3600\nBASH=\"/usr/bin/bash\"\nCURRENT_DIR=$(pwd)\nSKIP_TESTS_SUBCOMMAND=${SKIP_TESTS_SUBCOMMAND:-\"\"}\n\ntimeout -k 10 ${TEST_TIMEOUT} ${BASH} gpu_unit_tests/bash_unit -f tap ${SKIP_TESTS_SUBCOMMAND} -t gpu_unit_tests/${TRACE_LOG} gpu_unit_tests/tests/*test*.sh\n" }, { "path": "test/images/nvidia-inference/Dockerfile", "content": "###############################################################################\n# Base image, arguments, and environment\n###############################################################################\nARG CUDA_MAJOR_VERSION=12\nARG CUDA_MINOR_VERSION=8\n\nFROM nvidia/cuda:$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION.0-devel-ubuntu22.04\n\nARG CUDA_MAJOR_VERSION\nARG CUDA_MINOR_VERSION\n\n# Disable interactive prompts\nENV DEBIAN_FRONTEND=noninteractive\n\n###############################################################################\n# System packages\n###############################################################################\nRUN apt update \\\n && apt upgrade -y \\\n && apt install -y --no-install-recommends \\\n build-essential \\\n ca-certificates \\\n cmake \\\n curl \\\n emacs \\\n git \\\n jq \\\n libopencv-dev \\\n software-properties-common \\\n wget \\\n unzip \\\n vim \\\n pkg-config \\\n gdb \\\n lcov \\\n libbz2-dev \\\n zlib1g-dev \\\n openssl \\\n libssl-dev \\\n libsqlite3-dev \\\n libgdbm-dev \\\n libc6-dev \\\n libbz2-dev \\\n libncurses-dev \\\n tk-dev \\\n libffi-dev \\\n libcap-dev \\\n gnupg2 \\\n gpg-agent \\\n && rm -rf /var/lib/apt/lists/*\n\n###############################################################################\n# Build and install Python from source\n###############################################################################\nARG PYTHON=python3.10\nARG PYTHON_VERSION=3.10.12\n\nRUN curl -sL https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz | tar xvz -C /tmp \\\n && cd /tmp/Python-$PYTHON_VERSION \\\n && ./configure --enable-shared --prefix=/usr/local \\\n && make -j$(nproc) \\\n && make install \\\n && cd && rm -rf /tmp/Python-$PYTHON_VERSION\n\nRUN ln -s /usr/local/bin/pip3 /usr/bin/pip \\\n && ln -s /usr/local/bin/$PYTHON /usr/local/bin/python \\\n && pip3 --no-cache-dir install --upgrade pip setuptools\n\n###############################################################################\n# Install Pytorch from Source\n###############################################################################\nARG PYTORCH_BRANCH=v2.6.0\nARG PYTORCH_BUILD_ENV=\"MAX_JOBS=8 BUILD_TEST=0\"\n\n# envs needed to make the path of NVCC known to the compilation\nENV CUDA_HOME=/usr/local/cuda\nENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64\nENV PATH=$PATH:$CUDA_HOME/bin\nENV TORCH_CUDA_ARCH_LIST=\"7.5;8.0;8.6;8.7;8.9;9.0;10.0;12.0\"\n\nRUN pip3 install typing-extensions sympy pyyaml\nRUN git clone https://github.com/pytorch/pytorch.git /tmp/pytorch \\\n --recursive \\\n --branch $PYTORCH_BRANCH \\\n && cd /tmp/pytorch \\\n && eval \"$PYTORCH_BUILD_ENV python3 setup.py install\" \\\n && cd && rm -rf /tmp/pytorch\n\n###############################################################################\n# Application files and Python dependencies\n###############################################################################\nWORKDIR /app\nCOPY infer.py /app/\nCOPY requirements.txt /app/\nRUN pip install --no-cache-dir -r requirements.txt\n" }, { "path": "test/images/nvidia-inference/infer.py", "content": "import logging\nimport os\nimport sys\nimport time\nimport random\n\nimport torch\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom transformers import BertForPreTraining, BertTokenizer\n\nlogging.basicConfig(\n level=logging.INFO,\n format='[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s',\n handlers=[logging.StreamHandler(sys.stdout)]\n)\nlogger = logging.getLogger(\"BERTInference\")\n\n\ndef create_dummy_data(tokenizer, batch_size, num_samples=100, max_length=128, seed=42):\n \"\"\"\n Creates a realistic NSP-style dataset:\n - 50% true next-sentence pairs\n - 50% random second sentences\n Ensures the final number of samples is a multiple of 'batch_size'.\n \"\"\"\n random.seed(seed)\n\n if num_samples % batch_size != 0:\n adjusted = (num_samples // batch_size) * batch_size\n logger.info(\n f\"[INFO] Adjusting num_samples from {num_samples} to {adjusted} \"\n f\"to ensure full batches.\"\n )\n num_samples = adjusted\n\n sample_sentences = [\n \"The dog loves playing fetch in the park.\",\n \"Artificial intelligence is reshaping the future.\",\n \"Movies with complex storylines can be very engaging.\",\n \"This restaurant serves an amazing brunch on weekends.\",\n \"Many researchers are exploring neural network architectures.\",\n \"A day at the beach can reduce stress and improve well-being.\",\n \"ChatGPT is a popular large language model by OpenAI.\",\n \"The annual developer conference showcased innovative technologies.\",\n \"Hiking in the mountains offers both challenge and relaxation.\",\n \"Robotics and automation are revolutionizing many industries.\",\n ]\n\n sentences_a = []\n sentences_b = []\n nsp_labels = []\n\n for _ in range(num_samples):\n idx_a = random.randint(0, len(sample_sentences) - 1)\n if random.random() < 0.5:\n idx_b = (idx_a + 1) % len(sample_sentences)\n nsp_labels.append(1)\n else:\n idx_b = random.randint(0, len(sample_sentences) - 1)\n nsp_labels.append(0)\n\n sentences_a.append(sample_sentences[idx_a])\n sentences_b.append(sample_sentences[idx_b])\n\n tokenized_inputs = tokenizer(\n sentences_a,\n sentences_b,\n max_length=max_length,\n padding=\"max_length\",\n truncation=True,\n return_tensors=\"pt\",\n )\n\n return TensorDataset(\n tokenized_inputs.input_ids,\n tokenized_inputs.attention_mask,\n torch.tensor(nsp_labels, dtype=torch.long)\n )\n\n\ndef run_inference(model, tokenizer, batch_size, mode, device):\n \"\"\"\n Runs a dummy BERT inference workload using the given model and tokenizer.\n Calculates average time per batch and throughput.\n Expects 'device' to be GPU only (validated in main()).\n \"\"\"\n model.to(device)\n model.eval()\n\n try:\n dataset = create_dummy_data(tokenizer, batch_size=batch_size, num_samples=100, max_length=128)\n except Exception:\n logger.exception(\"[ERROR] Failed to create dummy data.\")\n raise\n\n dataloader = DataLoader(dataset, batch_size=batch_size)\n total_time = 0.0\n total_batches = len(dataloader)\n\n with torch.no_grad():\n for batch_idx, batch in enumerate(dataloader):\n try:\n inputs, masks, next_sentence_labels = batch\n inputs, masks, next_sentence_labels = (\n inputs.to(device),\n masks.to(device),\n next_sentence_labels.to(device),\n )\n\n start_time = time.time()\n _ = model(\n input_ids=inputs,\n attention_mask=masks,\n next_sentence_label=next_sentence_labels\n )\n end_time = time.time()\n\n except Exception:\n logger.exception(f\"[ERROR] Inference failed on batch {batch_idx}.\")\n raise\n\n total_time += (end_time - start_time)\n\n if total_time == 0.0:\n avg_time_per_batch = float('inf')\n throughput = 0.0\n else:\n avg_time_per_batch = total_time / total_batches\n throughput = (total_batches * batch_size) / total_time\n\n logger.info(\n \"[BERT_INFERENCE_METRICS] \"\n f\"mode={mode} \"\n f\"avg_time_per_batch={avg_time_per_batch:.6f} \"\n f\"throughput_samples_per_sec={throughput:.6f}\"\n )\n\n\ndef main():\n \"\"\"\n Main entry point. Checks for GPU availability, determines inference mode,\n sets batch size, and runs inference. Logs throughput and timing stats.\n \"\"\"\n if not torch.cuda.is_available():\n logger.error(\"[ERROR] GPU is not available. Exiting.\")\n sys.exit(1)\n\n device = torch.device(\"cuda\")\n num_gpus = torch.cuda.device_count()\n logger.info(f\"[INFO] Found {num_gpus} GPU(s). GPU is available.\")\n\n mode = os.environ.get(\"INFERENCE_MODE\", \"throughput\").lower()\n if mode not in [\"throughput\", \"latency\"]:\n logger.warning(\n f\"[WARNING] Unrecognized INFERENCE_MODE '{mode}'. \"\n \"Falling back to 'throughput'.\"\n )\n mode = \"throughput\"\n\n batch_size = 1 if mode == \"latency\" else 8\n logger.info(f\"[INFO] Running inference in {mode} mode with batch size {batch_size}.\")\n\n try:\n tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n model = BertForPreTraining.from_pretrained(\"bert-base-uncased\")\n except Exception:\n logger.exception(\"[ERROR] Failed to load model/tokenizer. Exiting.\")\n sys.exit(1)\n\n run_inference(model, tokenizer, batch_size, mode, device)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "test/images/nvidia-inference/requirements.txt", "content": "transformers==4.53.0\nnumpy==1.26\n" }, { "path": "test/images/nvidia-training/Dockerfile", "content": "ARG CUDA_MAJOR_VERSION=12\nARG CUDA_MINOR_VERSION=8\n\n# Use the NVIDIA CUDA runtime as a parent image\nFROM nvidia/cuda:$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION.0-devel-ubuntu22.04\n\n# Redeclare build arguments\nARG CUDA_MAJOR_VERSION\nARG CUDA_MINOR_VERSION\n\n# Set environment variable to disable interactive prompts\nENV DEBIAN_FRONTEND=noninteractive\n\n# Set default values for MASTER_ADDR, MASTER_PORT, and NUM_GPUS_PER_NODE\nENV MASTER_ADDR=127.0.0.1\nENV MASTER_PORT=12355\n\nRUN apt-get update \\\n && apt-get upgrade -y \\\n && apt-get install -y --no-install-recommends \\\n build-essential \\\n ca-certificates \\\n cmake \\\n curl \\\n emacs \\\n git \\\n jq \\\n libopencv-dev \\\n software-properties-common \\\n wget \\\n unzip \\\n vim \\\n pkg-config \\\n gdb \\\n lcov \\\n libbz2-dev \\\n zlib1g-dev \\\n openssl \\\n libssl-dev \\\n libsqlite3-dev \\\n libgdbm-dev \\\n libc6-dev \\\n libbz2-dev \\\n libncurses-dev \\\n tk-dev \\\n libffi-dev \\\n libcap-dev \\\n gnupg2 \\\n gpg-agent \\\n && rm -rf /var/lib/apt/lists/*\n\n# Install Python\nARG PYTHON=python3.10\nARG PYTHON_VERSION=3.10.12\n\nRUN curl -sL https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz | tar xvz -C /tmp \\\n && cd /tmp/Python-$PYTHON_VERSION \\\n && ./configure --enable-shared --prefix=/usr/local \\\n && make -j $(nproc) \\\n && make install \\\n && cd && rm -rf /tmp/Python-$PYTHON_VERSION\n\nRUN ln -s /usr/local/bin/pip3 /usr/bin/pip \\\n && ln -s /usr/local/bin/$PYTHON /usr/local/bin/python \\\n && pip --no-cache-dir install --upgrade pip setuptools\n\n# Install Pytorch from Source\nARG PYTORCH_BRANCH=v2.6.0\nARG PYTORCH_BUILD_ENV=\"MAX_JOBS=8 BUILD_TEST=0\"\n\nENV CUDA_HOME=/usr/local/cuda\nENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64\nENV PATH=$PATH:$CUDA_HOME/bin\nENV TORCH_CUDA_ARCH_LIST=\"7.5;8.0;8.6;8.7;8.9;9.0;10.0;12.0\"\n\nRUN pip install typing-extensions sympy pyyaml\nRUN git clone https://github.com/pytorch/pytorch.git /tmp/pytorch \\\n --recursive \\\n --branch $PYTORCH_BRANCH \\\n && cd /tmp/pytorch \\\n && eval \"$PYTORCH_BUILD_ENV python3 setup.py install\" \\\n && cd && rm -rf /tmp/pytorch\n\nRUN apt-get update -y && \\\n apt-get remove -y --allow-change-held-packages \\\n libmlx5-1 ibverbs-utils libibverbs-dev libibverbs1 libnccl2 libnccl-dev && \\\n rm -rf /opt/hpcx /usr/local/mpi /usr/local/ucx /etc/ld.so.conf.d/hpcx.conf\n\nRUN apt-get install -y --allow-unauthenticated \\\n sudo git gcc vim kmod openssh-client openssh-server build-essential \\\n wget curl autoconf libtool gdb automake python3-distutils cmake \\\n apt-utils devscripts debhelper libsubunit-dev check pkg-config libhwloc-dev\n\nRUN ldconfig\n\n# SSH configuration\nRUN mkdir -p /var/run/sshd && \\\n sed -i 's/[ #]\\(.*StrictHostKeyChecking \\).*/ \\1no/g' /etc/ssh/ssh_config && \\\n echo \" UserKnownHostsFile /dev/null\" >> /etc/ssh/ssh_config && \\\n sed -i 's/#\\(StrictModes \\).*/\\1no/g' /etc/ssh/sshd_config\n\n# Set environment variables for OpenMPI, CUDA, EFA, and NCCL\nENV LD_LIBRARY_PATH /opt/amazon/openmpi/lib64:/opt/amazon/openmpi/lib:/opt/amazon/efa/lib64:/opt/aws-ofi-nccl/install/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib/:/usr/lib64:/usr/lib/x86_64-linux-gnu/:/usr/lib/aarch64-linux-gnu/:$LD_LIBRARY_PATH\nENV PATH /usr/local/cuda/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/sbin:/usr/bin:/usr/local/bin:$PATH\n\n# Install EFA\nARG EFA_INSTALLER_VERSION=latest\nRUN curl -sL https://efa-installer.amazonaws.com/aws-efa-installer-$EFA_INSTALLER_VERSION.tar.gz | tar xvz -C /tmp \\\n && cd /tmp/aws-efa-installer \\\n && ./efa_installer.sh -y -g -d --skip-kmod --skip-limit-conf --no-verify \\\n && cd && rm -rf /tmp/aws-efa-installer\n\n# Install NCCL\nARG LIBNCCL_VERSION=2.28.7-1\nRUN git clone https://github.com/NVIDIA/nccl.git --branch v$LIBNCCL_VERSION /tmp/nccl \\\n && cd /tmp/nccl \\\n && make -j $(nproc) \\\n && make install \\\n && cd && rm -rf /tmp/nccl\n\n# Install AWS-OFI-NCCL plugin\nARG AWS_OFI_NCCL_VERSION=1.17.2\nRUN curl -sL https://github.com/aws/aws-ofi-nccl/releases/download/v$AWS_OFI_NCCL_VERSION/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION.tar.gz | tar xvz -C /tmp \\\n && cd /tmp/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION \\\n && ./configure \\\n --prefix=/opt/aws-ofi-nccl/install \\\n --with-mpi=/opt/amazon/openmpi \\\n --with-libfabric=/opt/amazon/efa \\\n --with-cuda=/usr/local/cuda \\\n --enable-platform-aws \\\n --disable-tests \\\n && make -j $(nproc) \\\n && make install \\\n && cd && rm -rf /tmp/aws-ofi-nccl-$AWS_OFI_NCCL_VERSION\n\nENV NCCL_PROTO simple\nENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH\n\nRUN rm -rf /var/lib/apt/lists/*\n\n# Set the working directory in the container\nWORKDIR /app\n\n# Copy the training script and install requirements\nCOPY train.py /app/\nCOPY requirements.txt /app/\nRUN pip install --no-cache-dir -r requirements.txt\n" }, { "path": "test/images/nvidia-training/requirements.txt", "content": "transformers==4.53.0\nnumpy==1.26\n" }, { "path": "test/images/nvidia-training/train.py", "content": "import os\nimport time\nimport torch\nimport torch.distributed as dist\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nfrom transformers import BertForPreTraining, BertTokenizer\nfrom torch.utils.data import DataLoader, TensorDataset\nimport numpy as np\n\n\ndef create_dummy_data(tokenizer, num_samples=100, max_length=128):\n sentences = [f\"This is a dummy sentence number {i}\" for i in range(num_samples)]\n tokenized_inputs = tokenizer(\n sentences,\n max_length=max_length,\n padding=\"max_length\",\n truncation=True,\n return_tensors=\"pt\",\n )\n labels = tokenized_inputs.input_ids.detach().clone()\n\n # MLM task: randomly mask some tokens\n mlm_probability = 0.15\n input_ids, labels = mask_tokens(tokenized_inputs.input_ids, tokenizer, mlm_probability)\n\n # NSP task: create dummy pairs\n next_sentence_labels = torch.randint(0, 2, (num_samples,))\n\n return TensorDataset(input_ids, tokenized_inputs.attention_mask, labels, next_sentence_labels)\n\n\ndef mask_tokens(inputs, tokenizer, mlm_probability):\n labels = inputs.clone()\n probability_matrix = torch.full(labels.shape, mlm_probability)\n special_tokens_mask = [\n tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True)\n for val in labels.tolist()\n ]\n probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0)\n masked_indices = torch.bernoulli(probability_matrix).bool()\n labels[~masked_indices] = -100\n inputs[masked_indices] = tokenizer.convert_tokens_to_ids(tokenizer.mask_token)\n return inputs, labels\n\n\ndef setup(rank, world_size, local_rank):\n master_addr = os.environ[\"MASTER_ADDR\"]\n master_port = os.environ[\"MASTER_PORT\"]\n dist.init_process_group(\n \"nccl\",\n init_method=f\"tcp://{master_addr}:{master_port}\",\n rank=rank,\n world_size=world_size,\n )\n torch.cuda.set_device(local_rank)\n print(f\"Process {rank} initialized, using GPU {local_rank}\")\n\n\ndef cleanup():\n dist.destroy_process_group()\n\n\ndef train_bert(rank, world_size, local_rank, model, tokenizer):\n setup(rank, world_size, local_rank)\n\n model = model.to(local_rank)\n ddp_model = DDP(model, device_ids=[local_rank])\n\n dataset = create_dummy_data(tokenizer)\n train_dataloader = DataLoader(dataset, batch_size=8)\n\n optimizer = torch.optim.AdamW(ddp_model.parameters(), lr=0.001)\n\n start_time = time.time()\n\n # Simple single-epoch training loop\n for epoch in range(1):\n ddp_model.train()\n for batch in train_dataloader:\n optimizer.zero_grad()\n inputs, masks, labels, next_sentence_labels = batch\n inputs = inputs.to(local_rank)\n masks = masks.to(local_rank)\n labels = labels.to(local_rank)\n next_sentence_labels = next_sentence_labels.to(local_rank)\n\n outputs = ddp_model(\n input_ids=inputs,\n attention_mask=masks,\n labels=labels,\n next_sentence_label=next_sentence_labels,\n )\n loss = outputs.loss\n loss.backward()\n optimizer.step()\n\n end_time = time.time()\n training_time = end_time - start_time\n throughput = len(dataset) / training_time\n\n print(f\"Process {rank} - Training time: {training_time:.2f} seconds\")\n print(f\"Process {rank} - Throughput: {throughput:.2f} samples/second\")\n\n cleanup()\n\n return throughput\n\n\ndef main():\n # Retrieve environment variables\n rank = int(os.getenv(\"OMPI_COMM_WORLD_RANK\", \"0\"))\n world_size = int(os.getenv(\"OMPI_COMM_WORLD_SIZE\", \"1\"))\n local_rank = int(os.getenv(\"OMPI_COMM_WORLD_LOCAL_RANK\", \"0\"))\n\n print(f\"Process started for rank {rank} with local rank {local_rank}\")\n\n # Pre-download model and tokenizer\n tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n model = BertForPreTraining.from_pretrained(\"bert-base-uncased\")\n\n print(f\"successfully downloaded model and tokenizer for rank: {rank}\")\n\n throughput = train_bert(rank, world_size, local_rank, model, tokenizer)\n\n # Only rank 0 prints the \"Average Throughput\" line\n if rank == 0:\n print(f\"Average Throughput: {throughput:.2f} samples/second\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "test/manifests/assets/cloudwatch-agent.yaml", "content": "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: prometheus-cwagentconfig\n namespace: amazon-cloudwatch\ndata:\n cwagentconfig.json: |\n {\n \"agent\": {\n \"debug\": true\n },\n \"logs\": {\n \"metrics_collected\": {\n \"prometheus\": {\n \"prometheus_config_path\": \"/etc/prometheusconfig/prometheus.yaml\",\n \"emf_processor\": {\n \"metric_declaration\": [\n {\n \"source_labels\": [\"job\"],\n \"label_matcher\": \"dcgm-exporter\",\n \"dimensions\": [[{{.DimensionKeys}}]],\n \"metric_selectors\": [\n \"^DCGM_FI_DEV_GPU_UTIL$\",\n \"^DCGM_FI_DEV_MEM_COPY_UTIL$\",\n \"^DCGM_FI_DEV_FB_USED$\",\n \"^DCGM_FI_DEV_FB_FREE$\",\n \"^DCGM_FI_DEV_POWER_USAGE$\"\n ]\n }\n ]\n }\n }\n },\n \"force_flush_interval\": 5\n }\n }\n\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: prometheus-config\n namespace: amazon-cloudwatch\ndata:\n prometheus.yaml: |\n global:\n scrape_interval: 1s\n scrape_timeout: 1s\n scrape_configs:\n - job_name: dcgm-exporter\n static_configs:\n - targets:\n - dcgm-exporter.kube-system.svc.cluster.local:9400\n metrics_path: /metrics\n metric_relabel_configs:\n{{- range $key, $value := .MetricDimensions}}\n - {action: replace, target_label: {{$key}}, replacement: '{{$value}}'}\n{{- end}}\n---\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: cwagent\n namespace: amazon-cloudwatch\nspec:\n selector:\n matchLabels:\n app: cwagent\n template:\n metadata:\n labels:\n app: cwagent\n spec:\n serviceAccountName: cwagent\n dnsPolicy: ClusterFirst \n containers:\n - name: cloudwatch-agent\n image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest\n imagePullPolicy: Always\n resources:\n limits:\n cpu: 1000m\n memory: 1000Mi\n requests:\n cpu: 200m\n memory: 200Mi\n volumeMounts:\n - name: prometheus-cwagentconfig\n mountPath: /etc/cwagentconfig\n - name: prometheus-config\n mountPath: /etc/prometheusconfig\n volumes:\n - name: prometheus-cwagentconfig\n configMap:\n name: prometheus-cwagentconfig\n - name: prometheus-config\n configMap:\n name: prometheus-config\n terminationGracePeriodSeconds: 60\n---" }, { "path": "test/manifests/assets/dcgm-exporter.yaml", "content": "# Derived from: Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: \"dcgm-exporter\"\n namespace: \"kube-system\"\n labels:\n app.kubernetes.io/name: \"dcgm-exporter\"\n app.kubernetes.io/version: \"4.1.3\"\nspec:\n updateStrategy:\n type: RollingUpdate\n selector:\n matchLabels:\n app.kubernetes.io/name: \"dcgm-exporter\"\n app.kubernetes.io/version: \"4.1.3\"\n template:\n metadata:\n labels:\n app.kubernetes.io/name: \"dcgm-exporter\"\n app.kubernetes.io/version: \"4.1.3\"\n name: \"dcgm-exporter\"\n spec:\n containers:\n - image: \"nvcr.io/nvidia/k8s/dcgm-exporter:4.2.3-4.1.3-ubuntu22.04\"\n env:\n - name: \"DCGM_EXPORTER_LISTEN\"\n value: \":9400\"\n - name: \"DCGM_EXPORTER_INTERVAL\"\n value: \"100\"\n - name: \"DCGM_EXPORTER_KUBERNETES\"\n value: \"true\"\n name: \"dcgm-exporter\"\n ports:\n - name: \"metrics\"\n containerPort: 9400\n securityContext:\n runAsNonRoot: false\n runAsUser: 0\n capabilities:\n add: [\"SYS_ADMIN\"]\n volumeMounts:\n - name: \"pod-gpu-resources\"\n readOnly: true\n mountPath: \"/var/lib/kubelet/pod-resources\"\n volumes:\n - name: \"pod-gpu-resources\"\n hostPath:\n path: \"/var/lib/kubelet/pod-resources\"\n\n---\n\nkind: Service\napiVersion: v1\nmetadata:\n name: \"dcgm-exporter\"\n namespace: \"kube-system\"\n labels:\n app.kubernetes.io/name: \"dcgm-exporter\"\n app.kubernetes.io/version: \"4.1.3\"\nspec:\n clusterIP: \"None\"\n selector:\n app.kubernetes.io/name: \"dcgm-exporter\"\n app.kubernetes.io/version: \"4.1.3\"\n ports:\n - name: \"metrics\"\n port: 9400" }, { "path": "test/manifests/assets/dranet.yaml", "content": "---\n# Source: aws-dranet/templates/serviceaccount.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: dranet-aws-dranet\n namespace: kube-system\n labels:\n helm.sh/chart: aws-dranet-1.0.0\n app.kubernetes.io/name: aws-dranet\n app.kubernetes.io/instance: dranet\n app.kubernetes.io/version: \"v1.2.0-eksbuild.2\"\n app.kubernetes.io/managed-by: Helm\n---\n# Source: aws-dranet/templates/clusterrole.yaml\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: dranet-aws-dranet\n labels:\n helm.sh/chart: aws-dranet-1.0.0\n app.kubernetes.io/name: aws-dranet\n app.kubernetes.io/instance: dranet\n app.kubernetes.io/version: \"v1.2.0-eksbuild.2\"\n app.kubernetes.io/managed-by: Helm\nrules:\n - apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - get\n - apiGroups:\n - \"resource.k8s.io\"\n resources:\n - resourceslices\n verbs:\n - list\n - watch\n - create\n - update\n - delete\n - apiGroups:\n - \"resource.k8s.io\"\n resources:\n - resourceclaims\n - deviceclasses\n verbs:\n - get\n - apiGroups:\n - \"resource.k8s.io\"\n resources:\n - resourceclaims/status\n verbs:\n - patch\n - update\n---\n# Source: aws-dranet/templates/clusterrolebinding.yaml\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: dranet-aws-dranet\n labels:\n helm.sh/chart: aws-dranet-1.0.0\n app.kubernetes.io/name: aws-dranet\n app.kubernetes.io/instance: dranet\n app.kubernetes.io/version: \"v1.2.0-eksbuild.2\"\n app.kubernetes.io/managed-by: Helm\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: dranet-aws-dranet\nsubjects:\n- kind: ServiceAccount\n name: dranet-aws-dranet\n namespace: kube-system\n---\n# Source: aws-dranet/templates/daemonset.yaml\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: dranet-aws-dranet\n namespace: kube-system\n labels:\n helm.sh/chart: aws-dranet-1.0.0\n app.kubernetes.io/name: aws-dranet\n app.kubernetes.io/instance: dranet\n app.kubernetes.io/version: \"v1.2.0-eksbuild.2\"\n app.kubernetes.io/managed-by: Helm\n tier: node\n app: dranet-aws-dranet\n k8s-app: dranet-aws-dranet\nspec:\n selector:\n matchLabels:\n app: dranet-aws-dranet\n template:\n metadata:\n labels:\n app.kubernetes.io/name: aws-dranet\n app.kubernetes.io/instance: dranet\n tier: node\n app: dranet-aws-dranet\n k8s-app: dranet-aws-dranet\n spec:\n priorityClassName: \"system-node-critical\"\n hostNetwork: true\n hostPID: false\n tolerations:\n - key: CriticalAddonsOnly\n operator: Exists\n serviceAccountName: dranet-aws-dranet\n containers:\n - name: dranet\n args:\n - /dranet\n - --v=4\n - --hostname-override=$(NODE_NAME)\n - \"--bind-address=:9177\"\n - --cloud-provider-hint=AWS\n - --filter=\"dra.net/pciDevice\" in attributes && attributes[\"dra.net/pciDevice\"].StringValue == \"Elastic Fabric Adapter (EFA)\"\n image: {{.RdmaDeviceDraDriverImage}}\n imagePullPolicy: IfNotPresent\n env:\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n resources:\n limits:\n cpu: 500m\n memory: 256Mi\n requests:\n cpu: 100m\n memory: 50Mi\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - ALL\n privileged: false\n readOnlyRootFilesystem: true\n runAsGroup: 0\n runAsUser: 0\n seccompProfile:\n type: RuntimeDefault\n readinessProbe:\n httpGet:\n path: /healthz\n port: 9177\n volumeMounts:\n - name: device-plugin\n mountPath: /var/lib/kubelet/plugins\n - name: plugin-registry\n mountPath: /var/lib/kubelet/plugins_registry\n - name: nri-plugin\n mountPath: /var/run/nri\n - name: netns\n mountPath: /var/run/netns\n mountPropagation: HostToContainer\n - name: infiniband\n mountPath: /dev/infiniband\n mountPropagation: HostToContainer\n - name: tmp\n mountPath: /tmp\n - name: dranet-run\n mountPath: /var/run/dranet\n volumes:\n - name: device-plugin\n hostPath:\n path: /var/lib/kubelet/plugins\n type: DirectoryOrCreate\n - name: plugin-registry\n hostPath:\n path: /var/lib/kubelet/plugins_registry\n type: DirectoryOrCreate\n - name: nri-plugin\n hostPath:\n path: /var/run/nri\n type: DirectoryOrCreate\n - name: netns\n hostPath:\n path: /var/run/netns\n type: DirectoryOrCreate\n - name: infiniband\n hostPath:\n path: /dev/infiniband\n type: DirectoryOrCreate\n - name: tmp\n emptyDir:\n medium: Memory\n sizeLimit: 10Mi\n - name: dranet-run\n hostPath:\n path: /var/run/dranet\n type: DirectoryOrCreate\n---\n# Source: aws-dranet/templates/deviceclass.yaml\napiVersion: resource.k8s.io/v1\nkind: DeviceClass\nmetadata:\n name: efa.networking.k8s.aws\n labels:\n helm.sh/chart: aws-dranet-1.0.0\n app.kubernetes.io/name: aws-dranet\n app.kubernetes.io/instance: dranet\n app.kubernetes.io/version: \"v1.2.0-eksbuild.2\"\n app.kubernetes.io/managed-by: Helm\nspec:\n selectors:\n - cel:\n expression: |\n device.driver == \"dra.net\" &&\n device.attributes[\"dra.net\"].pciDevice == 'Elastic Fabric Adapter (EFA)'\n" }, { "path": "test/manifests/assets/efa-device-plugin.yaml", "content": "# Source: https://raw.githubusercontent.com/aws-samples/aws-efa-eks/main/manifest/efa-k8s-device-plugin.yml\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: aws-efa-k8s-device-plugin-daemonset\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n name: aws-efa-k8s-device-plugin\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n labels:\n name: aws-efa-k8s-device-plugin\n spec:\n serviceAccount: default\n tolerations:\n - key: CriticalAddonsOnly\n operator: Exists\n - key: aws.amazon.com/efa\n operator: Exists\n effect: NoSchedule\n # Mark this pod as a critical add-on; when enabled, the critical add-on\n # scheduler reserves resources for critical add-on pods so that they can\n # be rescheduled after a failure.\n # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/\n priorityClassName: \"system-node-critical\"\n hostNetwork: true\n containers:\n - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efa-k8s-device-plugin:v0.5.8\n name: aws-efa-k8s-device-plugin\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop: [\"ALL\"]\n runAsNonRoot: false\n volumeMounts:\n - name: device-plugin\n mountPath: /var/lib/kubelet/device-plugins\n - name: infiniband-volume\n mountPath: /dev/infiniband\n resources:\n requests:\n cpu: 10m\n memory: 20Mi\n volumes:\n - name: device-plugin\n hostPath:\n path: /var/lib/kubelet/device-plugins\n - name: infiniband-volume\n hostPath:\n path: /dev/infiniband\n" }, { "path": "test/manifests/assets/k8s-neuron-device-plugin-rbac.yml", "content": "# Source: https://github.com/aws-neuron/aws-neuron-sdk/blob/master/src/k8/k8s-neuron-device-plugin-rbac.yml\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: neuron-device-plugin\nrules:\n- apiGroups:\n - \"\"\n resources:\n - nodes\n verbs:\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n- apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - update\n - patch\n - get\n - list\n - watch\n- apiGroups:\n - \"\"\n resources:\n - nodes/status\n verbs:\n - patch\n - update\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: neuron-device-plugin\n namespace: kube-system\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: neuron-device-plugin\n namespace: kube-system\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: neuron-device-plugin\nsubjects:\n- kind: ServiceAccount\n name: neuron-device-plugin\n namespace: kube-system\n" }, { "path": "test/manifests/assets/k8s-neuron-device-plugin.yml", "content": "# Source: https://github.com/aws-neuron/aws-neuron-sdk/blob/master/src/k8/k8s-neuron-device-plugin.yml\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: neuron-device-plugin-daemonset\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n name: neuron-device-plugin-ds\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n # Uncomment the annotation below if k8s version is 1.13 or lower\n # annotations:\n # scheduler.alpha.kubernetes.io/critical-pod: \"\"\n labels:\n name: neuron-device-plugin-ds\n spec:\n serviceAccount: neuron-device-plugin\n tolerations:\n - key: CriticalAddonsOnly\n operator: Exists\n - key: aws.amazon.com/neuron\n operator: Exists\n effect: NoSchedule\n # Mark this pod as a critical add-on; when enabled, the critical add-on\n # scheduler reserves resources for critical add-on pods so that they can\n # be rescheduled after a failure.\n # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/\n priorityClassName: \"system-node-critical\"\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchExpressions:\n - key: \"node.kubernetes.io/instance-type\"\n operator: In\n values:\n - inf1.xlarge\n - inf1.2xlarge\n - inf1.6xlarge\n - inf1.24xlarge\n - inf2.xlarge\n - inf2.8xlarge\n - inf2.24xlarge\n - inf2.48xlarge\n - trn1.2xlarge\n - trn1.32xlarge\n - trn1n.32xlarge\n - trn2.48xlarge\n - trn2u.48xlarge\n containers:\n # Find all neuron-device-plugin images at https://gallery.ecr.aws/neuron/neuron-device-plugin\n - image: public.ecr.aws/neuron/neuron-device-plugin:2.26.26.0\n imagePullPolicy: Always\n name: neuron-device-plugin\n env:\n - name: KUBECONFIG\n value: /etc/kubernetes/kubelet.conf\n - name: NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop: [\"ALL\"]\n volumeMounts:\n - name: device-plugin\n mountPath: /var/lib/kubelet/device-plugins\n - name: infa-map\n mountPath: /run\n volumes:\n - name: device-plugin\n hostPath:\n path: /var/lib/kubelet/device-plugins\n - name: infa-map\n hostPath:\n path: /run\n\n\n\n" }, { "path": "test/manifests/assets/mpi-operator.yaml", "content": "# --------------------------------------------------\n# - Single configuration deployment YAML for MPI-Operator\n# - Includes:\n# CRD\n# Namespace\n# RBAC\n# Controller deployment\n# --------------------------------------------------\n---\napiVersion: v1\nkind: Namespace\nmetadata:\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n name: mpi-operator\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: v0.20.1\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n name: mpijobs.kubeflow.org\nspec:\n group: kubeflow.org\n names:\n kind: MPIJob\n listKind: MPIJobList\n plural: mpijobs\n singular: mpijob\n scope: Namespaced\n versions:\n - name: v2beta1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n properties:\n launcherCreationPolicy:\n default: AtStartup\n description: launcherCreationPolicy if WaitForWorkersReady, the launcher\n is created only after all workers are in Ready state. Defaults to\n AtStartup.\n type: string\n mpiImplementation:\n default: OpenMPI\n description: |-\n MPIImplementation is the MPI implementation.\n Options are \"OpenMPI\" (default), \"Intel\" and \"MPICH\".\n enum:\n - OpenMPI\n - Intel\n - MPICH\n type: string\n mpiReplicaSpecs:\n additionalProperties:\n description: ReplicaSpec is a description of the replica\n properties:\n replicas:\n description: |-\n Replicas is the desired number of replicas of the given template.\n If unspecified, defaults to 1.\n format: int32\n type: integer\n restartPolicy:\n description: |-\n Restart policy for all replicas within the job.\n One of Always, OnFailure, Never and ExitCode.\n Default to Never.\n type: string\n template:\n description: |-\n Template is the object that describes the pod that\n will be created for this replica. RestartPolicy in PodTemplateSpec\n will be overide by RestartPolicy in ReplicaSpec\n properties:\n metadata:\n description: |-\n Standard object's metadata.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\n properties:\n annotations:\n additionalProperties:\n type: string\n type: object\n finalizers:\n items:\n type: string\n type: array\n labels:\n additionalProperties:\n type: string\n type: object\n name:\n type: string\n namespace:\n type: string\n type: object\n spec:\n description: |-\n Specification of the desired behavior of the pod.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status\n properties:\n activeDeadlineSeconds:\n description: |-\n Optional duration in seconds the pod may be active on the node relative to\n StartTime before the system will actively try to mark it failed and kill associated containers.\n Value must be a positive integer.\n format: int64\n type: integer\n affinity:\n description: If specified, the pod's scheduling constraints\n properties:\n nodeAffinity:\n description: Describes node affinity scheduling\n rules for the pod.\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node matches the corresponding matchExpressions; the\n node(s) with the highest sum are the most preferred.\n items:\n description: |-\n An empty preferred scheduling term matches all objects with implicit weight 0\n (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).\n properties:\n preference:\n description: A node selector term, associated\n with the corresponding weight.\n properties:\n matchExpressions:\n description: A list of node selector\n requirements by node's labels.\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n properties:\n key:\n description: The label key that\n the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchFields:\n description: A list of node selector\n requirements by node's fields.\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n properties:\n key:\n description: The label key that\n the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n type: object\n x-kubernetes-map-type: atomic\n weight:\n description: Weight associated with matching\n the corresponding nodeSelectorTerm,\n in the range 1-100.\n format: int32\n type: integer\n required:\n - preference\n - weight\n type: object\n type: array\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to an update), the system\n may or may not try to eventually evict the pod from its node.\n properties:\n nodeSelectorTerms:\n description: Required. A list of node selector\n terms. The terms are ORed.\n items:\n description: |-\n A null or empty node selector term matches no objects. The requirements of\n them are ANDed.\n The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.\n properties:\n matchExpressions:\n description: A list of node selector\n requirements by node's labels.\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n properties:\n key:\n description: The label key that\n the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchFields:\n description: A list of node selector\n requirements by node's fields.\n items:\n description: |-\n A node selector requirement is a selector that contains values, a key, and an operator\n that relates the key and values.\n properties:\n key:\n description: The label key that\n the selector applies to.\n type: string\n operator:\n description: |-\n Represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.\n type: string\n values:\n description: |-\n An array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. If the operator is Gt or Lt, the values\n array must have a single element, which will be interpreted as an integer.\n This array is replaced during a strategic merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n type: object\n x-kubernetes-map-type: atomic\n type: array\n x-kubernetes-list-type: atomic\n required:\n - nodeSelectorTerms\n type: object\n x-kubernetes-map-type: atomic\n type: object\n podAffinity:\n description: Describes pod affinity scheduling rules\n (e.g. co-locate this pod in the same node, zone,\n etc. as some other pod(s)).\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and adding\n \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the\n node(s) with the highest sum are the most preferred.\n items:\n description: The weights of all of the matched\n WeightedPodAffinityTerm fields are added\n per-node to find the most preferred node(s)\n properties:\n podAffinityTerm:\n description: Required. A pod affinity\n term, associated with the corresponding\n weight.\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n properties:\n matchExpressions:\n description: matchExpressions\n is a list of label selector\n requirements. The requirements\n are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the\n label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n properties:\n matchExpressions:\n description: matchExpressions\n is a list of label selector\n requirements. The requirements\n are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the\n label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n required:\n - topologyKey\n type: object\n weight:\n description: |-\n weight associated with matching the corresponding podAffinityTerm,\n in the range 1-100.\n format: int32\n type: integer\n required:\n - podAffinityTerm\n - weight\n type: object\n type: array\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to a pod label update), the\n system may or may not try to eventually evict the pod from its node.\n When there are multiple elements, the lists of nodes corresponding to each\n podAffinityTerm are intersected, i.e. all terms must be satisfied.\n items:\n description: |-\n Defines a set of pods (namely those matching the labelSelector\n relative to the given namespace(s)) that this pod should be\n co-located (affinity) or not co-located (anti-affinity) with,\n where co-located is defined as running on a node whose value of\n the label with key matches that of any node on which\n a pod of the set of pods is running\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n properties:\n matchExpressions:\n description: matchExpressions is a\n list of label selector requirements.\n The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label\n key that the selector applies\n to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n properties:\n matchExpressions:\n description: matchExpressions is a\n list of label selector requirements.\n The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label\n key that the selector applies\n to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n required:\n - topologyKey\n type: object\n type: array\n x-kubernetes-list-type: atomic\n type: object\n podAntiAffinity:\n description: Describes pod anti-affinity scheduling\n rules (e.g. avoid putting this pod in the same\n node, zone, etc. as some other pod(s)).\n properties:\n preferredDuringSchedulingIgnoredDuringExecution:\n description: |-\n The scheduler will prefer to schedule pods to nodes that satisfy\n the anti-affinity expressions specified by this field, but it may choose\n a node that violates one or more of the expressions. The node that is\n most preferred is the one with the greatest sum of weights, i.e.\n for each node that meets all of the scheduling requirements (resource\n request, requiredDuringScheduling anti-affinity expressions, etc.),\n compute a sum by iterating through the elements of this field and subtracting\n \"weight\" from the sum if the node has pods which matches the corresponding podAffinityTerm; the\n node(s) with the highest sum are the most preferred.\n items:\n description: The weights of all of the matched\n WeightedPodAffinityTerm fields are added\n per-node to find the most preferred node(s)\n properties:\n podAffinityTerm:\n description: Required. A pod affinity\n term, associated with the corresponding\n weight.\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n properties:\n matchExpressions:\n description: matchExpressions\n is a list of label selector\n requirements. The requirements\n are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the\n label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n properties:\n matchExpressions:\n description: matchExpressions\n is a list of label selector\n requirements. The requirements\n are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the\n label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n required:\n - topologyKey\n type: object\n weight:\n description: |-\n weight associated with matching the corresponding podAffinityTerm,\n in the range 1-100.\n format: int32\n type: integer\n required:\n - podAffinityTerm\n - weight\n type: object\n type: array\n x-kubernetes-list-type: atomic\n requiredDuringSchedulingIgnoredDuringExecution:\n description: |-\n If the anti-affinity requirements specified by this field are not met at\n scheduling time, the pod will not be scheduled onto the node.\n If the anti-affinity requirements specified by this field cease to be met\n at some point during pod execution (e.g. due to a pod label update), the\n system may or may not try to eventually evict the pod from its node.\n When there are multiple elements, the lists of nodes corresponding to each\n podAffinityTerm are intersected, i.e. all terms must be satisfied.\n items:\n description: |-\n Defines a set of pods (namely those matching the labelSelector\n relative to the given namespace(s)) that this pod should be\n co-located (affinity) or not co-located (anti-affinity) with,\n where co-located is defined as running on a node whose value of\n the label with key matches that of any node on which\n a pod of the set of pods is running\n properties:\n labelSelector:\n description: |-\n A label query over a set of resources, in this case pods.\n If it's null, this PodAffinityTerm matches with no Pods.\n properties:\n matchExpressions:\n description: matchExpressions is a\n list of label selector requirements.\n The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label\n key that the selector applies\n to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both matchLabelKeys and labelSelector.\n Also, matchLabelKeys cannot be set when labelSelector isn't set.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n mismatchLabelKeys:\n description: |-\n MismatchLabelKeys is a set of pod label keys to select which pods will\n be taken into consideration. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)`\n to select the group of existing pods which pods will be taken into consideration\n for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming\n pod labels will be ignored. The default value is empty.\n The same key is forbidden to exist in both mismatchLabelKeys and labelSelector.\n Also, mismatchLabelKeys cannot be set when labelSelector isn't set.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n namespaceSelector:\n description: |-\n A label query over the set of namespaces that the term applies to.\n The term is applied to the union of the namespaces selected by this field\n and the ones listed in the namespaces field.\n null selector and null or empty namespaces list means \"this pod's namespace\".\n An empty selector ({}) matches all namespaces.\n properties:\n matchExpressions:\n description: matchExpressions is a\n list of label selector requirements.\n The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label\n key that the selector applies\n to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n namespaces:\n description: |-\n namespaces specifies a static list of namespace names that the term applies to.\n The term is applied to the union of the namespaces listed in this field\n and the ones selected by namespaceSelector.\n null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n topologyKey:\n description: |-\n This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching\n the labelSelector in the specified namespaces, where co-located is defined as running on a node\n whose value of the label with key topologyKey matches that of any node on which any of the\n selected pods is running.\n Empty topologyKey is not allowed.\n type: string\n required:\n - topologyKey\n type: object\n type: array\n x-kubernetes-list-type: atomic\n type: object\n type: object\n automountServiceAccountToken:\n description: AutomountServiceAccountToken indicates\n whether a service account token should be automatically\n mounted.\n type: boolean\n containers:\n description: |-\n List of containers belonging to the pod.\n Containers cannot currently be added or removed.\n There must be at least one container in a Pod.\n Cannot be updated.\n items:\n description: A single application container that you\n want to run within a pod.\n properties:\n args:\n description: |-\n Arguments to the entrypoint.\n The container image's CMD is used if this is not provided.\n Variable references $(VAR_NAME) are expanded using the container's environment. If a variable\n cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will\n produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless\n of whether the variable exists or not. Cannot be updated.\n More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n command:\n description: |-\n Entrypoint array. Not executed within a shell.\n The container image's ENTRYPOINT is used if this is not provided.\n Variable references $(VAR_NAME) are expanded using the container's environment. If a variable\n cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will\n produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless\n of whether the variable exists or not. Cannot be updated.\n More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n env:\n description: |-\n List of environment variables to set in the container.\n Cannot be updated.\n items:\n description: EnvVar represents an environment\n variable present in a Container.\n properties:\n name:\n description: |-\n Name of the environment variable.\n May consist of any printable ASCII characters except '='.\n type: string\n value:\n description: |-\n Variable references $(VAR_NAME) are expanded\n using the previously defined environment variables in the container and\n any service environment variables. If a variable cannot be resolved,\n the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.\n \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\".\n Escaped references will never be expanded, regardless of whether the variable\n exists or not.\n Defaults to \"\".\n type: string\n valueFrom:\n description: Source for the environment\n variable's value. Cannot be used if value\n is not empty.\n properties:\n configMapKeyRef:\n description: Selects a key of a ConfigMap.\n properties:\n key:\n description: The key to select.\n type: string\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the\n ConfigMap or its key must be defined\n type: boolean\n required:\n - key\n type: object\n x-kubernetes-map-type: atomic\n fieldRef:\n description: |-\n Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`,\n spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.\n properties:\n apiVersion:\n description: Version of the schema\n the FieldPath is written in terms\n of, defaults to \"v1\".\n type: string\n fieldPath:\n description: Path of the field to\n select in the specified API version.\n type: string\n required:\n - fieldPath\n type: object\n x-kubernetes-map-type: atomic\n fileKeyRef:\n description: |-\n FileKeyRef selects a key of the env file.\n Requires the EnvFiles feature gate to be enabled.\n properties:\n key:\n description: |-\n The key within the env file. An invalid key will prevent the pod from starting.\n The keys defined within a source may consist of any printable ASCII characters except '='.\n During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.\n type: string\n optional:\n default: false\n description: |-\n Specify whether the file or its key must be defined. If the file or key\n does not exist, then the env var is not published.\n If optional is set to true and the specified key does not exist,\n the environment variable will not be set in the Pod's containers.\n\n If optional is set to false and the specified key does not exist,\n an error will be returned during Pod creation.\n type: boolean\n path:\n description: |-\n The path within the volume from which to select the file.\n Must be relative and may not contain the '..' path or start with '..'.\n type: string\n volumeName:\n description: The name of the volume\n mount containing the env file.\n type: string\n required:\n - key\n - path\n - volumeName\n type: object\n x-kubernetes-map-type: atomic\n resourceFieldRef:\n description: |-\n Selects a resource of the container: only resources limits and requests\n (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.\n properties:\n containerName:\n description: 'Container name: required\n for volumes, optional for env\n vars'\n type: string\n divisor:\n anyOf:\n - type: integer\n - type: string\n description: Specifies the output\n format of the exposed resources,\n defaults to \"1\"\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n resource:\n description: 'Required: resource\n to select'\n type: string\n required:\n - resource\n type: object\n x-kubernetes-map-type: atomic\n secretKeyRef:\n description: Selects a key of a secret\n in the pod's namespace\n properties:\n key:\n description: The key of the secret\n to select from. Must be a valid\n secret key.\n type: string\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the\n Secret or its key must be defined\n type: boolean\n required:\n - key\n type: object\n x-kubernetes-map-type: atomic\n type: object\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n envFrom:\n description: |-\n List of sources to populate environment variables in the container.\n The keys defined within a source may consist of any printable ASCII characters except '='.\n When a key exists in multiple\n sources, the value associated with the last source will take precedence.\n Values defined by an Env with a duplicate key will take precedence.\n Cannot be updated.\n items:\n description: EnvFromSource represents the source\n of a set of ConfigMaps or Secrets\n properties:\n configMapRef:\n description: The ConfigMap to select from\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the ConfigMap\n must be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n prefix:\n description: |-\n Optional text to prepend to the name of each environment variable.\n May consist of any printable ASCII characters except '='.\n type: string\n secretRef:\n description: The Secret to select from\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the Secret\n must be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n type: object\n type: array\n x-kubernetes-list-type: atomic\n image:\n description: |-\n Container image name.\n More info: https://kubernetes.io/docs/concepts/containers/images\n This field is optional to allow higher level config management to default or override\n container images in workload controllers like Deployments and StatefulSets.\n type: string\n imagePullPolicy:\n description: |-\n Image pull policy.\n One of Always, Never, IfNotPresent.\n Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\n type: string\n lifecycle:\n description: |-\n Actions that the management system should take in response to container lifecycle events.\n Cannot be updated.\n properties:\n postStart:\n description: |-\n PostStart is called immediately after a container is created. If the handler fails,\n the container is terminated and restarted according to its restart policy.\n Other management of the container blocks until the hook completes.\n More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\n properties:\n exec:\n description: Exec specifies a command\n to execute in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP\n GET request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set\n in the request. HTTP allows repeated\n headers.\n items:\n description: HTTPHeader describes\n a custom header to be used in\n HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field\n value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the\n HTTP server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n sleep:\n description: Sleep represents a duration\n that the container should sleep.\n properties:\n seconds:\n description: Seconds is the number\n of seconds to sleep.\n format: int64\n type: integer\n required:\n - seconds\n type: object\n tcpSocket:\n description: |-\n Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept\n for backward compatibility. There is no validation of this field and\n lifecycle hooks will fail at runtime when it is specified.\n properties:\n host:\n description: 'Optional: Host name\n to connect to, defaults to the pod\n IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n type: object\n preStop:\n description: |-\n PreStop is called immediately before a container is terminated due to an\n API request or management event such as liveness/startup probe failure,\n preemption, resource contention, etc. The handler is not called if the\n container crashes or exits. The Pod's termination grace period countdown begins before the\n PreStop hook is executed. Regardless of the outcome of the handler, the\n container will eventually terminate within the Pod's termination grace\n period (unless delayed by finalizers). Other management of the container blocks until the hook completes\n or until the termination grace period is reached.\n More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\n properties:\n exec:\n description: Exec specifies a command\n to execute in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP\n GET request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set\n in the request. HTTP allows repeated\n headers.\n items:\n description: HTTPHeader describes\n a custom header to be used in\n HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field\n value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the\n HTTP server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n sleep:\n description: Sleep represents a duration\n that the container should sleep.\n properties:\n seconds:\n description: Seconds is the number\n of seconds to sleep.\n format: int64\n type: integer\n required:\n - seconds\n type: object\n tcpSocket:\n description: |-\n Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept\n for backward compatibility. There is no validation of this field and\n lifecycle hooks will fail at runtime when it is specified.\n properties:\n host:\n description: 'Optional: Host name\n to connect to, defaults to the pod\n IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n type: object\n stopSignal:\n description: |-\n StopSignal defines which signal will be sent to a container when it is being stopped.\n If not specified, the default is defined by the container runtime in use.\n StopSignal can only be set for Pods with a non-empty .spec.os.name\n type: string\n type: object\n livenessProbe:\n description: |-\n Periodic probe of container liveness.\n Container will be restarted if the probe fails.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n name:\n description: |-\n Name of the container specified as a DNS_LABEL.\n Each container in a pod must have a unique name (DNS_LABEL).\n Cannot be updated.\n type: string\n ports:\n description: |-\n List of ports to expose from the container. Not specifying a port here\n DOES NOT prevent that port from being exposed. Any port which is\n listening on the default \"0.0.0.0\" address inside a container will be\n accessible from the network.\n Modifying this array with strategic merge patch may corrupt the data.\n For more information See https://github.com/kubernetes/kubernetes/issues/108255.\n Cannot be updated.\n items:\n description: ContainerPort represents a network\n port in a single container.\n properties:\n containerPort:\n description: |-\n Number of port to expose on the pod's IP address.\n This must be a valid port number, 0 < x < 65536.\n format: int32\n type: integer\n hostIP:\n description: What host IP to bind the external\n port to.\n type: string\n hostPort:\n description: |-\n Number of port to expose on the host.\n If specified, this must be a valid port number, 0 < x < 65536.\n If HostNetwork is specified, this must match ContainerPort.\n Most containers do not need this.\n format: int32\n type: integer\n name:\n description: |-\n If specified, this must be an IANA_SVC_NAME and unique within the pod. Each\n named port in a pod must have a unique name. Name for the port that can be\n referred to by services.\n type: string\n protocol:\n default: TCP\n description: |-\n Protocol for port. Must be UDP, TCP, or SCTP.\n Defaults to \"TCP\".\n type: string\n required:\n - containerPort\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - containerPort\n - protocol\n x-kubernetes-list-type: map\n readinessProbe:\n description: |-\n Periodic probe of container service readiness.\n Container will be removed from service endpoints if the probe fails.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n resizePolicy:\n description: |-\n Resources resize policy for the container.\n This field cannot be set on ephemeral containers.\n items:\n description: ContainerResizePolicy represents\n resource resize policy for the container.\n properties:\n resourceName:\n description: |-\n Name of the resource to which this resource resize policy applies.\n Supported values: cpu, memory.\n type: string\n restartPolicy:\n description: |-\n Restart policy to apply when specified resource is resized.\n If not specified, it defaults to NotRequired.\n type: string\n required:\n - resourceName\n - restartPolicy\n type: object\n type: array\n x-kubernetes-list-type: atomic\n resources:\n description: |-\n Compute Resources required by this container.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n properties:\n claims:\n description: |-\n Claims lists the names of resources, defined in spec.resourceClaims,\n that are used by this container.\n\n This field depends on the\n DynamicResourceAllocation feature gate.\n\n This field is immutable. It can only be set for containers.\n items:\n description: ResourceClaim references one\n entry in PodSpec.ResourceClaims.\n properties:\n name:\n description: |-\n Name must match the name of one entry in pod.spec.resourceClaims of\n the Pod where this field is used. It makes that resource available\n inside a container.\n type: string\n request:\n description: |-\n Request is the name chosen for a request in the referenced claim.\n If empty, everything from the claim is made available, otherwise\n only the result of this request.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n limits:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Limits describes the maximum amount of compute resources allowed.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n requests:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Requests describes the minimum amount of compute resources required.\n If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,\n otherwise to an implementation-defined value. Requests cannot exceed Limits.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n type: object\n restartPolicy:\n description: |-\n RestartPolicy defines the restart behavior of individual containers in a pod.\n This overrides the pod-level restart policy. When this field is not specified,\n the restart behavior is defined by the Pod's restart policy and the container type.\n Additionally, setting the RestartPolicy as \"Always\" for the init container will\n have the following effect:\n this init container will be continually restarted on\n exit until all regular containers have terminated. Once all regular\n containers have completed, all init containers with restartPolicy \"Always\"\n will be shut down. This lifecycle differs from normal init containers and\n is often referred to as a \"sidecar\" container. Although this init\n container still starts in the init container sequence, it does not wait\n for the container to complete before proceeding to the next init\n container. Instead, the next init container starts immediately after this\n init container is started, or after any startupProbe has successfully\n completed.\n type: string\n restartPolicyRules:\n description: |-\n Represents a list of rules to be checked to determine if the\n container should be restarted on exit. The rules are evaluated in\n order. Once a rule matches a container exit condition, the remaining\n rules are ignored. If no rule matches the container exit condition,\n the Container-level restart policy determines the whether the container\n is restarted or not. Constraints on the rules:\n - At most 20 rules are allowed.\n - Rules can have the same action.\n - Identical rules are not forbidden in validations.\n When rules are specified, container MUST set RestartPolicy explicitly\n even it if matches the Pod's RestartPolicy.\n items:\n description: ContainerRestartRule describes\n how a container exit is handled.\n properties:\n action:\n description: |-\n Specifies the action taken on a container exit if the requirements\n are satisfied. The only possible value is \"Restart\" to restart the\n container.\n type: string\n exitCodes:\n description: Represents the exit codes to\n check on container exits.\n properties:\n operator:\n description: |-\n Represents the relationship between the container exit code(s) and the\n specified values. Possible values are:\n - In: the requirement is satisfied if the container exit code is in the\n set of specified values.\n - NotIn: the requirement is satisfied if the container exit code is\n not in the set of specified values.\n type: string\n values:\n description: |-\n Specifies the set of values to check for container exit codes.\n At most 255 elements are allowed.\n items:\n format: int32\n type: integer\n type: array\n x-kubernetes-list-type: set\n required:\n - operator\n type: object\n required:\n - action\n type: object\n type: array\n x-kubernetes-list-type: atomic\n securityContext:\n description: |-\n SecurityContext defines the security options the container should be run with.\n If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext.\n More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/\n properties:\n allowPrivilegeEscalation:\n description: |-\n AllowPrivilegeEscalation controls whether a process can gain more\n privileges than its parent process. This bool directly controls if\n the no_new_privs flag will be set on the container process.\n AllowPrivilegeEscalation is true always when the container is:\n 1) run as Privileged\n 2) has CAP_SYS_ADMIN\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n appArmorProfile:\n description: |-\n appArmorProfile is the AppArmor options to use by this container. If set, this profile\n overrides the pod's appArmorProfile.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n localhostProfile:\n description: |-\n localhostProfile indicates a profile loaded on the node that should be used.\n The profile must be preconfigured on the node to work.\n Must match the loaded name of the profile.\n Must be set if and only if type is \"Localhost\".\n type: string\n type:\n description: |-\n type indicates which kind of AppArmor profile will be applied.\n Valid options are:\n Localhost - a profile pre-loaded on the node.\n RuntimeDefault - the container runtime's default profile.\n Unconfined - no AppArmor enforcement.\n type: string\n required:\n - type\n type: object\n capabilities:\n description: |-\n The capabilities to add/drop when running containers.\n Defaults to the default set of capabilities granted by the container runtime.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n add:\n description: Added capabilities\n items:\n description: Capability represent POSIX\n capabilities type\n type: string\n type: array\n x-kubernetes-list-type: atomic\n drop:\n description: Removed capabilities\n items:\n description: Capability represent POSIX\n capabilities type\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n privileged:\n description: |-\n Run container in privileged mode.\n Processes in privileged containers are essentially equivalent to root on the host.\n Defaults to false.\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n procMount:\n description: |-\n procMount denotes the type of proc mount to use for the containers.\n The default value is Default which uses the container runtime defaults for\n readonly paths and masked paths.\n This requires the ProcMountType feature flag to be enabled.\n Note that this field cannot be set when spec.os.name is windows.\n type: string\n readOnlyRootFilesystem:\n description: |-\n Whether this container has a read-only root filesystem.\n Default is false.\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n runAsGroup:\n description: |-\n The GID to run the entrypoint of the container process.\n Uses runtime default if unset.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n runAsNonRoot:\n description: |-\n Indicates that the container must run as a non-root user.\n If true, the Kubelet will validate the image at runtime to ensure that it\n does not run as UID 0 (root) and fail to start the container if it does.\n If unset or false, no such validation will be performed.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n type: boolean\n runAsUser:\n description: |-\n The UID to run the entrypoint of the container process.\n Defaults to user specified in image metadata if unspecified.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n seLinuxOptions:\n description: |-\n The SELinux context to be applied to the container.\n If unspecified, the container runtime will allocate a random SELinux context for each\n container. May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n level:\n description: Level is SELinux level label\n that applies to the container.\n type: string\n role:\n description: Role is a SELinux role label\n that applies to the container.\n type: string\n type:\n description: Type is a SELinux type label\n that applies to the container.\n type: string\n user:\n description: User is a SELinux user label\n that applies to the container.\n type: string\n type: object\n seccompProfile:\n description: |-\n The seccomp options to use by this container. If seccomp options are\n provided at both the pod & container level, the container options\n override the pod options.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n localhostProfile:\n description: |-\n localhostProfile indicates a profile defined in a file on the node should be used.\n The profile must be preconfigured on the node to work.\n Must be a descending path, relative to the kubelet's configured seccomp profile location.\n Must be set if type is \"Localhost\". Must NOT be set for any other type.\n type: string\n type:\n description: |-\n type indicates which kind of seccomp profile will be applied.\n Valid options are:\n\n Localhost - a profile defined in a file on the node should be used.\n RuntimeDefault - the container runtime default profile should be used.\n Unconfined - no profile should be applied.\n type: string\n required:\n - type\n type: object\n windowsOptions:\n description: |-\n The Windows specific settings applied to all containers.\n If unspecified, the options from the PodSecurityContext will be used.\n If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is linux.\n properties:\n gmsaCredentialSpec:\n description: |-\n GMSACredentialSpec is where the GMSA admission webhook\n (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the\n GMSA credential spec named by the GMSACredentialSpecName field.\n type: string\n gmsaCredentialSpecName:\n description: GMSACredentialSpecName is\n the name of the GMSA credential spec\n to use.\n type: string\n hostProcess:\n description: |-\n HostProcess determines if a container should be run as a 'Host Process' container.\n All of a Pod's containers must have the same effective HostProcess value\n (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).\n In addition, if HostProcess is true then HostNetwork must also be set to true.\n type: boolean\n runAsUserName:\n description: |-\n The UserName in Windows to run the entrypoint of the container process.\n Defaults to the user specified in image metadata if unspecified.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n type: string\n type: object\n type: object\n startupProbe:\n description: |-\n StartupProbe indicates that the Pod has successfully initialized.\n If specified, no other probes are executed until this completes successfully.\n If this probe fails, the Pod will be restarted, just as if the livenessProbe failed.\n This can be used to provide different probe parameters at the beginning of a Pod's lifecycle,\n when it might take a long time to load data or warm a cache, than during steady-state operation.\n This cannot be updated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n stdin:\n description: |-\n Whether this container should allocate a buffer for stdin in the container runtime. If this\n is not set, reads from stdin in the container will always result in EOF.\n Default is false.\n type: boolean\n stdinOnce:\n description: |-\n Whether the container runtime should close the stdin channel after it has been opened by\n a single attach. When stdin is true the stdin stream will remain open across multiple attach\n sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the\n first client attaches to stdin, and then remains open and accepts data until the client disconnects,\n at which time stdin is closed and remains closed until the container is restarted. If this\n flag is false, a container processes that reads from stdin will never receive an EOF.\n Default is false\n type: boolean\n terminationMessagePath:\n description: |-\n Optional: Path at which the file to which the container's termination message\n will be written is mounted into the container's filesystem.\n Message written is intended to be brief final status, such as an assertion failure message.\n Will be truncated by the node if greater than 4096 bytes. The total message length across\n all containers will be limited to 12kb.\n Defaults to /dev/termination-log.\n Cannot be updated.\n type: string\n terminationMessagePolicy:\n description: |-\n Indicate how the termination message should be populated. File will use the contents of\n terminationMessagePath to populate the container status message on both success and failure.\n FallbackToLogsOnError will use the last chunk of container log output if the termination\n message file is empty and the container exited with an error.\n The log output is limited to 2048 bytes or 80 lines, whichever is smaller.\n Defaults to File.\n Cannot be updated.\n type: string\n tty:\n description: |-\n Whether this container should allocate a TTY for itself, also requires 'stdin' to be true.\n Default is false.\n type: boolean\n volumeDevices:\n description: volumeDevices is the list of block\n devices to be used by the container.\n items:\n description: volumeDevice describes a mapping\n of a raw block device within a container.\n properties:\n devicePath:\n description: devicePath is the path inside\n of the container that the device will\n be mapped to.\n type: string\n name:\n description: name must match the name of\n a persistentVolumeClaim in the pod\n type: string\n required:\n - devicePath\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - devicePath\n x-kubernetes-list-type: map\n volumeMounts:\n description: |-\n Pod volumes to mount into the container's filesystem.\n Cannot be updated.\n items:\n description: VolumeMount describes a mounting\n of a Volume within a container.\n properties:\n mountPath:\n description: |-\n Path within the container at which the volume should be mounted. Must\n not contain ':'.\n type: string\n mountPropagation:\n description: |-\n mountPropagation determines how mounts are propagated from the host\n to container and the other way around.\n When not set, MountPropagationNone is used.\n This field is beta in 1.10.\n When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified\n (which defaults to None).\n type: string\n name:\n description: This must match the Name of\n a Volume.\n type: string\n readOnly:\n description: |-\n Mounted read-only if true, read-write otherwise (false or unspecified).\n Defaults to false.\n type: boolean\n recursiveReadOnly:\n description: |-\n RecursiveReadOnly specifies whether read-only mounts should be handled\n recursively.\n\n If ReadOnly is false, this field has no meaning and must be unspecified.\n\n If ReadOnly is true, and this field is set to Disabled, the mount is not made\n recursively read-only. If this field is set to IfPossible, the mount is made\n recursively read-only, if it is supported by the container runtime. If this\n field is set to Enabled, the mount is made recursively read-only if it is\n supported by the container runtime, otherwise the pod will not be started and\n an error will be generated to indicate the reason.\n\n If this field is set to IfPossible or Enabled, MountPropagation must be set to\n None (or be unspecified, which defaults to None).\n\n If this field is not specified, it is treated as an equivalent of Disabled.\n type: string\n subPath:\n description: |-\n Path within the volume from which the container's volume should be mounted.\n Defaults to \"\" (volume's root).\n type: string\n subPathExpr:\n description: |-\n Expanded path within the volume from which the container's volume should be mounted.\n Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.\n Defaults to \"\" (volume's root).\n SubPathExpr and SubPath are mutually exclusive.\n type: string\n required:\n - mountPath\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - mountPath\n x-kubernetes-list-type: map\n workingDir:\n description: |-\n Container's working directory.\n If not specified, the container runtime's default will be used, which\n might be configured in the container image.\n Cannot be updated.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n dnsConfig:\n description: |-\n Specifies the DNS parameters of a pod.\n Parameters specified here will be merged to the generated DNS\n configuration based on DNSPolicy.\n properties:\n nameservers:\n description: |-\n A list of DNS name server IP addresses.\n This will be appended to the base nameservers generated from DNSPolicy.\n Duplicated nameservers will be removed.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n options:\n description: |-\n A list of DNS resolver options.\n This will be merged with the base options generated from DNSPolicy.\n Duplicated entries will be removed. Resolution options given in Options\n will override those that appear in the base DNSPolicy.\n items:\n description: PodDNSConfigOption defines DNS resolver\n options of a pod.\n properties:\n name:\n description: |-\n Name is this DNS resolver option's name.\n Required.\n type: string\n value:\n description: Value is this DNS resolver option's\n value.\n type: string\n type: object\n type: array\n x-kubernetes-list-type: atomic\n searches:\n description: |-\n A list of DNS search domains for host-name lookup.\n This will be appended to the base search paths generated from DNSPolicy.\n Duplicated search paths will be removed.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n dnsPolicy:\n description: |-\n Set DNS policy for the pod.\n Defaults to \"ClusterFirst\".\n Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'.\n DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy.\n To have DNS options set along with hostNetwork, you have to specify DNS policy\n explicitly to 'ClusterFirstWithHostNet'.\n type: string\n enableServiceLinks:\n description: |-\n EnableServiceLinks indicates whether information about services should be injected into pod's\n environment variables, matching the syntax of Docker links.\n Optional: Defaults to true.\n type: boolean\n ephemeralContainers:\n description: |-\n List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing\n pod to perform user-initiated actions such as debugging. This list cannot be specified when\n creating a pod, and it cannot be modified by updating the pod spec. In order to add an\n ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource.\n items:\n description: |-\n An EphemeralContainer is a temporary container that you may add to an existing Pod for\n user-initiated activities such as debugging. Ephemeral containers have no resource or\n scheduling guarantees, and they will not be restarted when they exit or when a Pod is\n removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the\n Pod to exceed its resource allocation.\n\n To add an ephemeral container, use the ephemeralcontainers subresource of an existing\n Pod. Ephemeral containers may not be removed or restarted.\n properties:\n args:\n description: |-\n Arguments to the entrypoint.\n The image's CMD is used if this is not provided.\n Variable references $(VAR_NAME) are expanded using the container's environment. If a variable\n cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will\n produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless\n of whether the variable exists or not. Cannot be updated.\n More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n command:\n description: |-\n Entrypoint array. Not executed within a shell.\n The image's ENTRYPOINT is used if this is not provided.\n Variable references $(VAR_NAME) are expanded using the container's environment. If a variable\n cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will\n produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless\n of whether the variable exists or not. Cannot be updated.\n More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n env:\n description: |-\n List of environment variables to set in the container.\n Cannot be updated.\n items:\n description: EnvVar represents an environment\n variable present in a Container.\n properties:\n name:\n description: |-\n Name of the environment variable.\n May consist of any printable ASCII characters except '='.\n type: string\n value:\n description: |-\n Variable references $(VAR_NAME) are expanded\n using the previously defined environment variables in the container and\n any service environment variables. If a variable cannot be resolved,\n the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.\n \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\".\n Escaped references will never be expanded, regardless of whether the variable\n exists or not.\n Defaults to \"\".\n type: string\n valueFrom:\n description: Source for the environment\n variable's value. Cannot be used if value\n is not empty.\n properties:\n configMapKeyRef:\n description: Selects a key of a ConfigMap.\n properties:\n key:\n description: The key to select.\n type: string\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the\n ConfigMap or its key must be defined\n type: boolean\n required:\n - key\n type: object\n x-kubernetes-map-type: atomic\n fieldRef:\n description: |-\n Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`,\n spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.\n properties:\n apiVersion:\n description: Version of the schema\n the FieldPath is written in terms\n of, defaults to \"v1\".\n type: string\n fieldPath:\n description: Path of the field to\n select in the specified API version.\n type: string\n required:\n - fieldPath\n type: object\n x-kubernetes-map-type: atomic\n fileKeyRef:\n description: |-\n FileKeyRef selects a key of the env file.\n Requires the EnvFiles feature gate to be enabled.\n properties:\n key:\n description: |-\n The key within the env file. An invalid key will prevent the pod from starting.\n The keys defined within a source may consist of any printable ASCII characters except '='.\n During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.\n type: string\n optional:\n default: false\n description: |-\n Specify whether the file or its key must be defined. If the file or key\n does not exist, then the env var is not published.\n If optional is set to true and the specified key does not exist,\n the environment variable will not be set in the Pod's containers.\n\n If optional is set to false and the specified key does not exist,\n an error will be returned during Pod creation.\n type: boolean\n path:\n description: |-\n The path within the volume from which to select the file.\n Must be relative and may not contain the '..' path or start with '..'.\n type: string\n volumeName:\n description: The name of the volume\n mount containing the env file.\n type: string\n required:\n - key\n - path\n - volumeName\n type: object\n x-kubernetes-map-type: atomic\n resourceFieldRef:\n description: |-\n Selects a resource of the container: only resources limits and requests\n (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.\n properties:\n containerName:\n description: 'Container name: required\n for volumes, optional for env\n vars'\n type: string\n divisor:\n anyOf:\n - type: integer\n - type: string\n description: Specifies the output\n format of the exposed resources,\n defaults to \"1\"\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n resource:\n description: 'Required: resource\n to select'\n type: string\n required:\n - resource\n type: object\n x-kubernetes-map-type: atomic\n secretKeyRef:\n description: Selects a key of a secret\n in the pod's namespace\n properties:\n key:\n description: The key of the secret\n to select from. Must be a valid\n secret key.\n type: string\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the\n Secret or its key must be defined\n type: boolean\n required:\n - key\n type: object\n x-kubernetes-map-type: atomic\n type: object\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n envFrom:\n description: |-\n List of sources to populate environment variables in the container.\n The keys defined within a source may consist of any printable ASCII characters except '='.\n When a key exists in multiple\n sources, the value associated with the last source will take precedence.\n Values defined by an Env with a duplicate key will take precedence.\n Cannot be updated.\n items:\n description: EnvFromSource represents the source\n of a set of ConfigMaps or Secrets\n properties:\n configMapRef:\n description: The ConfigMap to select from\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the ConfigMap\n must be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n prefix:\n description: |-\n Optional text to prepend to the name of each environment variable.\n May consist of any printable ASCII characters except '='.\n type: string\n secretRef:\n description: The Secret to select from\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the Secret\n must be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n type: object\n type: array\n x-kubernetes-list-type: atomic\n image:\n description: |-\n Container image name.\n More info: https://kubernetes.io/docs/concepts/containers/images\n type: string\n imagePullPolicy:\n description: |-\n Image pull policy.\n One of Always, Never, IfNotPresent.\n Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\n type: string\n lifecycle:\n description: Lifecycle is not allowed for ephemeral\n containers.\n properties:\n postStart:\n description: |-\n PostStart is called immediately after a container is created. If the handler fails,\n the container is terminated and restarted according to its restart policy.\n Other management of the container blocks until the hook completes.\n More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\n properties:\n exec:\n description: Exec specifies a command\n to execute in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP\n GET request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set\n in the request. HTTP allows repeated\n headers.\n items:\n description: HTTPHeader describes\n a custom header to be used in\n HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field\n value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the\n HTTP server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n sleep:\n description: Sleep represents a duration\n that the container should sleep.\n properties:\n seconds:\n description: Seconds is the number\n of seconds to sleep.\n format: int64\n type: integer\n required:\n - seconds\n type: object\n tcpSocket:\n description: |-\n Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept\n for backward compatibility. There is no validation of this field and\n lifecycle hooks will fail at runtime when it is specified.\n properties:\n host:\n description: 'Optional: Host name\n to connect to, defaults to the pod\n IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n type: object\n preStop:\n description: |-\n PreStop is called immediately before a container is terminated due to an\n API request or management event such as liveness/startup probe failure,\n preemption, resource contention, etc. The handler is not called if the\n container crashes or exits. The Pod's termination grace period countdown begins before the\n PreStop hook is executed. Regardless of the outcome of the handler, the\n container will eventually terminate within the Pod's termination grace\n period (unless delayed by finalizers). Other management of the container blocks until the hook completes\n or until the termination grace period is reached.\n More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\n properties:\n exec:\n description: Exec specifies a command\n to execute in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP\n GET request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set\n in the request. HTTP allows repeated\n headers.\n items:\n description: HTTPHeader describes\n a custom header to be used in\n HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field\n value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the\n HTTP server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n sleep:\n description: Sleep represents a duration\n that the container should sleep.\n properties:\n seconds:\n description: Seconds is the number\n of seconds to sleep.\n format: int64\n type: integer\n required:\n - seconds\n type: object\n tcpSocket:\n description: |-\n Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept\n for backward compatibility. There is no validation of this field and\n lifecycle hooks will fail at runtime when it is specified.\n properties:\n host:\n description: 'Optional: Host name\n to connect to, defaults to the pod\n IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n type: object\n stopSignal:\n description: |-\n StopSignal defines which signal will be sent to a container when it is being stopped.\n If not specified, the default is defined by the container runtime in use.\n StopSignal can only be set for Pods with a non-empty .spec.os.name\n type: string\n type: object\n livenessProbe:\n description: Probes are not allowed for ephemeral\n containers.\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n name:\n description: |-\n Name of the ephemeral container specified as a DNS_LABEL.\n This name must be unique among all containers, init containers and ephemeral containers.\n type: string\n ports:\n description: Ports are not allowed for ephemeral\n containers.\n items:\n description: ContainerPort represents a network\n port in a single container.\n properties:\n containerPort:\n description: |-\n Number of port to expose on the pod's IP address.\n This must be a valid port number, 0 < x < 65536.\n format: int32\n type: integer\n hostIP:\n description: What host IP to bind the external\n port to.\n type: string\n hostPort:\n description: |-\n Number of port to expose on the host.\n If specified, this must be a valid port number, 0 < x < 65536.\n If HostNetwork is specified, this must match ContainerPort.\n Most containers do not need this.\n format: int32\n type: integer\n name:\n description: |-\n If specified, this must be an IANA_SVC_NAME and unique within the pod. Each\n named port in a pod must have a unique name. Name for the port that can be\n referred to by services.\n type: string\n protocol:\n default: TCP\n description: |-\n Protocol for port. Must be UDP, TCP, or SCTP.\n Defaults to \"TCP\".\n type: string\n required:\n - containerPort\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - containerPort\n - protocol\n x-kubernetes-list-type: map\n readinessProbe:\n description: Probes are not allowed for ephemeral\n containers.\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n resizePolicy:\n description: Resources resize policy for the container.\n items:\n description: ContainerResizePolicy represents\n resource resize policy for the container.\n properties:\n resourceName:\n description: |-\n Name of the resource to which this resource resize policy applies.\n Supported values: cpu, memory.\n type: string\n restartPolicy:\n description: |-\n Restart policy to apply when specified resource is resized.\n If not specified, it defaults to NotRequired.\n type: string\n required:\n - resourceName\n - restartPolicy\n type: object\n type: array\n x-kubernetes-list-type: atomic\n resources:\n description: |-\n Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources\n already allocated to the pod.\n properties:\n claims:\n description: |-\n Claims lists the names of resources, defined in spec.resourceClaims,\n that are used by this container.\n\n This field depends on the\n DynamicResourceAllocation feature gate.\n\n This field is immutable. It can only be set for containers.\n items:\n description: ResourceClaim references one\n entry in PodSpec.ResourceClaims.\n properties:\n name:\n description: |-\n Name must match the name of one entry in pod.spec.resourceClaims of\n the Pod where this field is used. It makes that resource available\n inside a container.\n type: string\n request:\n description: |-\n Request is the name chosen for a request in the referenced claim.\n If empty, everything from the claim is made available, otherwise\n only the result of this request.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n limits:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Limits describes the maximum amount of compute resources allowed.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n requests:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Requests describes the minimum amount of compute resources required.\n If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,\n otherwise to an implementation-defined value. Requests cannot exceed Limits.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n type: object\n restartPolicy:\n description: |-\n Restart policy for the container to manage the restart behavior of each\n container within a pod.\n You cannot set this field on ephemeral containers.\n type: string\n restartPolicyRules:\n description: |-\n Represents a list of rules to be checked to determine if the\n container should be restarted on exit. You cannot set this field on\n ephemeral containers.\n items:\n description: ContainerRestartRule describes\n how a container exit is handled.\n properties:\n action:\n description: |-\n Specifies the action taken on a container exit if the requirements\n are satisfied. The only possible value is \"Restart\" to restart the\n container.\n type: string\n exitCodes:\n description: Represents the exit codes to\n check on container exits.\n properties:\n operator:\n description: |-\n Represents the relationship between the container exit code(s) and the\n specified values. Possible values are:\n - In: the requirement is satisfied if the container exit code is in the\n set of specified values.\n - NotIn: the requirement is satisfied if the container exit code is\n not in the set of specified values.\n type: string\n values:\n description: |-\n Specifies the set of values to check for container exit codes.\n At most 255 elements are allowed.\n items:\n format: int32\n type: integer\n type: array\n x-kubernetes-list-type: set\n required:\n - operator\n type: object\n required:\n - action\n type: object\n type: array\n x-kubernetes-list-type: atomic\n securityContext:\n description: |-\n Optional: SecurityContext defines the security options the ephemeral container should be run with.\n If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext.\n properties:\n allowPrivilegeEscalation:\n description: |-\n AllowPrivilegeEscalation controls whether a process can gain more\n privileges than its parent process. This bool directly controls if\n the no_new_privs flag will be set on the container process.\n AllowPrivilegeEscalation is true always when the container is:\n 1) run as Privileged\n 2) has CAP_SYS_ADMIN\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n appArmorProfile:\n description: |-\n appArmorProfile is the AppArmor options to use by this container. If set, this profile\n overrides the pod's appArmorProfile.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n localhostProfile:\n description: |-\n localhostProfile indicates a profile loaded on the node that should be used.\n The profile must be preconfigured on the node to work.\n Must match the loaded name of the profile.\n Must be set if and only if type is \"Localhost\".\n type: string\n type:\n description: |-\n type indicates which kind of AppArmor profile will be applied.\n Valid options are:\n Localhost - a profile pre-loaded on the node.\n RuntimeDefault - the container runtime's default profile.\n Unconfined - no AppArmor enforcement.\n type: string\n required:\n - type\n type: object\n capabilities:\n description: |-\n The capabilities to add/drop when running containers.\n Defaults to the default set of capabilities granted by the container runtime.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n add:\n description: Added capabilities\n items:\n description: Capability represent POSIX\n capabilities type\n type: string\n type: array\n x-kubernetes-list-type: atomic\n drop:\n description: Removed capabilities\n items:\n description: Capability represent POSIX\n capabilities type\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n privileged:\n description: |-\n Run container in privileged mode.\n Processes in privileged containers are essentially equivalent to root on the host.\n Defaults to false.\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n procMount:\n description: |-\n procMount denotes the type of proc mount to use for the containers.\n The default value is Default which uses the container runtime defaults for\n readonly paths and masked paths.\n This requires the ProcMountType feature flag to be enabled.\n Note that this field cannot be set when spec.os.name is windows.\n type: string\n readOnlyRootFilesystem:\n description: |-\n Whether this container has a read-only root filesystem.\n Default is false.\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n runAsGroup:\n description: |-\n The GID to run the entrypoint of the container process.\n Uses runtime default if unset.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n runAsNonRoot:\n description: |-\n Indicates that the container must run as a non-root user.\n If true, the Kubelet will validate the image at runtime to ensure that it\n does not run as UID 0 (root) and fail to start the container if it does.\n If unset or false, no such validation will be performed.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n type: boolean\n runAsUser:\n description: |-\n The UID to run the entrypoint of the container process.\n Defaults to user specified in image metadata if unspecified.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n seLinuxOptions:\n description: |-\n The SELinux context to be applied to the container.\n If unspecified, the container runtime will allocate a random SELinux context for each\n container. May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n level:\n description: Level is SELinux level label\n that applies to the container.\n type: string\n role:\n description: Role is a SELinux role label\n that applies to the container.\n type: string\n type:\n description: Type is a SELinux type label\n that applies to the container.\n type: string\n user:\n description: User is a SELinux user label\n that applies to the container.\n type: string\n type: object\n seccompProfile:\n description: |-\n The seccomp options to use by this container. If seccomp options are\n provided at both the pod & container level, the container options\n override the pod options.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n localhostProfile:\n description: |-\n localhostProfile indicates a profile defined in a file on the node should be used.\n The profile must be preconfigured on the node to work.\n Must be a descending path, relative to the kubelet's configured seccomp profile location.\n Must be set if type is \"Localhost\". Must NOT be set for any other type.\n type: string\n type:\n description: |-\n type indicates which kind of seccomp profile will be applied.\n Valid options are:\n\n Localhost - a profile defined in a file on the node should be used.\n RuntimeDefault - the container runtime default profile should be used.\n Unconfined - no profile should be applied.\n type: string\n required:\n - type\n type: object\n windowsOptions:\n description: |-\n The Windows specific settings applied to all containers.\n If unspecified, the options from the PodSecurityContext will be used.\n If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is linux.\n properties:\n gmsaCredentialSpec:\n description: |-\n GMSACredentialSpec is where the GMSA admission webhook\n (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the\n GMSA credential spec named by the GMSACredentialSpecName field.\n type: string\n gmsaCredentialSpecName:\n description: GMSACredentialSpecName is\n the name of the GMSA credential spec\n to use.\n type: string\n hostProcess:\n description: |-\n HostProcess determines if a container should be run as a 'Host Process' container.\n All of a Pod's containers must have the same effective HostProcess value\n (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).\n In addition, if HostProcess is true then HostNetwork must also be set to true.\n type: boolean\n runAsUserName:\n description: |-\n The UserName in Windows to run the entrypoint of the container process.\n Defaults to the user specified in image metadata if unspecified.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n type: string\n type: object\n type: object\n startupProbe:\n description: Probes are not allowed for ephemeral\n containers.\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n stdin:\n description: |-\n Whether this container should allocate a buffer for stdin in the container runtime. If this\n is not set, reads from stdin in the container will always result in EOF.\n Default is false.\n type: boolean\n stdinOnce:\n description: |-\n Whether the container runtime should close the stdin channel after it has been opened by\n a single attach. When stdin is true the stdin stream will remain open across multiple attach\n sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the\n first client attaches to stdin, and then remains open and accepts data until the client disconnects,\n at which time stdin is closed and remains closed until the container is restarted. If this\n flag is false, a container processes that reads from stdin will never receive an EOF.\n Default is false\n type: boolean\n targetContainerName:\n description: |-\n If set, the name of the container from PodSpec that this ephemeral container targets.\n The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container.\n If not set then the ephemeral container uses the namespaces configured in the Pod spec.\n\n The container runtime must implement support for this feature. If the runtime does not\n support namespace targeting then the result of setting this field is undefined.\n type: string\n terminationMessagePath:\n description: |-\n Optional: Path at which the file to which the container's termination message\n will be written is mounted into the container's filesystem.\n Message written is intended to be brief final status, such as an assertion failure message.\n Will be truncated by the node if greater than 4096 bytes. The total message length across\n all containers will be limited to 12kb.\n Defaults to /dev/termination-log.\n Cannot be updated.\n type: string\n terminationMessagePolicy:\n description: |-\n Indicate how the termination message should be populated. File will use the contents of\n terminationMessagePath to populate the container status message on both success and failure.\n FallbackToLogsOnError will use the last chunk of container log output if the termination\n message file is empty and the container exited with an error.\n The log output is limited to 2048 bytes or 80 lines, whichever is smaller.\n Defaults to File.\n Cannot be updated.\n type: string\n tty:\n description: |-\n Whether this container should allocate a TTY for itself, also requires 'stdin' to be true.\n Default is false.\n type: boolean\n volumeDevices:\n description: volumeDevices is the list of block\n devices to be used by the container.\n items:\n description: volumeDevice describes a mapping\n of a raw block device within a container.\n properties:\n devicePath:\n description: devicePath is the path inside\n of the container that the device will\n be mapped to.\n type: string\n name:\n description: name must match the name of\n a persistentVolumeClaim in the pod\n type: string\n required:\n - devicePath\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - devicePath\n x-kubernetes-list-type: map\n volumeMounts:\n description: |-\n Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers.\n Cannot be updated.\n items:\n description: VolumeMount describes a mounting\n of a Volume within a container.\n properties:\n mountPath:\n description: |-\n Path within the container at which the volume should be mounted. Must\n not contain ':'.\n type: string\n mountPropagation:\n description: |-\n mountPropagation determines how mounts are propagated from the host\n to container and the other way around.\n When not set, MountPropagationNone is used.\n This field is beta in 1.10.\n When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified\n (which defaults to None).\n type: string\n name:\n description: This must match the Name of\n a Volume.\n type: string\n readOnly:\n description: |-\n Mounted read-only if true, read-write otherwise (false or unspecified).\n Defaults to false.\n type: boolean\n recursiveReadOnly:\n description: |-\n RecursiveReadOnly specifies whether read-only mounts should be handled\n recursively.\n\n If ReadOnly is false, this field has no meaning and must be unspecified.\n\n If ReadOnly is true, and this field is set to Disabled, the mount is not made\n recursively read-only. If this field is set to IfPossible, the mount is made\n recursively read-only, if it is supported by the container runtime. If this\n field is set to Enabled, the mount is made recursively read-only if it is\n supported by the container runtime, otherwise the pod will not be started and\n an error will be generated to indicate the reason.\n\n If this field is set to IfPossible or Enabled, MountPropagation must be set to\n None (or be unspecified, which defaults to None).\n\n If this field is not specified, it is treated as an equivalent of Disabled.\n type: string\n subPath:\n description: |-\n Path within the volume from which the container's volume should be mounted.\n Defaults to \"\" (volume's root).\n type: string\n subPathExpr:\n description: |-\n Expanded path within the volume from which the container's volume should be mounted.\n Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.\n Defaults to \"\" (volume's root).\n SubPathExpr and SubPath are mutually exclusive.\n type: string\n required:\n - mountPath\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - mountPath\n x-kubernetes-list-type: map\n workingDir:\n description: |-\n Container's working directory.\n If not specified, the container runtime's default will be used, which\n might be configured in the container image.\n Cannot be updated.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n hostAliases:\n description: |-\n HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts\n file if specified.\n items:\n description: |-\n HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the\n pod's hosts file.\n properties:\n hostnames:\n description: Hostnames for the above IP address.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n ip:\n description: IP address of the host file entry.\n type: string\n required:\n - ip\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - ip\n x-kubernetes-list-type: map\n hostIPC:\n description: |-\n Use the host's ipc namespace.\n Optional: Default to false.\n type: boolean\n hostNetwork:\n description: |-\n Host networking requested for this pod. Use the host's network namespace.\n When using HostNetwork you should specify ports so the scheduler is aware.\n When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`,\n and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`.\n Default to false.\n type: boolean\n hostPID:\n description: |-\n Use the host's pid namespace.\n Optional: Default to false.\n type: boolean\n hostUsers:\n description: |-\n Use the host's user namespace.\n Optional: Default to true.\n If set to true or not present, the pod will be run in the host user namespace, useful\n for when the pod needs a feature only available to the host user namespace, such as\n loading a kernel module with CAP_SYS_MODULE.\n When set to false, a new userns is created for the pod. Setting false is useful for\n mitigating container breakout vulnerabilities even allowing users to run their\n containers as root without actually having root privileges on the host.\n This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature.\n type: boolean\n hostname:\n description: |-\n Specifies the hostname of the Pod\n If not specified, the pod's hostname will be set to a system-defined value.\n type: string\n hostnameOverride:\n description: |-\n HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod.\n This field only specifies the pod's hostname and does not affect its DNS records.\n When this field is set to a non-empty string:\n - It takes precedence over the values set in `hostname` and `subdomain`.\n - The Pod's hostname will be set to this value.\n - `setHostnameAsFQDN` must be nil or set to false.\n - `hostNetwork` must be set to false.\n\n This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters.\n Requires the HostnameOverride feature gate to be enabled.\n type: string\n imagePullSecrets:\n description: |-\n ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec.\n If specified, these secrets will be passed to individual puller implementations for them to use.\n More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod\n items:\n description: |-\n LocalObjectReference contains enough information to let you locate the\n referenced object inside the same namespace.\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n initContainers:\n description: |-\n List of initialization containers belonging to the pod.\n Init containers are executed in order prior to containers being started. If any\n init container fails, the pod is considered to have failed and is handled according\n to its restartPolicy. The name for an init container or normal container must be\n unique among all containers.\n Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes.\n The resourceRequirements of an init container are taken into account during scheduling\n by finding the highest request/limit for each resource type, and then using the max of\n that value or the sum of the normal containers. Limits are applied to init containers\n in a similar fashion.\n Init containers cannot currently be added or removed.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/\n items:\n description: A single application container that you\n want to run within a pod.\n properties:\n args:\n description: |-\n Arguments to the entrypoint.\n The container image's CMD is used if this is not provided.\n Variable references $(VAR_NAME) are expanded using the container's environment. If a variable\n cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will\n produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless\n of whether the variable exists or not. Cannot be updated.\n More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n command:\n description: |-\n Entrypoint array. Not executed within a shell.\n The container image's ENTRYPOINT is used if this is not provided.\n Variable references $(VAR_NAME) are expanded using the container's environment. If a variable\n cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will\n produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless\n of whether the variable exists or not. Cannot be updated.\n More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n env:\n description: |-\n List of environment variables to set in the container.\n Cannot be updated.\n items:\n description: EnvVar represents an environment\n variable present in a Container.\n properties:\n name:\n description: |-\n Name of the environment variable.\n May consist of any printable ASCII characters except '='.\n type: string\n value:\n description: |-\n Variable references $(VAR_NAME) are expanded\n using the previously defined environment variables in the container and\n any service environment variables. If a variable cannot be resolved,\n the reference in the input string will be unchanged. Double $$ are reduced\n to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.\n \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\".\n Escaped references will never be expanded, regardless of whether the variable\n exists or not.\n Defaults to \"\".\n type: string\n valueFrom:\n description: Source for the environment\n variable's value. Cannot be used if value\n is not empty.\n properties:\n configMapKeyRef:\n description: Selects a key of a ConfigMap.\n properties:\n key:\n description: The key to select.\n type: string\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the\n ConfigMap or its key must be defined\n type: boolean\n required:\n - key\n type: object\n x-kubernetes-map-type: atomic\n fieldRef:\n description: |-\n Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`,\n spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.\n properties:\n apiVersion:\n description: Version of the schema\n the FieldPath is written in terms\n of, defaults to \"v1\".\n type: string\n fieldPath:\n description: Path of the field to\n select in the specified API version.\n type: string\n required:\n - fieldPath\n type: object\n x-kubernetes-map-type: atomic\n fileKeyRef:\n description: |-\n FileKeyRef selects a key of the env file.\n Requires the EnvFiles feature gate to be enabled.\n properties:\n key:\n description: |-\n The key within the env file. An invalid key will prevent the pod from starting.\n The keys defined within a source may consist of any printable ASCII characters except '='.\n During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.\n type: string\n optional:\n default: false\n description: |-\n Specify whether the file or its key must be defined. If the file or key\n does not exist, then the env var is not published.\n If optional is set to true and the specified key does not exist,\n the environment variable will not be set in the Pod's containers.\n\n If optional is set to false and the specified key does not exist,\n an error will be returned during Pod creation.\n type: boolean\n path:\n description: |-\n The path within the volume from which to select the file.\n Must be relative and may not contain the '..' path or start with '..'.\n type: string\n volumeName:\n description: The name of the volume\n mount containing the env file.\n type: string\n required:\n - key\n - path\n - volumeName\n type: object\n x-kubernetes-map-type: atomic\n resourceFieldRef:\n description: |-\n Selects a resource of the container: only resources limits and requests\n (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.\n properties:\n containerName:\n description: 'Container name: required\n for volumes, optional for env\n vars'\n type: string\n divisor:\n anyOf:\n - type: integer\n - type: string\n description: Specifies the output\n format of the exposed resources,\n defaults to \"1\"\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n resource:\n description: 'Required: resource\n to select'\n type: string\n required:\n - resource\n type: object\n x-kubernetes-map-type: atomic\n secretKeyRef:\n description: Selects a key of a secret\n in the pod's namespace\n properties:\n key:\n description: The key of the secret\n to select from. Must be a valid\n secret key.\n type: string\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the\n Secret or its key must be defined\n type: boolean\n required:\n - key\n type: object\n x-kubernetes-map-type: atomic\n type: object\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n envFrom:\n description: |-\n List of sources to populate environment variables in the container.\n The keys defined within a source may consist of any printable ASCII characters except '='.\n When a key exists in multiple\n sources, the value associated with the last source will take precedence.\n Values defined by an Env with a duplicate key will take precedence.\n Cannot be updated.\n items:\n description: EnvFromSource represents the source\n of a set of ConfigMaps or Secrets\n properties:\n configMapRef:\n description: The ConfigMap to select from\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the ConfigMap\n must be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n prefix:\n description: |-\n Optional text to prepend to the name of each environment variable.\n May consist of any printable ASCII characters except '='.\n type: string\n secretRef:\n description: The Secret to select from\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: Specify whether the Secret\n must be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n type: object\n type: array\n x-kubernetes-list-type: atomic\n image:\n description: |-\n Container image name.\n More info: https://kubernetes.io/docs/concepts/containers/images\n This field is optional to allow higher level config management to default or override\n container images in workload controllers like Deployments and StatefulSets.\n type: string\n imagePullPolicy:\n description: |-\n Image pull policy.\n One of Always, Never, IfNotPresent.\n Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/containers/images#updating-images\n type: string\n lifecycle:\n description: |-\n Actions that the management system should take in response to container lifecycle events.\n Cannot be updated.\n properties:\n postStart:\n description: |-\n PostStart is called immediately after a container is created. If the handler fails,\n the container is terminated and restarted according to its restart policy.\n Other management of the container blocks until the hook completes.\n More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\n properties:\n exec:\n description: Exec specifies a command\n to execute in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP\n GET request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set\n in the request. HTTP allows repeated\n headers.\n items:\n description: HTTPHeader describes\n a custom header to be used in\n HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field\n value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the\n HTTP server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n sleep:\n description: Sleep represents a duration\n that the container should sleep.\n properties:\n seconds:\n description: Seconds is the number\n of seconds to sleep.\n format: int64\n type: integer\n required:\n - seconds\n type: object\n tcpSocket:\n description: |-\n Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept\n for backward compatibility. There is no validation of this field and\n lifecycle hooks will fail at runtime when it is specified.\n properties:\n host:\n description: 'Optional: Host name\n to connect to, defaults to the pod\n IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n type: object\n preStop:\n description: |-\n PreStop is called immediately before a container is terminated due to an\n API request or management event such as liveness/startup probe failure,\n preemption, resource contention, etc. The handler is not called if the\n container crashes or exits. The Pod's termination grace period countdown begins before the\n PreStop hook is executed. Regardless of the outcome of the handler, the\n container will eventually terminate within the Pod's termination grace\n period (unless delayed by finalizers). Other management of the container blocks until the hook completes\n or until the termination grace period is reached.\n More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks\n properties:\n exec:\n description: Exec specifies a command\n to execute in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP\n GET request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set\n in the request. HTTP allows repeated\n headers.\n items:\n description: HTTPHeader describes\n a custom header to be used in\n HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field\n value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the\n HTTP server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n sleep:\n description: Sleep represents a duration\n that the container should sleep.\n properties:\n seconds:\n description: Seconds is the number\n of seconds to sleep.\n format: int64\n type: integer\n required:\n - seconds\n type: object\n tcpSocket:\n description: |-\n Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept\n for backward compatibility. There is no validation of this field and\n lifecycle hooks will fail at runtime when it is specified.\n properties:\n host:\n description: 'Optional: Host name\n to connect to, defaults to the pod\n IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n type: object\n stopSignal:\n description: |-\n StopSignal defines which signal will be sent to a container when it is being stopped.\n If not specified, the default is defined by the container runtime in use.\n StopSignal can only be set for Pods with a non-empty .spec.os.name\n type: string\n type: object\n livenessProbe:\n description: |-\n Periodic probe of container liveness.\n Container will be restarted if the probe fails.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n name:\n description: |-\n Name of the container specified as a DNS_LABEL.\n Each container in a pod must have a unique name (DNS_LABEL).\n Cannot be updated.\n type: string\n ports:\n description: |-\n List of ports to expose from the container. Not specifying a port here\n DOES NOT prevent that port from being exposed. Any port which is\n listening on the default \"0.0.0.0\" address inside a container will be\n accessible from the network.\n Modifying this array with strategic merge patch may corrupt the data.\n For more information See https://github.com/kubernetes/kubernetes/issues/108255.\n Cannot be updated.\n items:\n description: ContainerPort represents a network\n port in a single container.\n properties:\n containerPort:\n description: |-\n Number of port to expose on the pod's IP address.\n This must be a valid port number, 0 < x < 65536.\n format: int32\n type: integer\n hostIP:\n description: What host IP to bind the external\n port to.\n type: string\n hostPort:\n description: |-\n Number of port to expose on the host.\n If specified, this must be a valid port number, 0 < x < 65536.\n If HostNetwork is specified, this must match ContainerPort.\n Most containers do not need this.\n format: int32\n type: integer\n name:\n description: |-\n If specified, this must be an IANA_SVC_NAME and unique within the pod. Each\n named port in a pod must have a unique name. Name for the port that can be\n referred to by services.\n type: string\n protocol:\n default: TCP\n description: |-\n Protocol for port. Must be UDP, TCP, or SCTP.\n Defaults to \"TCP\".\n type: string\n required:\n - containerPort\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - containerPort\n - protocol\n x-kubernetes-list-type: map\n readinessProbe:\n description: |-\n Periodic probe of container service readiness.\n Container will be removed from service endpoints if the probe fails.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n resizePolicy:\n description: |-\n Resources resize policy for the container.\n This field cannot be set on ephemeral containers.\n items:\n description: ContainerResizePolicy represents\n resource resize policy for the container.\n properties:\n resourceName:\n description: |-\n Name of the resource to which this resource resize policy applies.\n Supported values: cpu, memory.\n type: string\n restartPolicy:\n description: |-\n Restart policy to apply when specified resource is resized.\n If not specified, it defaults to NotRequired.\n type: string\n required:\n - resourceName\n - restartPolicy\n type: object\n type: array\n x-kubernetes-list-type: atomic\n resources:\n description: |-\n Compute Resources required by this container.\n Cannot be updated.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n properties:\n claims:\n description: |-\n Claims lists the names of resources, defined in spec.resourceClaims,\n that are used by this container.\n\n This field depends on the\n DynamicResourceAllocation feature gate.\n\n This field is immutable. It can only be set for containers.\n items:\n description: ResourceClaim references one\n entry in PodSpec.ResourceClaims.\n properties:\n name:\n description: |-\n Name must match the name of one entry in pod.spec.resourceClaims of\n the Pod where this field is used. It makes that resource available\n inside a container.\n type: string\n request:\n description: |-\n Request is the name chosen for a request in the referenced claim.\n If empty, everything from the claim is made available, otherwise\n only the result of this request.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n limits:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Limits describes the maximum amount of compute resources allowed.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n requests:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Requests describes the minimum amount of compute resources required.\n If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,\n otherwise to an implementation-defined value. Requests cannot exceed Limits.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n type: object\n restartPolicy:\n description: |-\n RestartPolicy defines the restart behavior of individual containers in a pod.\n This overrides the pod-level restart policy. When this field is not specified,\n the restart behavior is defined by the Pod's restart policy and the container type.\n Additionally, setting the RestartPolicy as \"Always\" for the init container will\n have the following effect:\n this init container will be continually restarted on\n exit until all regular containers have terminated. Once all regular\n containers have completed, all init containers with restartPolicy \"Always\"\n will be shut down. This lifecycle differs from normal init containers and\n is often referred to as a \"sidecar\" container. Although this init\n container still starts in the init container sequence, it does not wait\n for the container to complete before proceeding to the next init\n container. Instead, the next init container starts immediately after this\n init container is started, or after any startupProbe has successfully\n completed.\n type: string\n restartPolicyRules:\n description: |-\n Represents a list of rules to be checked to determine if the\n container should be restarted on exit. The rules are evaluated in\n order. Once a rule matches a container exit condition, the remaining\n rules are ignored. If no rule matches the container exit condition,\n the Container-level restart policy determines the whether the container\n is restarted or not. Constraints on the rules:\n - At most 20 rules are allowed.\n - Rules can have the same action.\n - Identical rules are not forbidden in validations.\n When rules are specified, container MUST set RestartPolicy explicitly\n even it if matches the Pod's RestartPolicy.\n items:\n description: ContainerRestartRule describes\n how a container exit is handled.\n properties:\n action:\n description: |-\n Specifies the action taken on a container exit if the requirements\n are satisfied. The only possible value is \"Restart\" to restart the\n container.\n type: string\n exitCodes:\n description: Represents the exit codes to\n check on container exits.\n properties:\n operator:\n description: |-\n Represents the relationship between the container exit code(s) and the\n specified values. Possible values are:\n - In: the requirement is satisfied if the container exit code is in the\n set of specified values.\n - NotIn: the requirement is satisfied if the container exit code is\n not in the set of specified values.\n type: string\n values:\n description: |-\n Specifies the set of values to check for container exit codes.\n At most 255 elements are allowed.\n items:\n format: int32\n type: integer\n type: array\n x-kubernetes-list-type: set\n required:\n - operator\n type: object\n required:\n - action\n type: object\n type: array\n x-kubernetes-list-type: atomic\n securityContext:\n description: |-\n SecurityContext defines the security options the container should be run with.\n If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext.\n More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/\n properties:\n allowPrivilegeEscalation:\n description: |-\n AllowPrivilegeEscalation controls whether a process can gain more\n privileges than its parent process. This bool directly controls if\n the no_new_privs flag will be set on the container process.\n AllowPrivilegeEscalation is true always when the container is:\n 1) run as Privileged\n 2) has CAP_SYS_ADMIN\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n appArmorProfile:\n description: |-\n appArmorProfile is the AppArmor options to use by this container. If set, this profile\n overrides the pod's appArmorProfile.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n localhostProfile:\n description: |-\n localhostProfile indicates a profile loaded on the node that should be used.\n The profile must be preconfigured on the node to work.\n Must match the loaded name of the profile.\n Must be set if and only if type is \"Localhost\".\n type: string\n type:\n description: |-\n type indicates which kind of AppArmor profile will be applied.\n Valid options are:\n Localhost - a profile pre-loaded on the node.\n RuntimeDefault - the container runtime's default profile.\n Unconfined - no AppArmor enforcement.\n type: string\n required:\n - type\n type: object\n capabilities:\n description: |-\n The capabilities to add/drop when running containers.\n Defaults to the default set of capabilities granted by the container runtime.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n add:\n description: Added capabilities\n items:\n description: Capability represent POSIX\n capabilities type\n type: string\n type: array\n x-kubernetes-list-type: atomic\n drop:\n description: Removed capabilities\n items:\n description: Capability represent POSIX\n capabilities type\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n privileged:\n description: |-\n Run container in privileged mode.\n Processes in privileged containers are essentially equivalent to root on the host.\n Defaults to false.\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n procMount:\n description: |-\n procMount denotes the type of proc mount to use for the containers.\n The default value is Default which uses the container runtime defaults for\n readonly paths and masked paths.\n This requires the ProcMountType feature flag to be enabled.\n Note that this field cannot be set when spec.os.name is windows.\n type: string\n readOnlyRootFilesystem:\n description: |-\n Whether this container has a read-only root filesystem.\n Default is false.\n Note that this field cannot be set when spec.os.name is windows.\n type: boolean\n runAsGroup:\n description: |-\n The GID to run the entrypoint of the container process.\n Uses runtime default if unset.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n runAsNonRoot:\n description: |-\n Indicates that the container must run as a non-root user.\n If true, the Kubelet will validate the image at runtime to ensure that it\n does not run as UID 0 (root) and fail to start the container if it does.\n If unset or false, no such validation will be performed.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n type: boolean\n runAsUser:\n description: |-\n The UID to run the entrypoint of the container process.\n Defaults to user specified in image metadata if unspecified.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n seLinuxOptions:\n description: |-\n The SELinux context to be applied to the container.\n If unspecified, the container runtime will allocate a random SELinux context for each\n container. May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n level:\n description: Level is SELinux level label\n that applies to the container.\n type: string\n role:\n description: Role is a SELinux role label\n that applies to the container.\n type: string\n type:\n description: Type is a SELinux type label\n that applies to the container.\n type: string\n user:\n description: User is a SELinux user label\n that applies to the container.\n type: string\n type: object\n seccompProfile:\n description: |-\n The seccomp options to use by this container. If seccomp options are\n provided at both the pod & container level, the container options\n override the pod options.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n localhostProfile:\n description: |-\n localhostProfile indicates a profile defined in a file on the node should be used.\n The profile must be preconfigured on the node to work.\n Must be a descending path, relative to the kubelet's configured seccomp profile location.\n Must be set if type is \"Localhost\". Must NOT be set for any other type.\n type: string\n type:\n description: |-\n type indicates which kind of seccomp profile will be applied.\n Valid options are:\n\n Localhost - a profile defined in a file on the node should be used.\n RuntimeDefault - the container runtime default profile should be used.\n Unconfined - no profile should be applied.\n type: string\n required:\n - type\n type: object\n windowsOptions:\n description: |-\n The Windows specific settings applied to all containers.\n If unspecified, the options from the PodSecurityContext will be used.\n If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is linux.\n properties:\n gmsaCredentialSpec:\n description: |-\n GMSACredentialSpec is where the GMSA admission webhook\n (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the\n GMSA credential spec named by the GMSACredentialSpecName field.\n type: string\n gmsaCredentialSpecName:\n description: GMSACredentialSpecName is\n the name of the GMSA credential spec\n to use.\n type: string\n hostProcess:\n description: |-\n HostProcess determines if a container should be run as a 'Host Process' container.\n All of a Pod's containers must have the same effective HostProcess value\n (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).\n In addition, if HostProcess is true then HostNetwork must also be set to true.\n type: boolean\n runAsUserName:\n description: |-\n The UserName in Windows to run the entrypoint of the container process.\n Defaults to the user specified in image metadata if unspecified.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n type: string\n type: object\n type: object\n startupProbe:\n description: |-\n StartupProbe indicates that the Pod has successfully initialized.\n If specified, no other probes are executed until this completes successfully.\n If this probe fails, the Pod will be restarted, just as if the livenessProbe failed.\n This can be used to provide different probe parameters at the beginning of a Pod's lifecycle,\n when it might take a long time to load data or warm a cache, than during steady-state operation.\n This cannot be updated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n properties:\n exec:\n description: Exec specifies a command to execute\n in the container.\n properties:\n command:\n description: |-\n Command is the command line to execute inside the container, the working directory for the\n command is root ('/') in the container's filesystem. The command is simply exec'd, it is\n not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use\n a shell, you need to explicitly call out to that shell.\n Exit status of 0 is treated as live/healthy and non-zero is unhealthy.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n failureThreshold:\n description: |-\n Minimum consecutive failures for the probe to be considered failed after having succeeded.\n Defaults to 3. Minimum value is 1.\n format: int32\n type: integer\n grpc:\n description: GRPC specifies a GRPC HealthCheckRequest.\n properties:\n port:\n description: Port number of the gRPC service.\n Number must be in the range 1 to 65535.\n format: int32\n type: integer\n service:\n default: \"\"\n description: |-\n Service is the name of the service to place in the gRPC HealthCheckRequest\n (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\n If this is not specified, the default behavior is defined by gRPC.\n type: string\n required:\n - port\n type: object\n httpGet:\n description: HTTPGet specifies an HTTP GET\n request to perform.\n properties:\n host:\n description: |-\n Host name to connect to, defaults to the pod IP. You probably want to set\n \"Host\" in httpHeaders instead.\n type: string\n httpHeaders:\n description: Custom headers to set in\n the request. HTTP allows repeated headers.\n items:\n description: HTTPHeader describes a\n custom header to be used in HTTP probes\n properties:\n name:\n description: |-\n The header field name.\n This will be canonicalized upon output, so case-variant names will be understood as the same header.\n type: string\n value:\n description: The header field value\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: Path to access on the HTTP\n server.\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Name or number of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n scheme:\n description: |-\n Scheme to use for connecting to the host.\n Defaults to HTTP.\n type: string\n required:\n - port\n type: object\n initialDelaySeconds:\n description: |-\n Number of seconds after the container has started before liveness probes are initiated.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n periodSeconds:\n description: |-\n How often (in seconds) to perform the probe.\n Default to 10 seconds. Minimum value is 1.\n format: int32\n type: integer\n successThreshold:\n description: |-\n Minimum consecutive successes for the probe to be considered successful after having failed.\n Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.\n format: int32\n type: integer\n tcpSocket:\n description: TCPSocket specifies a connection\n to a TCP port.\n properties:\n host:\n description: 'Optional: Host name to connect\n to, defaults to the pod IP.'\n type: string\n port:\n anyOf:\n - type: integer\n - type: string\n description: |-\n Number or name of the port to access on the container.\n Number must be in the range 1 to 65535.\n Name must be an IANA_SVC_NAME.\n x-kubernetes-int-or-string: true\n required:\n - port\n type: object\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully upon probe failure.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this\n value overrides the value provided by the pod spec.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.\n Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.\n format: int64\n type: integer\n timeoutSeconds:\n description: |-\n Number of seconds after which the probe times out.\n Defaults to 1 second. Minimum value is 1.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes\n format: int32\n type: integer\n type: object\n stdin:\n description: |-\n Whether this container should allocate a buffer for stdin in the container runtime. If this\n is not set, reads from stdin in the container will always result in EOF.\n Default is false.\n type: boolean\n stdinOnce:\n description: |-\n Whether the container runtime should close the stdin channel after it has been opened by\n a single attach. When stdin is true the stdin stream will remain open across multiple attach\n sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the\n first client attaches to stdin, and then remains open and accepts data until the client disconnects,\n at which time stdin is closed and remains closed until the container is restarted. If this\n flag is false, a container processes that reads from stdin will never receive an EOF.\n Default is false\n type: boolean\n terminationMessagePath:\n description: |-\n Optional: Path at which the file to which the container's termination message\n will be written is mounted into the container's filesystem.\n Message written is intended to be brief final status, such as an assertion failure message.\n Will be truncated by the node if greater than 4096 bytes. The total message length across\n all containers will be limited to 12kb.\n Defaults to /dev/termination-log.\n Cannot be updated.\n type: string\n terminationMessagePolicy:\n description: |-\n Indicate how the termination message should be populated. File will use the contents of\n terminationMessagePath to populate the container status message on both success and failure.\n FallbackToLogsOnError will use the last chunk of container log output if the termination\n message file is empty and the container exited with an error.\n The log output is limited to 2048 bytes or 80 lines, whichever is smaller.\n Defaults to File.\n Cannot be updated.\n type: string\n tty:\n description: |-\n Whether this container should allocate a TTY for itself, also requires 'stdin' to be true.\n Default is false.\n type: boolean\n volumeDevices:\n description: volumeDevices is the list of block\n devices to be used by the container.\n items:\n description: volumeDevice describes a mapping\n of a raw block device within a container.\n properties:\n devicePath:\n description: devicePath is the path inside\n of the container that the device will\n be mapped to.\n type: string\n name:\n description: name must match the name of\n a persistentVolumeClaim in the pod\n type: string\n required:\n - devicePath\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - devicePath\n x-kubernetes-list-type: map\n volumeMounts:\n description: |-\n Pod volumes to mount into the container's filesystem.\n Cannot be updated.\n items:\n description: VolumeMount describes a mounting\n of a Volume within a container.\n properties:\n mountPath:\n description: |-\n Path within the container at which the volume should be mounted. Must\n not contain ':'.\n type: string\n mountPropagation:\n description: |-\n mountPropagation determines how mounts are propagated from the host\n to container and the other way around.\n When not set, MountPropagationNone is used.\n This field is beta in 1.10.\n When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified\n (which defaults to None).\n type: string\n name:\n description: This must match the Name of\n a Volume.\n type: string\n readOnly:\n description: |-\n Mounted read-only if true, read-write otherwise (false or unspecified).\n Defaults to false.\n type: boolean\n recursiveReadOnly:\n description: |-\n RecursiveReadOnly specifies whether read-only mounts should be handled\n recursively.\n\n If ReadOnly is false, this field has no meaning and must be unspecified.\n\n If ReadOnly is true, and this field is set to Disabled, the mount is not made\n recursively read-only. If this field is set to IfPossible, the mount is made\n recursively read-only, if it is supported by the container runtime. If this\n field is set to Enabled, the mount is made recursively read-only if it is\n supported by the container runtime, otherwise the pod will not be started and\n an error will be generated to indicate the reason.\n\n If this field is set to IfPossible or Enabled, MountPropagation must be set to\n None (or be unspecified, which defaults to None).\n\n If this field is not specified, it is treated as an equivalent of Disabled.\n type: string\n subPath:\n description: |-\n Path within the volume from which the container's volume should be mounted.\n Defaults to \"\" (volume's root).\n type: string\n subPathExpr:\n description: |-\n Expanded path within the volume from which the container's volume should be mounted.\n Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.\n Defaults to \"\" (volume's root).\n SubPathExpr and SubPath are mutually exclusive.\n type: string\n required:\n - mountPath\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - mountPath\n x-kubernetes-list-type: map\n workingDir:\n description: |-\n Container's working directory.\n If not specified, the container runtime's default will be used, which\n might be configured in the container image.\n Cannot be updated.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n nodeName:\n description: |-\n NodeName indicates in which node this pod is scheduled.\n If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName.\n Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod.\n This field should not be used to express a desire for the pod to be scheduled on a specific node.\n https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename\n type: string\n nodeSelector:\n additionalProperties:\n type: string\n description: |-\n NodeSelector is a selector which must be true for the pod to fit on a node.\n Selector which must match a node's labels for the pod to be scheduled on that node.\n More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/\n type: object\n x-kubernetes-map-type: atomic\n os:\n description: |-\n Specifies the OS of the containers in the pod.\n Some pod and container fields are restricted if this is set.\n\n If the OS field is set to linux, the following fields must be unset:\n -securityContext.windowsOptions\n\n If the OS field is set to windows, following fields must be unset:\n - spec.hostPID\n - spec.hostIPC\n - spec.hostUsers\n - spec.resources\n - spec.securityContext.appArmorProfile\n - spec.securityContext.seLinuxOptions\n - spec.securityContext.seccompProfile\n - spec.securityContext.fsGroup\n - spec.securityContext.fsGroupChangePolicy\n - spec.securityContext.sysctls\n - spec.shareProcessNamespace\n - spec.securityContext.runAsUser\n - spec.securityContext.runAsGroup\n - spec.securityContext.supplementalGroups\n - spec.securityContext.supplementalGroupsPolicy\n - spec.containers[*].securityContext.appArmorProfile\n - spec.containers[*].securityContext.seLinuxOptions\n - spec.containers[*].securityContext.seccompProfile\n - spec.containers[*].securityContext.capabilities\n - spec.containers[*].securityContext.readOnlyRootFilesystem\n - spec.containers[*].securityContext.privileged\n - spec.containers[*].securityContext.allowPrivilegeEscalation\n - spec.containers[*].securityContext.procMount\n - spec.containers[*].securityContext.runAsUser\n - spec.containers[*].securityContext.runAsGroup\n properties:\n name:\n description: |-\n Name is the name of the operating system. The currently supported values are linux and windows.\n Additional value may be defined in future and can be one of:\n https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration\n Clients should expect to handle additional values and treat unrecognized values in this field as os: null\n type: string\n required:\n - name\n type: object\n overhead:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Overhead represents the resource overhead associated with running a pod for a given RuntimeClass.\n This field will be autopopulated at admission time by the RuntimeClass admission controller. If\n the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests.\n The RuntimeClass admission controller will reject Pod create requests which have the overhead already\n set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value\n defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero.\n More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md\n type: object\n preemptionPolicy:\n description: |-\n PreemptionPolicy is the Policy for preempting pods with lower priority.\n One of Never, PreemptLowerPriority.\n Defaults to PreemptLowerPriority if unset.\n type: string\n priority:\n description: |-\n The priority value. Various system components use this field to find the\n priority of the pod. When Priority Admission Controller is enabled, it\n prevents users from setting this field. The admission controller populates\n this field from PriorityClassName.\n The higher the value, the higher the priority.\n format: int32\n type: integer\n priorityClassName:\n description: |-\n If specified, indicates the pod's priority. \"system-node-critical\" and\n \"system-cluster-critical\" are two special keywords which indicate the\n highest priorities with the former being the highest priority. Any other\n name must be defined by creating a PriorityClass object with that name.\n If not specified, the pod priority will be default or zero if there is no\n default.\n type: string\n readinessGates:\n description: |-\n If specified, all readiness gates will be evaluated for pod readiness.\n A pod is ready when all its containers are ready AND\n all conditions specified in the readiness gates have status equal to \"True\"\n More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates\n items:\n description: PodReadinessGate contains the reference\n to a pod condition\n properties:\n conditionType:\n description: ConditionType refers to a condition\n in the pod's condition list with matching type.\n type: string\n required:\n - conditionType\n type: object\n type: array\n x-kubernetes-list-type: atomic\n resourceClaims:\n description: |-\n ResourceClaims defines which ResourceClaims must be allocated\n and reserved before the Pod is allowed to start. The resources\n will be made available to those containers which consume them\n by name.\n\n This is a stable field but requires that the\n DynamicResourceAllocation feature gate is enabled.\n\n This field is immutable.\n items:\n description: |-\n PodResourceClaim references exactly one ResourceClaim, either directly\n or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim\n for the pod.\n\n It adds a name to it that uniquely identifies the ResourceClaim inside the Pod.\n Containers that need access to the ResourceClaim reference it with this name.\n properties:\n name:\n description: |-\n Name uniquely identifies this resource claim inside the pod.\n This must be a DNS_LABEL.\n type: string\n resourceClaimName:\n description: |-\n ResourceClaimName is the name of a ResourceClaim object in the same\n namespace as this pod.\n\n Exactly one of ResourceClaimName and ResourceClaimTemplateName must\n be set.\n type: string\n resourceClaimTemplateName:\n description: |-\n ResourceClaimTemplateName is the name of a ResourceClaimTemplate\n object in the same namespace as this pod.\n\n The template will be used to create a new ResourceClaim, which will\n be bound to this pod. When this pod is deleted, the ResourceClaim\n will also be deleted. The pod name and resource name, along with a\n generated component, will be used to form a unique name for the\n ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses.\n\n This field is immutable and no changes will be made to the\n corresponding ResourceClaim by the control plane after creating the\n ResourceClaim.\n\n Exactly one of ResourceClaimName and ResourceClaimTemplateName must\n be set.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n resources:\n description: |-\n Resources is the total amount of CPU and Memory resources required by all\n containers in the pod. It supports specifying Requests and Limits for\n \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\n This field enables fine-grained control over resource allocation for the\n entire pod, allowing resource sharing among containers in a pod.\n\n This is an alpha field and requires enabling the PodLevelResources feature\n gate.\n properties:\n claims:\n description: |-\n Claims lists the names of resources, defined in spec.resourceClaims,\n that are used by this container.\n\n This field depends on the\n DynamicResourceAllocation feature gate.\n\n This field is immutable. It can only be set for containers.\n items:\n description: ResourceClaim references one entry\n in PodSpec.ResourceClaims.\n properties:\n name:\n description: |-\n Name must match the name of one entry in pod.spec.resourceClaims of\n the Pod where this field is used. It makes that resource available\n inside a container.\n type: string\n request:\n description: |-\n Request is the name chosen for a request in the referenced claim.\n If empty, everything from the claim is made available, otherwise\n only the result of this request.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n limits:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Limits describes the maximum amount of compute resources allowed.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n requests:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Requests describes the minimum amount of compute resources required.\n If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,\n otherwise to an implementation-defined value. Requests cannot exceed Limits.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n type: object\n restartPolicy:\n description: |-\n Restart policy for all containers within the pod.\n One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted.\n Default to Always.\n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy\n type: string\n runtimeClassName:\n description: |-\n RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used\n to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run.\n If unset or empty, the \"legacy\" RuntimeClass will be used, which is an implicit class with an\n empty definition that uses the default runtime handler.\n More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class\n type: string\n schedulerName:\n description: |-\n If specified, the pod will be dispatched by specified scheduler.\n If not specified, the pod will be dispatched by default scheduler.\n type: string\n schedulingGates:\n description: |-\n SchedulingGates is an opaque list of values that if specified will block scheduling the pod.\n If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the\n scheduler will not attempt to schedule the pod.\n\n SchedulingGates can only be set at pod creation time, and be removed only afterwards.\n items:\n description: PodSchedulingGate is associated to a\n Pod to guard its scheduling.\n properties:\n name:\n description: |-\n Name of the scheduling gate.\n Each scheduling gate must have a unique name field.\n type: string\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n securityContext:\n description: |-\n SecurityContext holds pod-level security attributes and common container settings.\n Optional: Defaults to empty. See type description for default values of each field.\n properties:\n appArmorProfile:\n description: |-\n appArmorProfile is the AppArmor options to use by the containers in this pod.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n localhostProfile:\n description: |-\n localhostProfile indicates a profile loaded on the node that should be used.\n The profile must be preconfigured on the node to work.\n Must match the loaded name of the profile.\n Must be set if and only if type is \"Localhost\".\n type: string\n type:\n description: |-\n type indicates which kind of AppArmor profile will be applied.\n Valid options are:\n Localhost - a profile pre-loaded on the node.\n RuntimeDefault - the container runtime's default profile.\n Unconfined - no AppArmor enforcement.\n type: string\n required:\n - type\n type: object\n fsGroup:\n description: |-\n A special supplemental group that applies to all containers in a pod.\n Some volume types allow the Kubelet to change the ownership of that volume\n to be owned by the pod:\n\n 1. The owning GID will be the FSGroup\n 2. The setgid bit is set (new files created in the volume will be owned by FSGroup)\n 3. The permission bits are OR'd with rw-rw----\n\n If unset, the Kubelet will not modify the ownership and permissions of any volume.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n fsGroupChangePolicy:\n description: |-\n fsGroupChangePolicy defines behavior of changing ownership and permission of the volume\n before being exposed inside Pod. This field will only apply to\n volume types which support fsGroup based ownership(and permissions).\n It will have no effect on ephemeral volume types such as: secret, configmaps\n and emptydir.\n Valid values are \"OnRootMismatch\" and \"Always\". If not specified, \"Always\" is used.\n Note that this field cannot be set when spec.os.name is windows.\n type: string\n runAsGroup:\n description: |-\n The GID to run the entrypoint of the container process.\n Uses runtime default if unset.\n May also be set in SecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence\n for that container.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n runAsNonRoot:\n description: |-\n Indicates that the container must run as a non-root user.\n If true, the Kubelet will validate the image at runtime to ensure that it\n does not run as UID 0 (root) and fail to start the container if it does.\n If unset or false, no such validation will be performed.\n May also be set in SecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n type: boolean\n runAsUser:\n description: |-\n The UID to run the entrypoint of the container process.\n Defaults to user specified in image metadata if unspecified.\n May also be set in SecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence\n for that container.\n Note that this field cannot be set when spec.os.name is windows.\n format: int64\n type: integer\n seLinuxChangePolicy:\n description: |-\n seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod.\n It has no effect on nodes that do not support SELinux or to volumes does not support SELinux.\n Valid values are \"MountOption\" and \"Recursive\".\n\n \"Recursive\" means relabeling of all files on all Pod volumes by the container runtime.\n This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.\n\n \"MountOption\" mounts all eligible Pod volumes with `-o context` mount option.\n This requires all Pods that share the same volume to use the same SELinux label.\n It is not possible to share the same volume among privileged and unprivileged Pods.\n Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes\n whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their\n CSIDriver instance. Other volumes are always re-labelled recursively.\n \"MountOption\" value is allowed only when SELinuxMount feature gate is enabled.\n\n If not specified and SELinuxMount feature gate is enabled, \"MountOption\" is used.\n If not specified and SELinuxMount feature gate is disabled, \"MountOption\" is used for ReadWriteOncePod volumes\n and \"Recursive\" for all other volumes.\n\n This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.\n\n All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state.\n Note that this field cannot be set when spec.os.name is windows.\n type: string\n seLinuxOptions:\n description: |-\n The SELinux context to be applied to all containers.\n If unspecified, the container runtime will allocate a random SELinux context for each\n container. May also be set in SecurityContext. If set in\n both SecurityContext and PodSecurityContext, the value specified in SecurityContext\n takes precedence for that container.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n level:\n description: Level is SELinux level label that\n applies to the container.\n type: string\n role:\n description: Role is a SELinux role label that\n applies to the container.\n type: string\n type:\n description: Type is a SELinux type label that\n applies to the container.\n type: string\n user:\n description: User is a SELinux user label that\n applies to the container.\n type: string\n type: object\n seccompProfile:\n description: |-\n The seccomp options to use by the containers in this pod.\n Note that this field cannot be set when spec.os.name is windows.\n properties:\n localhostProfile:\n description: |-\n localhostProfile indicates a profile defined in a file on the node should be used.\n The profile must be preconfigured on the node to work.\n Must be a descending path, relative to the kubelet's configured seccomp profile location.\n Must be set if type is \"Localhost\". Must NOT be set for any other type.\n type: string\n type:\n description: |-\n type indicates which kind of seccomp profile will be applied.\n Valid options are:\n\n Localhost - a profile defined in a file on the node should be used.\n RuntimeDefault - the container runtime default profile should be used.\n Unconfined - no profile should be applied.\n type: string\n required:\n - type\n type: object\n supplementalGroups:\n description: |-\n A list of groups applied to the first process run in each container, in\n addition to the container's primary GID and fsGroup (if specified). If\n the SupplementalGroupsPolicy feature is enabled, the\n supplementalGroupsPolicy field determines whether these are in addition\n to or instead of any group memberships defined in the container image.\n If unspecified, no additional groups are added, though group memberships\n defined in the container image may still be used, depending on the\n supplementalGroupsPolicy field.\n Note that this field cannot be set when spec.os.name is windows.\n items:\n format: int64\n type: integer\n type: array\n x-kubernetes-list-type: atomic\n supplementalGroupsPolicy:\n description: |-\n Defines how supplemental groups of the first container processes are calculated.\n Valid values are \"Merge\" and \"Strict\". If not specified, \"Merge\" is used.\n (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled\n and the container runtime must implement support for this feature.\n Note that this field cannot be set when spec.os.name is windows.\n type: string\n sysctls:\n description: |-\n Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported\n sysctls (by the container runtime) might fail to launch.\n Note that this field cannot be set when spec.os.name is windows.\n items:\n description: Sysctl defines a kernel parameter\n to be set\n properties:\n name:\n description: Name of a property to set\n type: string\n value:\n description: Value of a property to set\n type: string\n required:\n - name\n - value\n type: object\n type: array\n x-kubernetes-list-type: atomic\n windowsOptions:\n description: |-\n The Windows specific settings applied to all containers.\n If unspecified, the options within a container's SecurityContext will be used.\n If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.\n Note that this field cannot be set when spec.os.name is linux.\n properties:\n gmsaCredentialSpec:\n description: |-\n GMSACredentialSpec is where the GMSA admission webhook\n (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the\n GMSA credential spec named by the GMSACredentialSpecName field.\n type: string\n gmsaCredentialSpecName:\n description: GMSACredentialSpecName is the name\n of the GMSA credential spec to use.\n type: string\n hostProcess:\n description: |-\n HostProcess determines if a container should be run as a 'Host Process' container.\n All of a Pod's containers must have the same effective HostProcess value\n (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers).\n In addition, if HostProcess is true then HostNetwork must also be set to true.\n type: boolean\n runAsUserName:\n description: |-\n The UserName in Windows to run the entrypoint of the container process.\n Defaults to the user specified in image metadata if unspecified.\n May also be set in PodSecurityContext. If set in both SecurityContext and\n PodSecurityContext, the value specified in SecurityContext takes precedence.\n type: string\n type: object\n type: object\n serviceAccount:\n description: |-\n DeprecatedServiceAccount is a deprecated alias for ServiceAccountName.\n Deprecated: Use serviceAccountName instead.\n type: string\n serviceAccountName:\n description: |-\n ServiceAccountName is the name of the ServiceAccount to use to run this pod.\n More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/\n type: string\n setHostnameAsFQDN:\n description: |-\n If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default).\n In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname).\n In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\CurrentControlSet\\\\Services\\\\Tcpip\\\\Parameters to FQDN.\n If a pod does not have FQDN, this has no effect.\n Default to false.\n type: boolean\n shareProcessNamespace:\n description: |-\n Share a single process namespace between all of the containers in a pod.\n When this is set containers will be able to view and signal processes from other containers\n in the same pod, and the first process in each container will not be assigned PID 1.\n HostPID and ShareProcessNamespace cannot both be set.\n Optional: Default to false.\n type: boolean\n subdomain:\n description: |-\n If specified, the fully qualified Pod hostname will be \"...svc.\".\n If not specified, the pod will not have a domainname at all.\n type: string\n terminationGracePeriodSeconds:\n description: |-\n Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request.\n Value must be non-negative integer. The value zero indicates stop immediately via\n the kill signal (no opportunity to shut down).\n If this value is nil, the default grace period will be used instead.\n The grace period is the duration in seconds after the processes running in the pod are sent\n a termination signal and the time when the processes are forcibly halted with a kill signal.\n Set this value longer than the expected cleanup time for your process.\n Defaults to 30 seconds.\n format: int64\n type: integer\n tolerations:\n description: If specified, the pod's tolerations.\n items:\n description: |-\n The pod this Toleration is attached to tolerates any taint that matches\n the triple using the matching operator .\n properties:\n effect:\n description: |-\n Effect indicates the taint effect to match. Empty means match all taint effects.\n When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.\n type: string\n key:\n description: |-\n Key is the taint key that the toleration applies to. Empty means match all taint keys.\n If the key is empty, operator must be Exists; this combination means to match all values and all keys.\n type: string\n operator:\n description: |-\n Operator represents a key's relationship to the value.\n Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal.\n Exists is equivalent to wildcard for value, so that a pod can\n tolerate all taints of a particular category.\n Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators).\n type: string\n tolerationSeconds:\n description: |-\n TolerationSeconds represents the period of time the toleration (which must be\n of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,\n it is not set, which means tolerate the taint forever (do not evict). Zero and\n negative values will be treated as 0 (evict immediately) by the system.\n format: int64\n type: integer\n value:\n description: |-\n Value is the taint value the toleration matches to.\n If the operator is Exists, the value should be empty, otherwise just a regular string.\n type: string\n type: object\n type: array\n x-kubernetes-list-type: atomic\n topologySpreadConstraints:\n description: |-\n TopologySpreadConstraints describes how a group of pods ought to spread across topology\n domains. Scheduler will schedule pods in a way which abides by the constraints.\n All topologySpreadConstraints are ANDed.\n items:\n description: TopologySpreadConstraint specifies how\n to spread matching pods among the given topology.\n properties:\n labelSelector:\n description: |-\n LabelSelector is used to find matching pods.\n Pods that match this label selector are counted to determine the number of pods\n in their corresponding topology domain.\n properties:\n matchExpressions:\n description: matchExpressions is a list of\n label selector requirements. The requirements\n are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that\n the selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n matchLabelKeys:\n description: |-\n MatchLabelKeys is a set of pod label keys to select the pods over which\n spreading will be calculated. The keys are used to lookup values from the\n incoming pod labels, those key-value labels are ANDed with labelSelector\n to select the group of existing pods over which spreading will be calculated\n for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector.\n MatchLabelKeys cannot be set when LabelSelector isn't set.\n Keys that don't exist in the incoming pod labels will\n be ignored. A null or empty list means only match against labelSelector.\n\n This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n maxSkew:\n description: |-\n MaxSkew describes the degree to which pods may be unevenly distributed.\n When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference\n between the number of matching pods in the target topology and the global minimum.\n The global minimum is the minimum number of matching pods in an eligible domain\n or zero if the number of eligible domains is less than MinDomains.\n For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same\n labelSelector spread as 2/2/1:\n In this case, the global minimum is 1.\n | zone1 | zone2 | zone3 |\n | P P | P P | P |\n - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2;\n scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2)\n violate MaxSkew(1).\n - if MaxSkew is 2, incoming pod can be scheduled onto any zone.\n When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence\n to topologies that satisfy it.\n It's a required field. Default value is 1 and 0 is not allowed.\n format: int32\n type: integer\n minDomains:\n description: |-\n MinDomains indicates a minimum number of eligible domains.\n When the number of eligible domains with matching topology keys is less than minDomains,\n Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed.\n And when the number of eligible domains with matching topology keys equals or greater than minDomains,\n this value has no effect on scheduling.\n As a result, when the number of eligible domains is less than minDomains,\n scheduler won't schedule more than maxSkew Pods to those domains.\n If value is nil, the constraint behaves as if MinDomains is equal to 1.\n Valid values are integers greater than 0.\n When value is not nil, WhenUnsatisfiable must be DoNotSchedule.\n\n For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same\n labelSelector spread as 2/2/2:\n | zone1 | zone2 | zone3 |\n | P P | P P | P P |\n The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0.\n In this situation, new pod with the same labelSelector cannot be scheduled,\n because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones,\n it will violate MaxSkew.\n format: int32\n type: integer\n nodeAffinityPolicy:\n description: |-\n NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector\n when calculating pod topology spread skew. Options are:\n - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations.\n - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.\n\n If this value is nil, the behavior is equivalent to the Honor policy.\n type: string\n nodeTaintsPolicy:\n description: |-\n NodeTaintsPolicy indicates how we will treat node taints when calculating\n pod topology spread skew. Options are:\n - Honor: nodes without taints, along with tainted nodes for which the incoming pod\n has a toleration, are included.\n - Ignore: node taints are ignored. All nodes are included.\n\n If this value is nil, the behavior is equivalent to the Ignore policy.\n type: string\n topologyKey:\n description: |-\n TopologyKey is the key of node labels. Nodes that have a label with this key\n and identical values are considered to be in the same topology.\n We consider each as a \"bucket\", and try to put balanced number\n of pods into each bucket.\n We define a domain as a particular instance of a topology.\n Also, we define an eligible domain as a domain whose nodes meet the requirements of\n nodeAffinityPolicy and nodeTaintsPolicy.\n e.g. If TopologyKey is \"kubernetes.io/hostname\", each Node is a domain of that topology.\n And, if TopologyKey is \"topology.kubernetes.io/zone\", each zone is a domain of that topology.\n It's a required field.\n type: string\n whenUnsatisfiable:\n description: |-\n WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy\n the spread constraint.\n - DoNotSchedule (default) tells the scheduler not to schedule it.\n - ScheduleAnyway tells the scheduler to schedule the pod in any location,\n but giving higher precedence to topologies that would help reduce the\n skew.\n A constraint is considered \"Unsatisfiable\" for an incoming pod\n if and only if every possible node assignment for that pod would violate\n \"MaxSkew\" on some topology.\n For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same\n labelSelector spread as 3/1/1:\n | zone1 | zone2 | zone3 |\n | P P P | P | P |\n If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled\n to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies\n MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler\n won't make it *more* imbalanced.\n It's a required field.\n type: string\n required:\n - maxSkew\n - topologyKey\n - whenUnsatisfiable\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - topologyKey\n - whenUnsatisfiable\n x-kubernetes-list-type: map\n volumes:\n description: |-\n List of volumes that can be mounted by containers belonging to the pod.\n More info: https://kubernetes.io/docs/concepts/storage/volumes\n items:\n description: Volume represents a named volume in a\n pod that may be accessed by any container in the\n pod.\n properties:\n awsElasticBlockStore:\n description: |-\n awsElasticBlockStore represents an AWS Disk resource that is attached to a\n kubelet's host machine and then exposed to the pod.\n Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree\n awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore\n properties:\n fsType:\n description: |-\n fsType is the filesystem type of the volume that you want to mount.\n Tip: Ensure that the filesystem type is supported by the host operating system.\n Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore\n type: string\n partition:\n description: |-\n partition is the partition in the volume that you want to mount.\n If omitted, the default is to mount by volume name.\n Examples: For volume /dev/sda1, you specify the partition as \"1\".\n Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty).\n format: int32\n type: integer\n readOnly:\n description: |-\n readOnly value true will force the readOnly setting in VolumeMounts.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore\n type: boolean\n volumeID:\n description: |-\n volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume).\n More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore\n type: string\n required:\n - volumeID\n type: object\n azureDisk:\n description: |-\n azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.\n Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type\n are redirected to the disk.csi.azure.com CSI driver.\n properties:\n cachingMode:\n description: 'cachingMode is the Host Caching\n mode: None, Read Only, Read Write.'\n type: string\n diskName:\n description: diskName is the Name of the data\n disk in the blob storage\n type: string\n diskURI:\n description: diskURI is the URI of data disk\n in the blob storage\n type: string\n fsType:\n default: ext4\n description: |-\n fsType is Filesystem type to mount.\n Must be a filesystem type supported by the host operating system.\n Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n type: string\n kind:\n description: 'kind expected values are Shared:\n multiple blob disks per storage account Dedicated:\n single blob disk per storage account Managed:\n azure managed data disk (only in managed\n availability set). defaults to shared'\n type: string\n readOnly:\n default: false\n description: |-\n readOnly Defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n type: boolean\n required:\n - diskName\n - diskURI\n type: object\n azureFile:\n description: |-\n azureFile represents an Azure File Service mount on the host and bind mount to the pod.\n Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type\n are redirected to the file.csi.azure.com CSI driver.\n properties:\n readOnly:\n description: |-\n readOnly defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n type: boolean\n secretName:\n description: secretName is the name of secret\n that contains Azure Storage Account Name\n and Key\n type: string\n shareName:\n description: shareName is the azure share\n Name\n type: string\n required:\n - secretName\n - shareName\n type: object\n cephfs:\n description: |-\n cephFS represents a Ceph FS mount on the host that shares a pod's lifetime.\n Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported.\n properties:\n monitors:\n description: |-\n monitors is Required: Monitors is a collection of Ceph monitors\n More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n path:\n description: 'path is Optional: Used as the\n mounted root, rather than the full Ceph\n tree, default is /'\n type: string\n readOnly:\n description: |-\n readOnly is Optional: Defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\n type: boolean\n secretFile:\n description: |-\n secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret\n More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\n type: string\n secretRef:\n description: |-\n secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty.\n More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n user:\n description: |-\n user is optional: User is the rados user name, default is admin\n More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it\n type: string\n required:\n - monitors\n type: object\n cinder:\n description: |-\n cinder represents a cinder volume attached and mounted on kubelets host machine.\n Deprecated: Cinder is deprecated. All operations for the in-tree cinder type\n are redirected to the cinder.csi.openstack.org CSI driver.\n More info: https://examples.k8s.io/mysql-cinder-pd/README.md\n properties:\n fsType:\n description: |-\n fsType is the filesystem type to mount.\n Must be a filesystem type supported by the host operating system.\n Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n More info: https://examples.k8s.io/mysql-cinder-pd/README.md\n type: string\n readOnly:\n description: |-\n readOnly defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n More info: https://examples.k8s.io/mysql-cinder-pd/README.md\n type: boolean\n secretRef:\n description: |-\n secretRef is optional: points to a secret object containing parameters used to connect\n to OpenStack.\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n volumeID:\n description: |-\n volumeID used to identify the volume in cinder.\n More info: https://examples.k8s.io/mysql-cinder-pd/README.md\n type: string\n required:\n - volumeID\n type: object\n configMap:\n description: configMap represents a configMap\n that should populate this volume\n properties:\n defaultMode:\n description: |-\n defaultMode is optional: mode bits used to set permissions on created files by default.\n Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n Defaults to 0644.\n Directories within the path are not affected by this setting.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n items:\n description: |-\n items if unspecified, each key-value pair in the Data field of the referenced\n ConfigMap will be projected into the volume as a file whose name is the\n key and content is the value. If specified, the listed keys will be\n projected into the specified paths, and unlisted keys will not be\n present. If a key is specified which is not present in the ConfigMap,\n the volume setup will error unless it is marked optional. Paths must be\n relative and may not contain the '..' path or start with '..'.\n items:\n description: Maps a string key to a path\n within a volume.\n properties:\n key:\n description: key is the key to project.\n type: string\n mode:\n description: |-\n mode is Optional: mode bits used to set permissions on this file.\n Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n If not specified, the volume defaultMode will be used.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n path:\n description: |-\n path is the relative path of the file to map the key to.\n May not be an absolute path.\n May not contain the path element '..'.\n May not start with the string '..'.\n type: string\n required:\n - key\n - path\n type: object\n type: array\n x-kubernetes-list-type: atomic\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: optional specify whether the\n ConfigMap or its keys must be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n csi:\n description: csi (Container Storage Interface)\n represents ephemeral storage that is handled\n by certain external CSI drivers.\n properties:\n driver:\n description: |-\n driver is the name of the CSI driver that handles this volume.\n Consult with your admin for the correct name as registered in the cluster.\n type: string\n fsType:\n description: |-\n fsType to mount. Ex. \"ext4\", \"xfs\", \"ntfs\".\n If not provided, the empty value is passed to the associated CSI driver\n which will determine the default filesystem to apply.\n type: string\n nodePublishSecretRef:\n description: |-\n nodePublishSecretRef is a reference to the secret object containing\n sensitive information to pass to the CSI driver to complete the CSI\n NodePublishVolume and NodeUnpublishVolume calls.\n This field is optional, and may be empty if no secret is required. If the\n secret object contains more than one secret, all secret references are passed.\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n readOnly:\n description: |-\n readOnly specifies a read-only configuration for the volume.\n Defaults to false (read/write).\n type: boolean\n volumeAttributes:\n additionalProperties:\n type: string\n description: |-\n volumeAttributes stores driver-specific properties that are passed to the CSI\n driver. Consult your driver's documentation for supported values.\n type: object\n required:\n - driver\n type: object\n downwardAPI:\n description: downwardAPI represents downward API\n about the pod that should populate this volume\n properties:\n defaultMode:\n description: |-\n Optional: mode bits to use on created files by default. Must be a\n Optional: mode bits used to set permissions on created files by default.\n Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n Defaults to 0644.\n Directories within the path are not affected by this setting.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n items:\n description: Items is a list of downward API\n volume file\n items:\n description: DownwardAPIVolumeFile represents\n information to create the file containing\n the pod field\n properties:\n fieldRef:\n description: 'Required: Selects a field\n of the pod: only annotations, labels,\n name, namespace and uid are supported.'\n properties:\n apiVersion:\n description: Version of the schema\n the FieldPath is written in terms\n of, defaults to \"v1\".\n type: string\n fieldPath:\n description: Path of the field to\n select in the specified API version.\n type: string\n required:\n - fieldPath\n type: object\n x-kubernetes-map-type: atomic\n mode:\n description: |-\n Optional: mode bits used to set permissions on this file, must be an octal value\n between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n If not specified, the volume defaultMode will be used.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n path:\n description: 'Required: Path is the\n relative path name of the file to\n be created. Must not be absolute or\n contain the ''..'' path. Must be utf-8\n encoded. The first item of the relative\n path must not start with ''..'''\n type: string\n resourceFieldRef:\n description: |-\n Selects a resource of the container: only resources limits and requests\n (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.\n properties:\n containerName:\n description: 'Container name: required\n for volumes, optional for env\n vars'\n type: string\n divisor:\n anyOf:\n - type: integer\n - type: string\n description: Specifies the output\n format of the exposed resources,\n defaults to \"1\"\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n resource:\n description: 'Required: resource\n to select'\n type: string\n required:\n - resource\n type: object\n x-kubernetes-map-type: atomic\n required:\n - path\n type: object\n type: array\n x-kubernetes-list-type: atomic\n type: object\n emptyDir:\n description: |-\n emptyDir represents a temporary directory that shares a pod's lifetime.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\n properties:\n medium:\n description: |-\n medium represents what type of storage medium should back this directory.\n The default is \"\" which means to use the node's default medium.\n Must be an empty string (default) or Memory.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\n type: string\n sizeLimit:\n anyOf:\n - type: integer\n - type: string\n description: |-\n sizeLimit is the total amount of local storage required for this EmptyDir volume.\n The size limit is also applicable for memory medium.\n The maximum usage on memory medium EmptyDir would be the minimum value between\n the SizeLimit specified here and the sum of memory limits of all containers in a pod.\n The default is nil which means that the limit is undefined.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n type: object\n ephemeral:\n description: |-\n ephemeral represents a volume that is handled by a cluster storage driver.\n The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts,\n and deleted when the pod is removed.\n\n Use this if:\n a) the volume is only needed while the pod runs,\n b) features of normal volumes like restoring from snapshot or capacity\n tracking are needed,\n c) the storage driver is specified through a storage class, and\n d) the storage driver supports dynamic volume provisioning through\n a PersistentVolumeClaim (see EphemeralVolumeSource for more\n information on the connection between this volume type\n and PersistentVolumeClaim).\n\n Use PersistentVolumeClaim or one of the vendor-specific\n APIs for volumes that persist for longer than the lifecycle\n of an individual pod.\n\n Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to\n be used that way - see the documentation of the driver for\n more information.\n\n A pod can use both types of ephemeral volumes and\n persistent volumes at the same time.\n properties:\n volumeClaimTemplate:\n description: |-\n Will be used to create a stand-alone PVC to provision the volume.\n The pod in which this EphemeralVolumeSource is embedded will be the\n owner of the PVC, i.e. the PVC will be deleted together with the\n pod. The name of the PVC will be `-` where\n `` is the name from the `PodSpec.Volumes` array\n entry. Pod validation will reject the pod if the concatenated name\n is not valid for a PVC (for example, too long).\n\n An existing PVC with that name that is not owned by the pod\n will *not* be used for the pod to avoid using an unrelated\n volume by mistake. Starting the pod is then blocked until\n the unrelated PVC is removed. If such a pre-created PVC is\n meant to be used by the pod, the PVC has to updated with an\n owner reference to the pod once the pod exists. Normally\n this should not be necessary, but it may be useful when\n manually reconstructing a broken cluster.\n\n This field is read-only and no changes will be made by Kubernetes\n to the PVC after it has been created.\n\n Required, must not be nil.\n properties:\n metadata:\n description: |-\n May contain labels and annotations that will be copied into the PVC\n when creating it. No other fields are allowed and will be rejected during\n validation.\n properties:\n annotations:\n additionalProperties:\n type: string\n type: object\n finalizers:\n items:\n type: string\n type: array\n labels:\n additionalProperties:\n type: string\n type: object\n name:\n type: string\n namespace:\n type: string\n type: object\n spec:\n description: |-\n The specification for the PersistentVolumeClaim. The entire content is\n copied unchanged into the PVC that gets created from this\n template. The same fields as in a PersistentVolumeClaim\n are also valid here.\n properties:\n accessModes:\n description: |-\n accessModes contains the desired access modes the volume should have.\n More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n dataSource:\n description: |-\n dataSource field can be used to specify either:\n * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot)\n * An existing PVC (PersistentVolumeClaim)\n If the provisioner or an external controller can support the specified data source,\n it will create a new volume based on the contents of the specified data source.\n When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef,\n and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified.\n If the namespace is specified, then dataSourceRef will not be copied to dataSource.\n properties:\n apiGroup:\n description: |-\n APIGroup is the group for the resource being referenced.\n If APIGroup is not specified, the specified Kind must be in the core API group.\n For any other third-party types, APIGroup is required.\n type: string\n kind:\n description: Kind is the type\n of resource being referenced\n type: string\n name:\n description: Name is the name\n of resource being referenced\n type: string\n required:\n - kind\n - name\n type: object\n x-kubernetes-map-type: atomic\n dataSourceRef:\n description: |-\n dataSourceRef specifies the object from which to populate the volume with data, if a non-empty\n volume is desired. This may be any object from a non-empty API group (non\n core object) or a PersistentVolumeClaim object.\n When this field is specified, volume binding will only succeed if the type of\n the specified object matches some installed volume populator or dynamic\n provisioner.\n This field will replace the functionality of the dataSource field and as such\n if both fields are non-empty, they must have the same value. For backwards\n compatibility, when namespace isn't specified in dataSourceRef,\n both fields (dataSource and dataSourceRef) will be set to the same\n value automatically if one of them is empty and the other is non-empty.\n When namespace is specified in dataSourceRef,\n dataSource isn't set to the same value and must be empty.\n There are three important differences between dataSource and dataSourceRef:\n * While dataSource only allows two specific types of objects, dataSourceRef\n allows any non-core object, as well as PersistentVolumeClaim objects.\n * While dataSource ignores disallowed values (dropping them), dataSourceRef\n preserves all values, and generates an error if a disallowed value is\n specified.\n * While dataSource only allows local objects, dataSourceRef allows objects\n in any namespaces.\n (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled.\n (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled.\n properties:\n apiGroup:\n description: |-\n APIGroup is the group for the resource being referenced.\n If APIGroup is not specified, the specified Kind must be in the core API group.\n For any other third-party types, APIGroup is required.\n type: string\n kind:\n description: Kind is the type\n of resource being referenced\n type: string\n name:\n description: Name is the name\n of resource being referenced\n type: string\n namespace:\n description: |-\n Namespace is the namespace of resource being referenced\n Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details.\n (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled.\n type: string\n required:\n - kind\n - name\n type: object\n resources:\n description: |-\n resources represents the minimum resources the volume should have.\n Users are allowed to specify resource requirements\n that are lower than previous value but must still be higher than capacity recorded in the\n status field of the claim.\n More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources\n properties:\n limits:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Limits describes the maximum amount of compute resources allowed.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n requests:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n Requests describes the minimum amount of compute resources required.\n If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,\n otherwise to an implementation-defined value. Requests cannot exceed Limits.\n More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n type: object\n type: object\n selector:\n description: selector is a label query\n over volumes to consider for binding.\n properties:\n matchExpressions:\n description: matchExpressions\n is a list of label selector\n requirements. The requirements\n are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the\n label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n storageClassName:\n description: |-\n storageClassName is the name of the StorageClass required by the claim.\n More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1\n type: string\n volumeAttributesClassName:\n description: |-\n volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim.\n If specified, the CSI driver will create or update the volume with the attributes defined\n in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName,\n it can be changed after the claim is created. An empty string or nil value indicates that no\n VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state,\n this field can be reset to its previous value (including nil) to cancel the modification.\n If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be\n set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource\n exists.\n More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/\n type: string\n volumeMode:\n description: |-\n volumeMode defines what type of volume is required by the claim.\n Value of Filesystem is implied when not included in claim spec.\n type: string\n volumeName:\n description: volumeName is the binding\n reference to the PersistentVolume\n backing this claim.\n type: string\n type: object\n required:\n - spec\n type: object\n type: object\n fc:\n description: fc represents a Fibre Channel resource\n that is attached to a kubelet's host machine\n and then exposed to the pod.\n properties:\n fsType:\n description: |-\n fsType is the filesystem type to mount.\n Must be a filesystem type supported by the host operating system.\n Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n type: string\n lun:\n description: 'lun is Optional: FC target lun\n number'\n format: int32\n type: integer\n readOnly:\n description: |-\n readOnly is Optional: Defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n type: boolean\n targetWWNs:\n description: 'targetWWNs is Optional: FC target\n worldwide names (WWNs)'\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n wwids:\n description: |-\n wwids Optional: FC volume world wide identifiers (wwids)\n Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n type: object\n flexVolume:\n description: |-\n flexVolume represents a generic volume resource that is\n provisioned/attached using an exec based plugin.\n Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead.\n properties:\n driver:\n description: driver is the name of the driver\n to use for this volume.\n type: string\n fsType:\n description: |-\n fsType is the filesystem type to mount.\n Must be a filesystem type supported by the host operating system.\n Ex. \"ext4\", \"xfs\", \"ntfs\". The default filesystem depends on FlexVolume script.\n type: string\n options:\n additionalProperties:\n type: string\n description: 'options is Optional: this field\n holds extra command options if any.'\n type: object\n readOnly:\n description: |-\n readOnly is Optional: defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n type: boolean\n secretRef:\n description: |-\n secretRef is Optional: secretRef is reference to the secret object containing\n sensitive information to pass to the plugin scripts. This may be\n empty if no secret object is specified. If the secret object\n contains more than one secret, all secrets are passed to the plugin\n scripts.\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n required:\n - driver\n type: object\n flocker:\n description: |-\n flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running.\n Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported.\n properties:\n datasetName:\n description: |-\n datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker\n should be considered as deprecated\n type: string\n datasetUUID:\n description: datasetUUID is the UUID of the\n dataset. This is unique identifier of a\n Flocker dataset\n type: string\n type: object\n gcePersistentDisk:\n description: |-\n gcePersistentDisk represents a GCE Disk resource that is attached to a\n kubelet's host machine and then exposed to the pod.\n Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree\n gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\n properties:\n fsType:\n description: |-\n fsType is filesystem type of the volume that you want to mount.\n Tip: Ensure that the filesystem type is supported by the host operating system.\n Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\n type: string\n partition:\n description: |-\n partition is the partition in the volume that you want to mount.\n If omitted, the default is to mount by volume name.\n Examples: For volume /dev/sda1, you specify the partition as \"1\".\n Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty).\n More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\n format: int32\n type: integer\n pdName:\n description: |-\n pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\n type: string\n readOnly:\n description: |-\n readOnly here will force the ReadOnly setting in VolumeMounts.\n Defaults to false.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk\n type: boolean\n required:\n - pdName\n type: object\n gitRepo:\n description: |-\n gitRepo represents a git repository at a particular revision.\n Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an\n EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir\n into the Pod's container.\n properties:\n directory:\n description: |-\n directory is the target directory name.\n Must not contain or start with '..'. If '.' is supplied, the volume directory will be the\n git repository. Otherwise, if specified, the volume will contain the git repository in\n the subdirectory with the given name.\n type: string\n repository:\n description: repository is the URL\n type: string\n revision:\n description: revision is the commit hash for\n the specified revision.\n type: string\n required:\n - repository\n type: object\n glusterfs:\n description: |-\n glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime.\n Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported.\n properties:\n endpoints:\n description: endpoints is the endpoint name\n that details Glusterfs topology.\n type: string\n path:\n description: |-\n path is the Glusterfs volume path.\n More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod\n type: string\n readOnly:\n description: |-\n readOnly here will force the Glusterfs volume to be mounted with read-only permissions.\n Defaults to false.\n More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod\n type: boolean\n required:\n - endpoints\n - path\n type: object\n hostPath:\n description: |-\n hostPath represents a pre-existing file or directory on the host\n machine that is directly exposed to the container. This is generally\n used for system agents or other privileged things that are allowed\n to see the host machine. Most containers will NOT need this.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath\n properties:\n path:\n description: |-\n path of the directory on the host.\n If the path is a symlink, it will follow the link to the real path.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath\n type: string\n type:\n description: |-\n type for HostPath Volume\n Defaults to \"\"\n More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath\n type: string\n required:\n - path\n type: object\n image:\n description: |-\n image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine.\n The volume is resolved at pod startup depending on which PullPolicy value is provided:\n\n - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails.\n - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present.\n - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails.\n\n The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation.\n A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message.\n The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field.\n The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images.\n The volume will be mounted read-only (ro) and non-executable files (noexec).\n Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33.\n The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type.\n properties:\n pullPolicy:\n description: |-\n Policy for pulling OCI objects. Possible values are:\n Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails.\n Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present.\n IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails.\n Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.\n type: string\n reference:\n description: |-\n Required: Image or artifact reference to be used.\n Behaves in the same way as pod.spec.containers[*].image.\n Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets.\n More info: https://kubernetes.io/docs/concepts/containers/images\n This field is optional to allow higher level config management to default or override\n container images in workload controllers like Deployments and StatefulSets.\n type: string\n type: object\n iscsi:\n description: |-\n iscsi represents an ISCSI Disk resource that is attached to a\n kubelet's host machine and then exposed to the pod.\n More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi\n properties:\n chapAuthDiscovery:\n description: chapAuthDiscovery defines whether\n support iSCSI Discovery CHAP authentication\n type: boolean\n chapAuthSession:\n description: chapAuthSession defines whether\n support iSCSI Session CHAP authentication\n type: boolean\n fsType:\n description: |-\n fsType is the filesystem type of the volume that you want to mount.\n Tip: Ensure that the filesystem type is supported by the host operating system.\n Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi\n type: string\n initiatorName:\n description: |-\n initiatorName is the custom iSCSI Initiator Name.\n If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface\n : will be created for the connection.\n type: string\n iqn:\n description: iqn is the target iSCSI Qualified\n Name.\n type: string\n iscsiInterface:\n default: default\n description: |-\n iscsiInterface is the interface Name that uses an iSCSI transport.\n Defaults to 'default' (tcp).\n type: string\n lun:\n description: lun represents iSCSI Target Lun\n number.\n format: int32\n type: integer\n portals:\n description: |-\n portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port\n is other than default (typically TCP ports 860 and 3260).\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n readOnly:\n description: |-\n readOnly here will force the ReadOnly setting in VolumeMounts.\n Defaults to false.\n type: boolean\n secretRef:\n description: secretRef is the CHAP Secret\n for iSCSI target and initiator authentication\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n targetPortal:\n description: |-\n targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port\n is other than default (typically TCP ports 860 and 3260).\n type: string\n required:\n - iqn\n - lun\n - targetPortal\n type: object\n name:\n description: |-\n name of the volume.\n Must be a DNS_LABEL and unique within the pod.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n nfs:\n description: |-\n nfs represents an NFS mount on the host that shares a pod's lifetime\n More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs\n properties:\n path:\n description: |-\n path that is exported by the NFS server.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs\n type: string\n readOnly:\n description: |-\n readOnly here will force the NFS export to be mounted with read-only permissions.\n Defaults to false.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs\n type: boolean\n server:\n description: |-\n server is the hostname or IP address of the NFS server.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs\n type: string\n required:\n - path\n - server\n type: object\n persistentVolumeClaim:\n description: |-\n persistentVolumeClaimVolumeSource represents a reference to a\n PersistentVolumeClaim in the same namespace.\n More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\n properties:\n claimName:\n description: |-\n claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume.\n More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims\n type: string\n readOnly:\n description: |-\n readOnly Will force the ReadOnly setting in VolumeMounts.\n Default false.\n type: boolean\n required:\n - claimName\n type: object\n photonPersistentDisk:\n description: |-\n photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine.\n Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported.\n properties:\n fsType:\n description: |-\n fsType is the filesystem type to mount.\n Must be a filesystem type supported by the host operating system.\n Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n type: string\n pdID:\n description: pdID is the ID that identifies\n Photon Controller persistent disk\n type: string\n required:\n - pdID\n type: object\n portworxVolume:\n description: |-\n portworxVolume represents a portworx volume attached and mounted on kubelets host machine.\n Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type\n are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate\n is on.\n properties:\n fsType:\n description: |-\n fSType represents the filesystem type to mount\n Must be a filesystem type supported by the host operating system.\n Ex. \"ext4\", \"xfs\". Implicitly inferred to be \"ext4\" if unspecified.\n type: string\n readOnly:\n description: |-\n readOnly defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n type: boolean\n volumeID:\n description: volumeID uniquely identifies\n a Portworx volume\n type: string\n required:\n - volumeID\n type: object\n projected:\n description: projected items for all in one resources\n secrets, configmaps, and downward API\n properties:\n defaultMode:\n description: |-\n defaultMode are the mode bits used to set permissions on created files by default.\n Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n Directories within the path are not affected by this setting.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n sources:\n description: |-\n sources is the list of volume projections. Each entry in this list\n handles one source.\n items:\n description: |-\n Projection that may be projected along with other supported volume types.\n Exactly one of these fields must be set.\n properties:\n clusterTrustBundle:\n description: |-\n ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field\n of ClusterTrustBundle objects in an auto-updating file.\n\n Alpha, gated by the ClusterTrustBundleProjection feature gate.\n\n ClusterTrustBundle objects can either be selected by name, or by the\n combination of signer name and a label selector.\n\n Kubelet performs aggressive normalization of the PEM contents written\n into the pod filesystem. Esoteric PEM features such as inter-block\n comments and block headers are stripped. Certificates are deduplicated.\n The ordering of certificates within the file is arbitrary, and Kubelet\n may change the order over time.\n properties:\n labelSelector:\n description: |-\n Select all ClusterTrustBundles that match this label selector. Only has\n effect if signerName is set. Mutually-exclusive with name. If unset,\n interpreted as \"match nothing\". If set but empty, interpreted as \"match\n everything\".\n properties:\n matchExpressions:\n description: matchExpressions\n is a list of label selector\n requirements. The requirements\n are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the\n label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n name:\n description: |-\n Select a single ClusterTrustBundle by object name. Mutually-exclusive\n with signerName and labelSelector.\n type: string\n optional:\n description: |-\n If true, don't block pod startup if the referenced ClusterTrustBundle(s)\n aren't available. If using name, then the named ClusterTrustBundle is\n allowed not to exist. If using signerName, then the combination of\n signerName and labelSelector is allowed to match zero\n ClusterTrustBundles.\n type: boolean\n path:\n description: Relative path from\n the volume root to write the bundle.\n type: string\n signerName:\n description: |-\n Select all ClusterTrustBundles that match this signer name.\n Mutually-exclusive with name. The contents of all selected\n ClusterTrustBundles will be unified and deduplicated.\n type: string\n required:\n - path\n type: object\n configMap:\n description: configMap information about\n the configMap data to project\n properties:\n items:\n description: |-\n items if unspecified, each key-value pair in the Data field of the referenced\n ConfigMap will be projected into the volume as a file whose name is the\n key and content is the value. If specified, the listed keys will be\n projected into the specified paths, and unlisted keys will not be\n present. If a key is specified which is not present in the ConfigMap,\n the volume setup will error unless it is marked optional. Paths must be\n relative and may not contain the '..' path or start with '..'.\n items:\n description: Maps a string key\n to a path within a volume.\n properties:\n key:\n description: key is the key\n to project.\n type: string\n mode:\n description: |-\n mode is Optional: mode bits used to set permissions on this file.\n Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n If not specified, the volume defaultMode will be used.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n path:\n description: |-\n path is the relative path of the file to map the key to.\n May not be an absolute path.\n May not contain the path element '..'.\n May not start with the string '..'.\n type: string\n required:\n - key\n - path\n type: object\n type: array\n x-kubernetes-list-type: atomic\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: optional specify whether\n the ConfigMap or its keys must\n be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n downwardAPI:\n description: downwardAPI information\n about the downwardAPI data to project\n properties:\n items:\n description: Items is a list of\n DownwardAPIVolume file\n items:\n description: DownwardAPIVolumeFile\n represents information to create\n the file containing the pod\n field\n properties:\n fieldRef:\n description: 'Required: Selects\n a field of the pod: only\n annotations, labels, name,\n namespace and uid are supported.'\n properties:\n apiVersion:\n description: Version of\n the schema the FieldPath\n is written in terms\n of, defaults to \"v1\".\n type: string\n fieldPath:\n description: Path of the\n field to select in the\n specified API version.\n type: string\n required:\n - fieldPath\n type: object\n x-kubernetes-map-type: atomic\n mode:\n description: |-\n Optional: mode bits used to set permissions on this file, must be an octal value\n between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n If not specified, the volume defaultMode will be used.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n path:\n description: 'Required: Path\n is the relative path name\n of the file to be created.\n Must not be absolute or\n contain the ''..'' path.\n Must be utf-8 encoded. The\n first item of the relative\n path must not start with\n ''..'''\n type: string\n resourceFieldRef:\n description: |-\n Selects a resource of the container: only resources limits and requests\n (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.\n properties:\n containerName:\n description: 'Container\n name: required for volumes,\n optional for env vars'\n type: string\n divisor:\n anyOf:\n - type: integer\n - type: string\n description: Specifies\n the output format of\n the exposed resources,\n defaults to \"1\"\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n resource:\n description: 'Required:\n resource to select'\n type: string\n required:\n - resource\n type: object\n x-kubernetes-map-type: atomic\n required:\n - path\n type: object\n type: array\n x-kubernetes-list-type: atomic\n type: object\n podCertificate:\n description: |-\n Projects an auto-rotating credential bundle (private key and certificate\n chain) that the pod can use either as a TLS client or server.\n\n Kubelet generates a private key and uses it to send a\n PodCertificateRequest to the named signer. Once the signer approves the\n request and issues a certificate chain, Kubelet writes the key and\n certificate chain to the pod filesystem. The pod does not start until\n certificates have been issued for each podCertificate projected volume\n source in its spec.\n\n Kubelet will begin trying to rotate the certificate at the time indicated\n by the signer using the PodCertificateRequest.Status.BeginRefreshAt\n timestamp.\n\n Kubelet can write a single file, indicated by the credentialBundlePath\n field, or separate files, indicated by the keyPath and\n certificateChainPath fields.\n\n The credential bundle is a single file in PEM format. The first PEM\n entry is the private key (in PKCS#8 format), and the remaining PEM\n entries are the certificate chain issued by the signer (typically,\n signers will return their certificate chain in leaf-to-root order).\n\n Prefer using the credential bundle format, since your application code\n can read it atomically. If you use keyPath and certificateChainPath,\n your application must make two separate file reads. If these coincide\n with a certificate rotation, it is possible that the private key and leaf\n certificate you read may not correspond to each other. Your application\n will need to check for this condition, and re-read until they are\n consistent.\n\n The named signer controls chooses the format of the certificate it\n issues; consult the signer implementation's documentation to learn how to\n use the certificates it issues.\n properties:\n certificateChainPath:\n description: |-\n Write the certificate chain at this path in the projected volume.\n\n Most applications should use credentialBundlePath. When using keyPath\n and certificateChainPath, your application needs to check that the key\n and leaf certificate are consistent, because it is possible to read the\n files mid-rotation.\n type: string\n credentialBundlePath:\n description: |-\n Write the credential bundle at this path in the projected volume.\n\n The credential bundle is a single file that contains multiple PEM blocks.\n The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private\n key.\n\n The remaining blocks are CERTIFICATE blocks, containing the issued\n certificate chain from the signer (leaf and any intermediates).\n\n Using credentialBundlePath lets your Pod's application code make a single\n atomic read that retrieves a consistent key and certificate chain. If you\n project them to separate files, your application code will need to\n additionally check that the leaf certificate was issued to the key.\n type: string\n keyPath:\n description: |-\n Write the key at this path in the projected volume.\n\n Most applications should use credentialBundlePath. When using keyPath\n and certificateChainPath, your application needs to check that the key\n and leaf certificate are consistent, because it is possible to read the\n files mid-rotation.\n type: string\n keyType:\n description: |-\n The type of keypair Kubelet will generate for the pod.\n\n Valid values are \"RSA3072\", \"RSA4096\", \"ECDSAP256\", \"ECDSAP384\",\n \"ECDSAP521\", and \"ED25519\".\n type: string\n maxExpirationSeconds:\n description: |-\n maxExpirationSeconds is the maximum lifetime permitted for the\n certificate.\n\n Kubelet copies this value verbatim into the PodCertificateRequests it\n generates for this projection.\n\n If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver\n will reject values shorter than 3600 (1 hour). The maximum allowable\n value is 7862400 (91 days).\n\n The signer implementation is then free to issue a certificate with any\n lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600\n seconds (1 hour). This constraint is enforced by kube-apiserver.\n `kubernetes.io` signers will never issue certificates with a lifetime\n longer than 24 hours.\n format: int32\n type: integer\n signerName:\n description: Kubelet's generated\n CSRs will be addressed to this\n signer.\n type: string\n userAnnotations:\n additionalProperties:\n type: string\n description: |-\n userAnnotations allow pod authors to pass additional information to\n the signer implementation. Kubernetes does not restrict or validate this\n metadata in any way.\n\n These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of\n the PodCertificateRequest objects that Kubelet creates.\n\n Entries are subject to the same validation as object metadata annotations,\n with the addition that all keys must be domain-prefixed. No restrictions\n are placed on values, except an overall size limitation on the entire field.\n\n Signers should document the keys and values they support. Signers should\n deny requests that contain keys they do not recognize.\n type: object\n required:\n - keyType\n - signerName\n type: object\n secret:\n description: secret information about\n the secret data to project\n properties:\n items:\n description: |-\n items if unspecified, each key-value pair in the Data field of the referenced\n Secret will be projected into the volume as a file whose name is the\n key and content is the value. If specified, the listed keys will be\n projected into the specified paths, and unlisted keys will not be\n present. If a key is specified which is not present in the Secret,\n the volume setup will error unless it is marked optional. Paths must be\n relative and may not contain the '..' path or start with '..'.\n items:\n description: Maps a string key\n to a path within a volume.\n properties:\n key:\n description: key is the key\n to project.\n type: string\n mode:\n description: |-\n mode is Optional: mode bits used to set permissions on this file.\n Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n If not specified, the volume defaultMode will be used.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n path:\n description: |-\n path is the relative path of the file to map the key to.\n May not be an absolute path.\n May not contain the path element '..'.\n May not start with the string '..'.\n type: string\n required:\n - key\n - path\n type: object\n type: array\n x-kubernetes-list-type: atomic\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n optional:\n description: optional field specify\n whether the Secret or its key\n must be defined\n type: boolean\n type: object\n x-kubernetes-map-type: atomic\n serviceAccountToken:\n description: serviceAccountToken is\n information about the serviceAccountToken\n data to project\n properties:\n audience:\n description: |-\n audience is the intended audience of the token. A recipient of a token\n must identify itself with an identifier specified in the audience of the\n token, and otherwise should reject the token. The audience defaults to the\n identifier of the apiserver.\n type: string\n expirationSeconds:\n description: |-\n expirationSeconds is the requested duration of validity of the service\n account token. As the token approaches expiration, the kubelet volume\n plugin will proactively rotate the service account token. The kubelet will\n start trying to rotate the token if the token is older than 80 percent of\n its time to live or if the token is older than 24 hours.Defaults to 1 hour\n and must be at least 10 minutes.\n format: int64\n type: integer\n path:\n description: |-\n path is the path relative to the mount point of the file to project the\n token into.\n type: string\n required:\n - path\n type: object\n type: object\n type: array\n x-kubernetes-list-type: atomic\n type: object\n quobyte:\n description: |-\n quobyte represents a Quobyte mount on the host that shares a pod's lifetime.\n Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported.\n properties:\n group:\n description: |-\n group to map volume access to\n Default is no group\n type: string\n readOnly:\n description: |-\n readOnly here will force the Quobyte volume to be mounted with read-only permissions.\n Defaults to false.\n type: boolean\n registry:\n description: |-\n registry represents a single or multiple Quobyte Registry services\n specified as a string as host:port pair (multiple entries are separated with commas)\n which acts as the central registry for volumes\n type: string\n tenant:\n description: |-\n tenant owning the given Quobyte volume in the Backend\n Used with dynamically provisioned Quobyte volumes, value is set by the plugin\n type: string\n user:\n description: |-\n user to map volume access to\n Defaults to serivceaccount user\n type: string\n volume:\n description: volume is a string that references\n an already created Quobyte volume by name.\n type: string\n required:\n - registry\n - volume\n type: object\n rbd:\n description: |-\n rbd represents a Rados Block Device mount on the host that shares a pod's lifetime.\n Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported.\n properties:\n fsType:\n description: |-\n fsType is the filesystem type of the volume that you want to mount.\n Tip: Ensure that the filesystem type is supported by the host operating system.\n Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd\n type: string\n image:\n description: |-\n image is the rados image name.\n More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\n type: string\n keyring:\n default: /etc/ceph/keyring\n description: |-\n keyring is the path to key ring for RBDUser.\n Default is /etc/ceph/keyring.\n More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\n type: string\n monitors:\n description: |-\n monitors is a collection of Ceph monitors.\n More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n pool:\n default: rbd\n description: |-\n pool is the rados pool name.\n Default is rbd.\n More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\n type: string\n readOnly:\n description: |-\n readOnly here will force the ReadOnly setting in VolumeMounts.\n Defaults to false.\n More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\n type: boolean\n secretRef:\n description: |-\n secretRef is name of the authentication secret for RBDUser. If provided\n overrides keyring.\n Default is nil.\n More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n user:\n default: admin\n description: |-\n user is the rados user name.\n Default is admin.\n More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it\n type: string\n required:\n - image\n - monitors\n type: object\n scaleIO:\n description: |-\n scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes.\n Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported.\n properties:\n fsType:\n default: xfs\n description: |-\n fsType is the filesystem type to mount.\n Must be a filesystem type supported by the host operating system.\n Ex. \"ext4\", \"xfs\", \"ntfs\".\n Default is \"xfs\".\n type: string\n gateway:\n description: gateway is the host address of\n the ScaleIO API Gateway.\n type: string\n protectionDomain:\n description: protectionDomain is the name\n of the ScaleIO Protection Domain for the\n configured storage.\n type: string\n readOnly:\n description: |-\n readOnly Defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n type: boolean\n secretRef:\n description: |-\n secretRef references to the secret for ScaleIO user and other\n sensitive information. If this is not provided, Login operation will fail.\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n sslEnabled:\n description: sslEnabled Flag enable/disable\n SSL communication with Gateway, default\n false\n type: boolean\n storageMode:\n default: ThinProvisioned\n description: |-\n storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned.\n Default is ThinProvisioned.\n type: string\n storagePool:\n description: storagePool is the ScaleIO Storage\n Pool associated with the protection domain.\n type: string\n system:\n description: system is the name of the storage\n system as configured in ScaleIO.\n type: string\n volumeName:\n description: |-\n volumeName is the name of a volume already created in the ScaleIO system\n that is associated with this volume source.\n type: string\n required:\n - gateway\n - secretRef\n - system\n type: object\n secret:\n description: |-\n secret represents a secret that should populate this volume.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#secret\n properties:\n defaultMode:\n description: |-\n defaultMode is Optional: mode bits used to set permissions on created files by default.\n Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values\n for mode bits. Defaults to 0644.\n Directories within the path are not affected by this setting.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n items:\n description: |-\n items If unspecified, each key-value pair in the Data field of the referenced\n Secret will be projected into the volume as a file whose name is the\n key and content is the value. If specified, the listed keys will be\n projected into the specified paths, and unlisted keys will not be\n present. If a key is specified which is not present in the Secret,\n the volume setup will error unless it is marked optional. Paths must be\n relative and may not contain the '..' path or start with '..'.\n items:\n description: Maps a string key to a path\n within a volume.\n properties:\n key:\n description: key is the key to project.\n type: string\n mode:\n description: |-\n mode is Optional: mode bits used to set permissions on this file.\n Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511.\n YAML accepts both octal and decimal values, JSON requires decimal values for mode bits.\n If not specified, the volume defaultMode will be used.\n This might be in conflict with other options that affect the file\n mode, like fsGroup, and the result can be other mode bits set.\n format: int32\n type: integer\n path:\n description: |-\n path is the relative path of the file to map the key to.\n May not be an absolute path.\n May not contain the path element '..'.\n May not start with the string '..'.\n type: string\n required:\n - key\n - path\n type: object\n type: array\n x-kubernetes-list-type: atomic\n optional:\n description: optional field specify whether\n the Secret or its keys must be defined\n type: boolean\n secretName:\n description: |-\n secretName is the name of the secret in the pod's namespace to use.\n More info: https://kubernetes.io/docs/concepts/storage/volumes#secret\n type: string\n type: object\n storageos:\n description: |-\n storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes.\n Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported.\n properties:\n fsType:\n description: |-\n fsType is the filesystem type to mount.\n Must be a filesystem type supported by the host operating system.\n Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n type: string\n readOnly:\n description: |-\n readOnly defaults to false (read/write). ReadOnly here will force\n the ReadOnly setting in VolumeMounts.\n type: boolean\n secretRef:\n description: |-\n secretRef specifies the secret to use for obtaining the StorageOS API\n credentials. If not specified, default values will be attempted.\n properties:\n name:\n default: \"\"\n description: |-\n Name of the referent.\n This field is effectively required, but due to backwards compatibility is\n allowed to be empty. Instances of this type with an empty value here are\n almost certainly wrong.\n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n type: string\n type: object\n x-kubernetes-map-type: atomic\n volumeName:\n description: |-\n volumeName is the human-readable name of the StorageOS volume. Volume\n names are only unique within a namespace.\n type: string\n volumeNamespace:\n description: |-\n volumeNamespace specifies the scope of the volume within StorageOS. If no\n namespace is specified then the Pod's namespace will be used. This allows the\n Kubernetes name scoping to be mirrored within StorageOS for tighter integration.\n Set VolumeName to any name to override the default behaviour.\n Set to \"default\" if you are not using namespaces within StorageOS.\n Namespaces that do not pre-exist within StorageOS will be created.\n type: string\n type: object\n vsphereVolume:\n description: |-\n vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine.\n Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type\n are redirected to the csi.vsphere.vmware.com CSI driver.\n properties:\n fsType:\n description: |-\n fsType is filesystem type to mount.\n Must be a filesystem type supported by the host operating system.\n Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.\n type: string\n storagePolicyID:\n description: storagePolicyID is the storage\n Policy Based Management (SPBM) profile ID\n associated with the StoragePolicyName.\n type: string\n storagePolicyName:\n description: storagePolicyName is the storage\n Policy Based Management (SPBM) profile name.\n type: string\n volumePath:\n description: volumePath is the path that identifies\n vSphere volume vmdk\n type: string\n required:\n - volumePath\n type: object\n required:\n - name\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - name\n x-kubernetes-list-type: map\n workloadRef:\n description: |-\n WorkloadRef provides a reference to the Workload object that this Pod belongs to.\n This field is used by the scheduler to identify the PodGroup and apply the\n correct group scheduling policies. The Workload object referenced\n by this field may not exist at the time the Pod is created.\n This field is immutable, but a Workload object with the same name\n may be recreated with different policies. Doing this during pod scheduling\n may result in the placement not conforming to the expected policies.\n properties:\n name:\n description: |-\n Name defines the name of the Workload object this Pod belongs to.\n Workload must be in the same namespace as the Pod.\n If it doesn't match any existing Workload, the Pod will remain unschedulable\n until a Workload object is created and observed by the kube-scheduler.\n It must be a DNS subdomain.\n type: string\n podGroup:\n description: |-\n PodGroup is the name of the PodGroup within the Workload that this Pod\n belongs to. If it doesn't match any existing PodGroup within the Workload,\n the Pod will remain unschedulable until the Workload object is recreated\n and observed by the kube-scheduler. It must be a DNS label.\n type: string\n podGroupReplicaKey:\n description: |-\n PodGroupReplicaKey specifies the replica key of the PodGroup to which this\n Pod belongs. It is used to distinguish pods belonging to different replicas\n of the same pod group. The pod group policy is applied separately to each replica.\n When set, it must be a DNS label.\n type: string\n required:\n - name\n - podGroup\n type: object\n required:\n - containers\n type: object\n type: object\n type: object\n description: |-\n MPIReplicaSpecs contains maps from `MPIReplicaType` to `ReplicaSpec` that\n specify the MPI replicas to run.\n type: object\n runLauncherAsWorker:\n default: false\n description: |-\n RunLauncherAsWorker indicates whether to run worker process in launcher\n Defaults to false.\n type: boolean\n runPolicy:\n description: RunPolicy encapsulates various runtime policies of the\n job.\n properties:\n activeDeadlineSeconds:\n description: |-\n Specifies the duration in seconds relative to the startTime that the job may be active\n before the system tries to terminate it; value must be positive integer.\n format: int64\n type: integer\n backoffLimit:\n description: Optional number of retries before marking this job\n failed.\n format: int32\n type: integer\n cleanPodPolicy:\n description: |-\n CleanPodPolicy defines the policy to kill pods after the job completes.\n Default to Running.\n type: string\n managedBy:\n description: |-\n ManagedBy is used to indicate the controller or entity that manages a MPIJob.\n The value must be either empty, 'kubeflow.org/mpi-operator' or\n 'kueue.x-k8s.io/multikueue'.\n The mpi-operator reconciles a MPIJob which doesn't have this\n field at all or the field value is the reserved string\n 'kubeflow.org/mpi-operator', but delegates reconciling the MPIJob\n with 'kueue.x-k8s.io/multikueue' to the Kueue.\n The field is immutable.\n type: string\n schedulingPolicy:\n description: SchedulingPolicy defines the policy related to scheduling,\n e.g. gang-scheduling\n properties:\n minAvailable:\n description: |-\n MinAvailable defines the minimal number of member to run the PodGroup.\n If the gang-scheduling isn't empty, input is passed to `.spec.minMember` in PodGroup.\n Note that, when using this field,\n you need to make sure the application supports resizing (e.g., Elastic Horovod).\n\n If not set, it defaults to the number of workers.\n format: int32\n type: integer\n minResources:\n additionalProperties:\n anyOf:\n - type: integer\n - type: string\n pattern: ^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$\n x-kubernetes-int-or-string: true\n description: |-\n MinResources defines the minimal resources of members to run the PodGroup.\n If the gang-scheduling isn't empty,\n input is passed to `.spec.minResources` in PodGroup for scheduler-plugins.\n type: object\n priorityClass:\n description: |-\n PriorityClass defines the PodGroup's PriorityClass.\n If the gang-scheduling is set to the volcano,\n input is passed to `.spec.priorityClassName` in PodGroup for volcano,\n and if it is set to the scheduler-plugins,\n input isn't passed to PodGroup for scheduler-plugins.\n type: string\n queue:\n description: |-\n Queue defines the queue name to allocate resource for PodGroup.\n If the gang-scheduling is set to the volcano,\n input is passed to `.spec.queue` in PodGroup for the volcano,\n and if it is set to the scheduler-plugins,\n input isn't passed to PodGroup.\n type: string\n scheduleTimeoutSeconds:\n description: |-\n SchedulerTimeoutSeconds defines the maximal time of members to wait before run the PodGroup.\n If the gang-scheduling is set to the scheduler-plugins,\n input is passed to `.spec.scheduleTimeoutSeconds` in PodGroup for the scheduler-plugins,\n and if it is set to the volcano, input isn't passed to PodGroup.\n format: int32\n type: integer\n type: object\n suspend:\n default: false\n description: |-\n suspend specifies whether the MPIJob controller should create Pods or not.\n If a MPIJob is created with suspend set to true, no Pods are created by\n the MPIJob controller. If a MPIJob is suspended after creation (i.e. the\n flag goes from false to true), the MPIJob controller will delete all\n active Pods and PodGroups associated with this MPIJob. Also, it will suspend the\n Launcher Job. Users must design their workload to gracefully handle this.\n Suspending a Job will reset the StartTime field of the MPIJob.\n\n Defaults to false.\n type: boolean\n ttlSecondsAfterFinished:\n description: |-\n TTLSecondsAfterFinished is the TTL to clean up jobs.\n It may take extra ReconcilePeriod seconds for the cleanup, since\n reconcile gets called periodically.\n Default to infinite.\n format: int32\n type: integer\n type: object\n slotsPerWorker:\n default: 1\n description: |-\n Specifies the number of slots per worker used in hostfile.\n Defaults to 1.\n format: int32\n type: integer\n sshAuthMountPath:\n default: /root/.ssh\n description: |-\n SSHAuthMountPath is the directory where SSH keys are mounted.\n Defaults to \"/root/.ssh\".\n type: string\n required:\n - mpiReplicaSpecs\n type: object\n status:\n description: JobStatus represents the current observed state of the training\n Job.\n properties:\n completionTime:\n description: |-\n Represents time when the job was completed. It is not guaranteed to\n be set in happens-before order across separate operations.\n It is represented in RFC3339 form and is in UTC.\n format: date-time\n type: string\n conditions:\n description: conditions is a list of current observed job conditions.\n items:\n description: JobCondition describes the state of the job at a certain\n point.\n properties:\n lastTransitionTime:\n description: Last time the condition transitioned from one status\n to another.\n format: date-time\n type: string\n lastUpdateTime:\n description: The last time this condition was updated.\n format: date-time\n type: string\n message:\n description: A human-readable message indicating details about\n the transition.\n type: string\n reason:\n description: The reason for the condition's last transition.\n type: string\n status:\n description: status of the condition, one of True, False, Unknown.\n enum:\n - \"True\"\n - \"False\"\n - Unknown\n type: string\n type:\n description: type of job condition.\n type: string\n required:\n - status\n - type\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - type\n x-kubernetes-list-type: map\n lastReconcileTime:\n description: |-\n Represents last time when the job was reconciled. It is not guaranteed to\n be set in happens-before order across separate operations.\n It is represented in RFC3339 form and is in UTC.\n format: date-time\n type: string\n replicaStatuses:\n additionalProperties:\n description: ReplicaStatus represents the current observed state\n of the replica.\n properties:\n active:\n description: The number of actively running pods.\n format: int32\n type: integer\n failed:\n description: The number of pods which reached phase failed.\n format: int32\n type: integer\n labelSelector:\n description: 'Deprecated: Use selector instead'\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n x-kubernetes-list-type: atomic\n required:\n - key\n - operator\n type: object\n type: array\n x-kubernetes-list-type: atomic\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n selector:\n description: |-\n A selector is a label query over a set of resources. The result of matchLabels and\n matchExpressions are ANDed. An empty selector matches all objects. A null\n selector matches no objects.\n type: string\n succeeded:\n description: The number of pods which reached phase succeeded.\n format: int32\n type: integer\n type: object\n description: |-\n replicaStatuses is map of ReplicaType and ReplicaStatus,\n specifies the status of each replica.\n type: object\n startTime:\n description: |-\n Represents time when the job was acknowledged by the job controller.\n It is not guaranteed to be set in happens-before order across separate operations.\n It is represented in RFC3339 form and is in UTC.\n format: date-time\n type: string\n type: object\n type: object\n served: true\n storage: true\n subresources:\n status: {}\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n name: mpi-operator\n namespace: mpi-operator\n---\naggregationRule:\n clusterRoleSelectors:\n - matchLabels:\n rbac.authorization.kubeflow.org/aggregate-to-kubeflow-mpijobs-admin: \"true\"\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: \"true\"\n name: kubeflow-mpijobs-admin\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: \"true\"\n rbac.authorization.kubeflow.org/aggregate-to-kubeflow-mpijobs-admin: \"true\"\n name: kubeflow-mpijobs-edit\nrules:\n- apiGroups:\n - kubeflow.org\n resources:\n - mpijobs\n - mpijobs/status\n verbs:\n - get\n - list\n - watch\n - create\n - delete\n - deletecollection\n - patch\n - update\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: \"true\"\n name: kubeflow-mpijobs-view\nrules:\n- apiGroups:\n - kubeflow.org\n resources:\n - mpijobs\n - mpijobs/status\n verbs:\n - get\n - list\n - watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n name: mpi-operator\nrules:\n- apiGroups:\n - \"\"\n resources:\n - configmaps\n - secrets\n - services\n verbs:\n - create\n - list\n - watch\n - update\n- apiGroups:\n - \"\"\n resources:\n - pods\n verbs:\n - create\n - get\n - list\n - watch\n - delete\n - update\n - patch\n- apiGroups:\n - \"\"\n resources:\n - pods/exec\n verbs:\n - create\n- apiGroups:\n - \"\"\n resources:\n - endpoints\n verbs:\n - create\n - get\n - update\n- apiGroups:\n - \"\"\n resources:\n - events\n verbs:\n - create\n - patch\n- apiGroups:\n - batch\n resources:\n - jobs\n verbs:\n - create\n - list\n - update\n - watch\n- apiGroups:\n - apiextensions.k8s.io\n resources:\n - customresourcedefinitions\n verbs:\n - create\n - get\n- apiGroups:\n - kubeflow.org\n resources:\n - mpijobs\n - mpijobs/finalizers\n - mpijobs/status\n verbs:\n - '*'\n- apiGroups:\n - coordination.k8s.io\n resources:\n - leases\n verbs:\n - '*'\n- apiGroups:\n - scheduling.incubator.k8s.io\n - scheduling.sigs.dev\n - scheduling.volcano.sh\n resources:\n - queues\n - podgroups\n verbs:\n - '*'\n- apiGroups:\n - scheduling.x-k8s.io\n resources:\n - podgroups\n verbs:\n - '*'\n- apiGroups:\n - scheduling.k8s.io\n resources:\n - priorityclasses\n verbs:\n - get\n - list\n - watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n name: mpi-operator\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: mpi-operator\nsubjects:\n- kind: ServiceAccount\n name: mpi-operator\n namespace: mpi-operator\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n name: mpi-operator\n namespace: mpi-operator\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n template:\n metadata:\n annotations:\n sidecar.istio.io/inject: \"false\"\n labels:\n app: mpi-operator\n app.kubernetes.io/component: mpijob\n app.kubernetes.io/name: mpi-operator\n kustomize.component: mpi-operator\n spec:\n containers:\n - args:\n - -alsologtostderr\n - --lock-namespace=mpi-operator\n image: mpioperator/mpi-operator:0.8.0\n name: mpi-operator\n serviceAccountName: mpi-operator\n" }, { "path": "test/manifests/assets/nvidia-device-plugin.yaml", "content": "# Source: https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/main/deployments/static/nvidia-device-plugin.yml\n\n# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: nvidia-device-plugin-daemonset\n namespace: kube-system\nspec:\n selector:\n matchLabels:\n name: nvidia-device-plugin-ds\n updateStrategy:\n type: RollingUpdate\n template:\n metadata:\n labels:\n name: nvidia-device-plugin-ds\n spec:\n tolerations:\n - key: nvidia.com/gpu\n operator: Exists\n effect: NoSchedule\n # Mark this pod as a critical add-on; when enabled, the critical add-on\n # scheduler reserves resources for critical add-on pods so that they can\n # be rescheduled after a failure.\n # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/\n priorityClassName: \"system-node-critical\"\n containers:\n - image: nvcr.io/nvidia/k8s-device-plugin:v0.17.2\n name: nvidia-device-plugin-ctr\n env:\n - name: FAIL_ON_INIT_ERROR\n value: \"false\"\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop: [\"ALL\"]\n volumeMounts:\n - name: device-plugin\n mountPath: /var/lib/kubelet/device-plugins\n volumes:\n - name: device-plugin\n hostPath:\n path: /var/lib/kubelet/device-plugins\n" }, { "path": "test/manifests/raw.go", "content": "package manifests\n\nimport (\n\t_ \"embed\"\n)\n\nvar (\n\t//go:embed assets/nvidia-device-plugin.yaml\n\tNvidiaDevicePluginManifest []byte\n\t//go:embed assets/mpi-operator.yaml\n\tMpiOperatorManifest []byte\n\n\t//go:embed assets/efa-device-plugin.yaml\n\tEfaDevicePluginManifest []byte\n\n\t//go:embed assets/k8s-neuron-device-plugin-rbac.yml\n\tNeuronDevicePluginRbacManifest []byte\n\t//go:embed assets/k8s-neuron-device-plugin.yml\n\tNeuronDevicePluginManifest []byte\n\n\t//go:embed assets/dranet.yaml\n\tDranetManifest []byte\n\n\t//go:embed assets/dcgm-exporter.yaml\n\tDCGMExporterManifest []byte\n\n\t//go:embed assets/cloudwatch-agent.yaml\n\tcloudWatchAgentManifestTemplate []byte\n)\n" }, { "path": "test/manifests/rendered.go", "content": "package manifests\n\nimport (\n\t\"html/template\"\n\t\"strings\"\n\n\tfwext \"github.com/aws/aws-k8s-tester/internal/e2e\"\n)\n\n// RenderCloudWatchAgentManifest renders the CloudWatch Agent manifest with dynamic dimensions\nfunc RenderCloudWatchAgentManifest(metricDimensions map[string]string) ([]byte, error) {\n\tvar keys []string\n\tfor key := range metricDimensions {\n\t\tkeys = append(keys, `\"`+key+`\"`)\n\t}\n\tdimensionsStr := strings.Join(keys, \", \")\n\treturn fwext.RenderManifests(cloudWatchAgentManifestTemplate, map[string]interface{}{\n\t\t\"MetricDimensions\": metricDimensions,\n\t\t\"DimensionKeys\": template.HTML(dimensionsStr),\n\t})\n}\n" } ]