[ { "path": ".github/.jira_sync_config.yaml", "content": "settings:\n # Jira project key to create the issue in\n jira_project_key: \"KU\"\n\n # Dictionary mapping GitHub issue status to Jira issue status\n status_mapping:\n opened: Untriaged\n closed: done\n\n # (Optional) Jira project components that should be attached to the created issue\n # Component names are case-sensitive\n components:\n - Microk8s snap\n\n # (Optional) GitHub labels. Only issues with one of those labels will be synchronized.\n # If not specified, all issues will be synchronized\n # labels: []\n\n # (Optional) (Default: false) Add a new comment in GitHub with a link to Jira created issue\n add_gh_comment: false\n\n # (Optional) (Default: true) Synchronize issue description from GitHub to Jira\n sync_description: true\n\n # (Optional) (Default: true) Synchronize comments from GitHub to Jira\n sync_comments: true\n\n # (Optional) (Default: None) Parent Epic key to link the issue to\n epic_key: \"KU-925\"\n\n # (Optional) Dictionary mapping GitHub issue labels to Jira issue types.\n # If label on the issue is not in specified list, this issue will be created as a Bug\n label_mapping:\n enhancement: Story\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug Report\nabout: Something is not working\n---\n\n\n\n#### Summary\n\n\n#### What Should Happen Instead?\n\n\n#### Reproduction Steps\n\n\n1. ...\n2. ...\n\n#### Introspection Report\n\n\n#### Can you suggest a fix?\n\n\n#### Are you interested in contributing with a fix?\n\n\n\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature Request\nabout: Suggest a new feature\n---\n\n\n\n#### Summary\n\n\n#### Why is this important?\n\n\n#### Are you interested in contributing to this feature?\n\n\n\n\n" }, { "path": ".github/ISSUE_TEMPLATE/question.yml", "content": "contact_links:\n - name: Ask a question\n url: https://kubernetes.slack.com/archives/CAUNWQ85V\n about: \"For discussions and/or other questions related to MicroK8s, please use the #microk8s channel on the Kubernetes Slack\"\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "\n\n#### Summary\n\n\n#### Changes\n\n\n#### Testing\n\n\n#### Possible Regressions\n\n\n#### Checklist\n\n\n* [ ] Read the [contributions](https://github.com/canonical/microk8s/blob/master/CONTRIBUTING.md) page.\n* [ ] Submitted the [CLA form](https://ubuntu.com/legal/contributors/agreement), if you are a first time contributor.\n* [ ] The introduced changes are covered by unit and/or integration tests.\n\n#### Notes\n\n" }, { "path": ".github/actions/test-prep/action.yaml", "content": "name: Prepare test prerequisites\n\nruns:\n using: \"composite\"\n steps:\n - name: Setup Python\n uses: actions/setup-python@v5\n with:\n python-version: \"3.10\"\n - name: Install test dependencies\n shell: bash\n run: |\n set -x\n sudo pip3 install -r ./tests/requirements.txt\n # Docker sets iptables rules that interfere with LXD and K8s.\n # https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker\n - name: Apply Docker iptables workaround\n shell: bash\n run: sudo iptables -I DOCKER-USER -j ACCEPT\n - name: Fetch snap\n uses: actions/download-artifact@v4\n with:\n name: microk8s.snap\n path: build\n" }, { "path": ".github/dependabot.yml", "content": "# Set update schedule for GitHub Actions\n\nversion: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n # Check for updates to GitHub Actions every weekday\n interval: \"daily\"\n" }, { "path": ".github/workflows/backport.yml", "content": "name: Backport merged pull request\non:\n pull_request_target:\n types: [closed]\n issue_comment:\n types: [created]\npermissions:\n contents: write # so it can comment\n pull-requests: write # so it can create pull requests\njobs:\n backport:\n name: Backport pull request\n runs-on: ubuntu-latest\n\n # Only run when pull request is merged\n # or when a comment containing `/backport` is created by someone other than the\n # https://github.com/backport-action bot user (user id: 97796249). Note that if you use your\n # own PAT as `github_token`, that you should replace this id with yours.\n # cdkbot's user ID is 99445902.\n if: >\n (\n github.event_name == 'pull_request_target' &&\n github.event.pull_request.merged\n ) || (\n github.event_name == 'issue_comment' &&\n github.event.issue.pull_request &&\n github.event.comment.user.id != 99445902 &&\n contains(github.event.comment.body, '/backport')\n )\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n - name: Create backport pull requests\n uses: korthout/backport-action@v3\n with:\n # Set (default) action parameters explicitly.\n branch_name: backport-${pull_number}-to-${target_branch}\n cherry_picking: auto\n copy_assignees: false\n copy_milestone: false\n copy_requested_reviewers: false\n experimental: >\n {\n \"conflict_resolution\": \"fail\"\n }\n github_token: ${{ secrets.BOT_TOKEN }}\n github_workspace: ${{ github.workspace }}\n label_pattern: ^backport ([^ ]+)$\n merge_commits: fail\n pull_description: |-\n # Description\n Backport of #${pull_number} to `${target_branch}`.\n pull_title: >-\n [Backport ${target_branch}] ${pull_title}\n" }, { "path": ".github/workflows/build-installer.yml", "content": "name: Build MicroK8s Installers\n\non:\n push:\n branches:\n - \"**install**\"\n pull_request:\n branches:\n - \"**install**\"\n\njobs:\n windows:\n runs-on: windows-latest\n defaults:\n run:\n working-directory: ${{ github.workspace }}/installer/windows\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n - name: Set up Python 3.8\n uses: actions/setup-python@v5.5.0\n with:\n python-version: 3.8\n - name: Install Python requirements\n run: python -m pip install -r ../requirements.txt\n - name: Build exe\n working-directory: ${{ github.workspace }}/installer\n run: pyinstaller.exe ./microk8s.spec\n - name: Move exe to installer build directory\n working-directory: ${{ github.workspace }}/installer\n run: move microk8s.exe ./windows/microk8s.exe\n - name: Download EnVar plugin for NSIS\n uses: carlosperate/download-file-action@v2.0.2\n with:\n file-url: https://github.com/GsNSIS/EnVar/releases/download/v0.3.1/EnVar-Plugin.zip\n file-name: envar_plugin.zip\n location: ${{ github.workspace }}\n - name: Extract EnVar plugin\n run: 7z x -o\"C:/Program Files (x86)/NSIS\" \"${{ github.workspace }}/envar_plugin.zip\"\n - name: Download Multipass installer\n uses: carlosperate/download-file-action@v2.0.2\n with:\n file-url: https://github.com/canonical/multipass/releases/download/v1.12.2/multipass-1.12.2+win-win64.exe\n file-name: multipass.exe\n location: ${{ github.workspace }}/installer/windows\n - name: Download kubectl\n uses: carlosperate/download-file-action@v2.0.2\n with:\n file-url: https://storage.googleapis.com/kubernetes-release/release/v1.28.3/bin/windows/amd64/kubectl.exe\n file-name: kubectl.exe\n location: ${{ github.workspace }}/installer/windows\n - name: Create installer\n run: makensis.exe ${{ github.workspace }}/installer/windows/microk8s.nsi\n - name: Upload installer\n uses: actions/upload-artifact@v4\n with:\n name: Windows installer\n path: ${{ github.workspace }}/installer/windows/microk8s-installer.exe\n" }, { "path": ".github/workflows/build-snap.yml", "content": "name: Build and test MicroK8s snap\n\non:\n pull_request:\n branches:\n - master\n\njobs:\n build:\n name: Create snap package\n runs-on: ubuntu-latest\n\n steps:\n - name: Checking out repo\n uses: actions/checkout@v4\n - name: Install lxd\n run: |\n sudo lxd init --auto\n sudo usermod --append --groups lxd $USER\n # `newgrp` does not work in GitHub Actions; use `sudo --user` instead\n # See https://github.com/actions/runner-images/issues/9932#issuecomment-2573170305\n sudo --user \"$USER\" --preserve-env --preserve-env=PATH -- env -- lxc version\n # Docker sets iptables rules that interfere with LXD or K8s.\n # https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker\n - name: Apply Docker iptables workaround\n shell: bash\n run: sudo iptables -I DOCKER-USER -j ACCEPT\n - name: Install snapcraft\n run: |\n sudo snap install snapcraft --classic\n - name: Install snapd from candidate\n run: |\n # TODO(neoaggelos): revert this after latest/beta is working again\n sudo snap refresh snapd --channel=latest/stable\n - name: Build snap\n run: |\n sudo --user \"$USER\" --preserve-env --preserve-env=PATH -- env -- snapcraft --use-lxd\n sudo mv microk8s*.snap microk8s.snap\n - name: Uploading snap\n uses: actions/upload-artifact@v4\n with:\n name: microk8s.snap\n path: microk8s.snap\n\n test-upgrade:\n name: Upgrade path test\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 30\n steps:\n - name: Checking out repo\n uses: actions/checkout@v4\n - name: Prepare test prerequisites\n uses: ./.github/actions/test-prep\n - name: Running upgrade path test\n run: |\n sudo -E UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade-path.py\n\n test-addons-core:\n name: Test core addons\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 60\n env:\n # Avoid truncated \"ps\" output\n COLUMNS: 2048\n steps:\n - name: Checking out repo\n uses: actions/checkout@v4\n - name: Prepare test prerequisites\n uses: ./.github/actions/test-prep\n - name: Running addons tests\n env:\n UNDER_TIME_PRESSURE: ${{ !contains(github.event.pull_request.labels.*.name, 'run-all-tests') }}\n run: |\n set -x\n sudo snap install build/microk8s.snap --classic --dangerous\n ./tests/smoke-test.sh\n # The GitHub runner is using the 10.1.0.0/16 CIDR, which would conflict with\n # kube-ovn's default POD_CIDR. They have to be different.\n export POD_CIDR=\"10.200.0.0/16\"\n export POD_GATEWAY=\"10.200.0.1\"\n export SKIP_PROMETHEUS=\"False\"\n export UNDER_TIME_PRESSURE=${UNDER_TIME_PRESSURE@u}\n sudo -E bash -c \"cd /var/snap/microk8s/common/addons/core/tests; pytest -s -ra test-addons.py\"\n\n test-addons-community:\n name: Test community addons\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 60\n steps:\n - name: Checking out repo\n uses: actions/checkout@v4\n - name: Prepare test prerequisites\n uses: ./.github/actions/test-prep\n - name: Running addons tests\n env:\n UNDER_TIME_PRESSURE: ${{ !contains(github.event.pull_request.labels.*.name, 'run-all-tests') }}\n run: |\n set -x\n sudo snap install build/microk8s.snap --classic --dangerous\n sudo microk8s enable community\n export UNDER_TIME_PRESSURE=${UNDER_TIME_PRESSURE@u}\n sudo -E bash -c \"cd /var/snap/microk8s/common/addons/community/; pytest -s -ra ./tests/\"\n\n test-addons-core-upgrade:\n name: Test core addons upgrade\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 30\n steps:\n - name: Checking out repo\n uses: actions/checkout@v4\n - name: Prepare test prerequisites\n uses: ./.github/actions/test-prep\n - name: Running upgrade tests\n env:\n UNDER_TIME_PRESSURE: ${{ !contains(github.event.pull_request.labels.*.name, 'run-all-tests') }}\n run: |\n set -x\n export UNDER_TIME_PRESSURE=${UNDER_TIME_PRESSURE@u}\n sudo -E bash -c \"UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=$PWD/build/microk8s.snap pytest -s ./tests/test-upgrade.py\"\n\n test-cluster-agent:\n name: Cluster agent health check\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 30\n steps:\n - name: Checking out repo\n uses: actions/checkout@v4\n - name: Prepare test prerequisites\n uses: ./.github/actions/test-prep\n - name: Running cluster agent health check\n run: |\n set -x\n sudo snap install build/microk8s.snap --classic --dangerous\n sudo -E bash -c \"pytest -s ./tests/test-cluster-agent.py\"\n\n test-airgap:\n name: Test airgap installation\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 30\n steps:\n - name: Checking out repo\n uses: actions/checkout@v4\n - name: Prepare test prerequisites\n uses: ./.github/actions/test-prep\n - name: Initialize LXD\n run: |\n sudo lxd init --auto\n sudo lxc network set lxdbr0 ipv6.address=none\n sudo usermod --append --groups lxd $USER\n # `newgrp` does not work in GitHub Actions; use `sudo --user` instead\n # See https://github.com/actions/runner-images/issues/9932#issuecomment-2573170305\n sudo --user \"$USER\" --preserve-env --preserve-env=PATH -- env -- lxc version\n - name: Run airgap tests\n run: |\n sudo -E bash -x -c \"./tests/libs/airgap.sh --distro ubuntu:22.04 --channel $PWD/build/microk8s.snap\"\n\n security-scan:\n name: Security scan\n outputs:\n sarif_files: ${{ steps.get_sarif_files.outputs.sarif-files }}\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 30\n steps:\n - name: Checking out repo\n uses: actions/checkout@v4\n - name: Fetch snap\n uses: actions/download-artifact@v4\n with:\n name: microk8s.snap\n path: build\n - name: Create sarifs directory\n run: |\n mkdir -p sarifs\n - name: Install Trivy vulnerability scanner\n uses: aquasecurity/setup-trivy@v0.2.2\n - name: Run Trivy vulnerability scanner on codebase\n run: |\n trivy fs . --format sarif --severity CRITICAL > sarifs/trivy-microk8s-repo-scan--results.sarif\n - name: Run Trivy vulnerability scanner on images\n run: |\n for i in $(cat ./build-scripts/images.txt) ; do\n name=$(echo $i | awk -F ':|/' '{print $(NF-1)}')\n trivy image $i --format sarif > sarifs/$name.sarif\n done\n - name: Run Trivy vulnerability scanner on the snap\n run: |\n cp build/microk8s.snap .\n unsquashfs microk8s.snap\n trivy rootfs ./squashfs-root/ --format sarif > sarifs/snap.sarif\n - name: Generate list of SARIF files\n id: get_sarif_files\n run: |\n sarif_files=$(find sarifs -name \"*.sarif\" -printf \"%P\\n\" | jq -R -s -c 'split(\"\\n\") | map(select(length > 0))')\n echo \"sarif-files=$sarif_files\" >> \"$GITHUB_OUTPUT\"\n - name: Upload SARIF files artifact\n uses: actions/upload-artifact@v4\n with:\n name: sarifs\n path: sarifs\n retention-days: 1\n\n upload_sarifs_matrix:\n needs: security-scan\n runs-on: ubuntu-latest\n strategy:\n fail-fast: true\n matrix:\n sarif_file_path: ${{ fromJson(needs.security-scan.outputs.sarif_files) }}\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n - name: Download SARIF files artifact\n uses: actions/download-artifact@v4\n with:\n name: sarifs\n path: sarifs\n - name: Prepare SARIF category\n id: prepare_category\n run: |\n sarif_file=\"${{ matrix.sarif_file_path }}\"\n base_name=$(basename \"$sarif_file\" .sarif)\n echo \"category=$base_name\" >> \"$GITHUB_OUTPUT\"\n - name: Upload SARIF file\n uses: github/codeql-action/upload-sarif@v3\n with:\n sarif_file: sarifs/${{ matrix.sarif_file_path }}\n category: ${{ steps.prepare_category.outputs.category }}\n" }, { "path": ".github/workflows/check-formatting.yml", "content": "name: Lint Code\n\non:\n - pull_request\n\njobs:\n check-formatting:\n name: Check Formatting\n runs-on: ubuntu-latest\n\n steps:\n - name: Check out code\n uses: actions/checkout@v4\n\n - name: Install dependencies\n run: |\n sudo apt-get update\n sudo apt-get install tox --fix-missing\n sudo snap install node --classic\n sudo npm install --save-dev --save-exact -g prettier\n - name: Check Python formatting\n run: |\n tox -e lint\n - name: Check YAML formatting\n run: |\n set -eux\n prettier --check $(find . -name \"*.yaml\" -o -name \"*.yml\" | \\\n grep -v \"./microk8s-resources/actions/ingress.yaml\" | \\\n grep -v \"./microk8s-resources/actions/metallb.yaml\" | \\\n grep -v invalid.yaml | \\\n grep -v calico)\n" }, { "path": ".github/workflows/check-unit-tests.yml", "content": "name: Unit Tests\n\non:\n - pull_request\n\njobs:\n check-unit-tests:\n name: Check Unit Tests\n runs-on: ubuntu-latest\n\n steps:\n - name: Check out code\n uses: actions/checkout@v4\n\n - name: Install dependencies\n run: |\n sudo apt-get update\n sudo apt-get install tox --fix-missing\n sudo pip3 install -U pytest==7.1.3 sh==1.14.3\n - name: Check Units\n run: |\n tox -e scripts\n tox -e wrappers\n tox -e cluster\n - name: Verify branches\n run: |\n pytest -s ./tests/verify-branches.py\n" }, { "path": ".github/workflows/cla-check.yml", "content": "name: cla-check\n\non:\n pull_request:\n branches: [master, default]\n\njobs:\n cla-check:\n runs-on: ubuntu-latest\n steps:\n - name: Check if CLA signed\n uses: canonical/has-signed-canonical-cla@v2\n" }, { "path": ".github/workflows/stale-cron.yaml", "content": "name: Close inactive issues or PRs\non:\n schedule:\n - cron: \"0 0 * * *\" # Runs every midnight\n pull_request:\n paths:\n - .github/workflows/stale-cron.yaml\n\njobs:\n close-issues:\n runs-on: ubuntu-latest\n permissions:\n issues: write\n pull-requests: write\n steps:\n - uses: actions/stale@v9\n with:\n days-before-stale: 330\n days-before-close: 30\n stale-issue-label: inactive\n stale-issue-message: >\n This issue has been automatically marked as stale because it has\n not had recent activity. It will be closed if no further activity\n occurs. Thank you for your contributions.\n close-issue-message: >\n This issue was closed because it has been inactive for 30 days\n since being marked as stale.\n exempt-pr-labels: pinned,security\n exempt-issue-labels: pinned,security\n repo-token: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".github/workflows/update-images.yml", "content": "name: Update list of images\n\non:\n workflow_dispatch:\n schedule:\n - cron: \"0 10 * * *\"\n\npermissions:\n contents: write\n pull-requests: write\n\njobs:\n update:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n include:\n # Latest branches\n - { branch: master, channel: latest/edge }\n # Stable branches\n - { branch: \"1.32\", channel: \"1.32\" }\n - { branch: \"1.31\", channel: \"1.31\" }\n - { branch: \"1.30\", channel: \"1.30\" }\n - { branch: \"1.29\", channel: \"1.29\" }\n - { branch: \"1.28\", channel: \"1.28\" }\n - { branch: \"1.27\", channel: \"1.27\" }\n # Stable strict branches\n - { branch: 1.32-strict, channel: 1.32-strict }\n - { branch: 1.31-strict, channel: 1.31-strict }\n - { branch: 1.30-strict, channel: 1.30-strict }\n - { branch: 1.29-strict, channel: 1.29-strict }\n - { branch: 1.28-strict, channel: 1.28-strict }\n - { branch: 1.27-strict, channel: 1.27-strict }\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n with:\n ref: ${{ matrix.branch }}\n - name: Update image list\n run: |\n ./build-scripts/update-images.sh ${{ matrix.channel }} build-scripts/images.txt\n - name: Create pull request\n uses: peter-evans/create-pull-request@v7\n with:\n commit-message: update list of images used by ${{ matrix.channel }}\n title: \"[${{ matrix.channel }}] Update MicroK8s images\"\n body: update list of images used by ${{ matrix.channel }}\n reviewers: berkayoz,ktsakalozos\n branch: auto-update-images/${{ matrix.branch }}\n delete-branch: true\n base: ${{ matrix.branch }}\n" }, { "path": ".gitignore", "content": "/build/\n/parts/\n/prime/\n/stage/\n!/snap/hooks/\n*.snap\nmicrok8s.egg-info/\nmicrok8s_source.tar.bz2\ntests/__pycache__\n.idea/\n**/tests/*.pyc\n.venv/\n.vscode/\n/installer/.venv/\n/installer/**/__pycache__\n/installer/dist/\n/installer/build/\n.tox/\n.tox_env/\n__pycache__/\nmicrok8s_*.txt #Remote build log\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "MicroK8s has adopted the [Ubuntu Code of Conduct v2.0](https://ubuntu.com/community/ethos/code-of-conduct)\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributor Guide\n\nMicroK8s is open source ([Apache License 2.0](./LICENSE)) and actively seeks any community contributions for code, add-ons, suggestions and documentation.\nMany of the features currently part of MicroK8s originated in the community, and we are very keen for that to continue. This page details a few notes, \nworkflows and suggestions for how to make contributions most effective and help us all build a better MicroK8s for everyone - please give them a read before\nworking on any contributions.\n\n## Licensing\n\nMicroK8s has been created under the [Apache License 2.0](./LICENSE), which will cover any contributions you may make to this project. Please familiarise\nyourself with the terms of the license.\n\nAdditionally, MicroK8s uses the Harmony CLA agreement. It’s the easiest way for you to give us permission to use your contributions. \nIn effect, you’re giving us a licence, but you still own the copyright — so you retain the right to modify your code and use it in\nother projects. Please [sign the CLA here](https://ubuntu.com/legal/contributors/agreement) before making any contributions.\n\n## Code of conduct\n\nMicroK8s has adopted the Ubuntu code of Conduct. You can read this in full [here](https://ubuntu.com/community/code-of-conduct).\n\n## Contributing code\n\nThe workflow for contributing code is as follows:\n\n1. **Create/choose an issue**: MicroK8s tracks issues at [https://github.com/canonical/microk8s/issues](https://github.com/canonical/microk8s/issues). If you\n want to work on a new feature, create an issue first! This gives everyone a place to discuss scope and implementation.\n2. **Create a fork of the MicroK8s repo**\n3. **Make a new branch for your contribution**. Write your code there.\n4. For details on how to **build and test MicroK8s**, see the [build instructions](./docs/build.md). Add new tests as needed,\n and make sure the existing tests continue to pass when your changes are complete.\n5. **Submit a pull request** to get changes from your branch into master. You can add \"#Fixes xxx\" where `xxx` is the issue number to \n automatically link to the issue you chose or created earlier.\n6. Someone will review your PR and may make suggestions or have comments, so **keep an eye on the PR status** in case there are changes to make\n7. **Please make sure you have submitted your [CLA form](https://ubuntu.com/legal/contributors/agreement) if you are a first time contributor**.\n8. Thanks!\n\n## Documentation\n\nDocs for MicroK8s are published online at [https://microk8s.io/docs](https://microk8s.io/docs). You can make suggestions and edit the pages themselves by joining \nthe Kubernetes discourse at [discuss.kubernetes.io](https://discuss.kubernetes.io/t/introduction-to-microk8s/11243) or follow the link at\nthe bottom of any of the pages published at [https://microk8s.io/docs](https://microk8s.io/docs)\nThere is a documentation page which describes how to write and edit docs, [published as part of the documentation](https://microk8s.io/docs/docs).\n\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2018 Canonical, Ltd.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "\n\n[![](https://github.com/canonical/microk8s/actions/workflows/build-snap.yml/badge.svg)](https://github.com/canonical/microk8s/actions/workflows/build-snap.yml)\n[![](https://snapcraft.io/microk8s/badge.svg)](https://snapcraft.io/microk8s)\n\n\n\n## The smallest, fastest Kubernetes\n\nSingle-package fully conformant lightweight Kubernetes that works on [42\nflavours of Linux](https://snapcraft.io/microk8s). Perfect for:\n\n- Developer workstations\n- IoT\n- Edge\n- CI/CD\n\n > Canonical might have assembled the easiest way to provision a single node Kubernetes cluster - [Kelsey Hightower](https://twitter.com/kelseyhightower/status/1120834594138406912)\n\n## Why MicroK8s?\n\n- **Small**. Developers want the smallest K8s for laptop and workstation\n development. MicroK8s provides a standalone K8s compatible with Azure\n AKS, Amazon EKS, Google GKE when you run it on Ubuntu.\n\n- **Simple**. Minimize administration and operations with a single-package\n install that has no moving parts for simplicity and certainty. All\n dependencies and batteries included.\n\n- **Secure**. Updates are available for all security issues and can be\n applied immediately or scheduled to suit your maintenance cycle.\n\n- **Current**. MicroK8s tracks upstream and releases beta, RC and final bits\n the same day as upstream K8s. You can track latest K8s or stick to any\n release version from 1.10 onwards.\n\n- **Comprehensive**. MicroK8s includes a curated collection of manifests for\n common K8s capabilities and services:\n\n - Service Mesh: Istio, Linkerd\n - Serverless: Knative\n - Monitoring: Fluentd, Prometheus, Grafana, Metrics\n - Ingress, DNS, Dashboard, Clustering\n - Automatic updates to the latest Kubernetes version\n - GPGPU bindings for AI/ML\n\nDrop us a line at [MicroK8s in the Wild](docs/community.md) if you are\ndoing something fun with MicroK8s!\n\n## Quickstart\n\nInstall MicroK8s with:\n\n```\nsnap install microk8s --classic\n```\n\nMicroK8s includes a `microk8s kubectl` command:\n\n```\nsudo microk8s kubectl get nodes\nsudo microk8s kubectl get services\n```\n\nTo use MicroK8s with your existing kubectl:\n\n```\nsudo microk8s kubectl config view --raw > $HOME/.kube/config\n```\n\n#### User access without sudo\nThe *microk8s* user group is created during the snap installation. Users in that group\nare granted access to `microk8s` commands. To add a user to that group:\n```\nsudo usermod -a -G microk8s \n```\n\n\n#### Kubernetes add-ons\n\nMicroK8s installs a barebones upstream Kubernetes. Additional services like dns and the Kubernetes dashboard can be enabled using the `microk8s enable` command.\n\n```\nsudo microk8s enable dns\nsudo microk8s enable dashboard\n```\n\nUse `microk8s status` to see a list of enabled and available addons. You can find the addon manifests and/or scripts under `${SNAP}/actions/`, with `${SNAP}` pointing by default to `/snap/microk8s/current`.\n\n## Documentation\n\nThe [official docs](https://microk8s.io/docs/) are maintained in the\nKubernetes upstream Discourse.\n\nTake a look at the [build instructions](docs/build.md) if you want to\ncontribute to MicroK8s.\n\n\n \"Get\n \n\n\n \n\n" }, { "path": "build-scripts/.gitignore", "content": ".build\n.install\n" }, { "path": "build-scripts/addons/repositories.sh", "content": "#!/bin/bash -x\n\n# List of addon repositories to bundle in the snap\n# (name),(repository),(reference)\nADDONS_REPOS=\"\ncore,https://github.com/canonical/microk8s-core-addons,main\ncommunity,https://github.com/canonical/microk8s-community-addons,main\n\"\n\n# List of addon repositories to automatically enable\nADDONS_REPOS_ENABLED=\"core\"\n\nINSTALL=\"${1}\"\nif [ -d \"${INSTALL}/addons\" ]; then\n rm -rf \"${INSTALL}/addons\"\nfi\nif [ -d addons ]; then\n rm -rf addons\nfi\n\nIFS=';'\necho \"${ADDONS_REPOS}\" | while read line; do\n if [ -z \"${line}\" ];\n then continue\n fi\n name=\"$(echo ${line} | cut -f1 -d',')\"\n repository=\"$(echo ${line} | cut -f2 -d',')\"\n reference=\"$(echo ${line} | cut -f3 -d',')\"\n git clone \"${repository}\" -b \"${reference}\" \"addons/${name}\"\ndone\necho \"${ADDONS_REPOS_ENABLED}\" > addons/.auto-add\n\ncp -r \"addons\" \"${INSTALL}/addons\"\n" }, { "path": "build-scripts/build-component.sh", "content": "#!/bin/bash\n\nset -ex\n\nDIR=`realpath $(dirname \"${0}\")`\n\nBUILD_DIRECTORY=\"${SNAPCRAFT_PART_BUILD:-${DIR}/.build}\"\nINSTALL_DIRECTORY=\"${SNAPCRAFT_PART_INSTALL:-${DIR}/.install}\"\n\nmkdir -p \"${BUILD_DIRECTORY}\" \"${INSTALL_DIRECTORY}\"\n\nCOMPONENT_NAME=\"${1}\"\nCOMPONENT_DIRECTORY=\"${DIR}/components/${COMPONENT_NAME}\"\n\nGIT_REPOSITORY=\"$(cat \"${COMPONENT_DIRECTORY}/repository\")\"\nGIT_TAG=\"$(\"${COMPONENT_DIRECTORY}/version.sh\")\"\n\nCOMPONENT_BUILD_DIRECTORY=\"${BUILD_DIRECTORY}/${COMPONENT_NAME}\"\n\n# cleanup git repository if we cannot git checkout to the build tag\nif [ -d \"${COMPONENT_BUILD_DIRECTORY}\" ]; then\n cd \"${COMPONENT_BUILD_DIRECTORY}\"\n if ! git checkout \"${GIT_TAG}\"; then\n cd \"${BUILD_DIRECTORY}\"\n rm -rf \"${COMPONENT_BUILD_DIRECTORY}\"\n fi\nfi\n\nif [ ! -d \"${COMPONENT_BUILD_DIRECTORY}\" ]; then\n git clone \"${GIT_REPOSITORY}\" --depth 1 -b \"${GIT_TAG}\" \"${COMPONENT_BUILD_DIRECTORY}\"\nfi\n\ncd \"${COMPONENT_BUILD_DIRECTORY}\"\ngit config user.name \"MicroK8s builder bot\"\ngit config user.email \"microk8s-builder-bot@canonical.com\"\n\nif [ -e \"${COMPONENT_DIRECTORY}/pre-patch.sh\" ]; then\n bash -xe \"${COMPONENT_DIRECTORY}/pre-patch.sh\"\nfi\n\nfor patch in $(python3 \"${DIR}/print-patches-for.py\" \"${COMPONENT_NAME}\" \"${GIT_TAG}\"); do\n git am \"${patch}\"\ndone\n\nbash -xe \"${COMPONENT_DIRECTORY}/build.sh\" \"${INSTALL_DIRECTORY}\" \"${GIT_TAG}\"\n" }, { "path": "build-scripts/components/README.md", "content": "# Parts directory\n\nThis directory contains the build scripts for Go components built into MicroK8s.\n\nThe directory structure looks like this:\n\n```\nbuild-scripts/\n build-component.sh <-- runs as `build-component.sh $component_name`\n - checks out the git repository\n - runs the `pre-patch.sh` script (if any)\n - applies the patches (if any)\n - runs the `build.sh` script to build the component\n component/\n $component_name/\n repository <-- git repository to clone\n version.sh <-- prints the repository tag or commit to checkout\n build.sh <-- runs as `build.sh $output $version`\n first argument is the output directory where\n binaries should be placed, second is the component version\n pre-patch.sh <-- runs as `pre-patch.sh`. takes any action needed before applying\n the component patches\n patches/ <-- list of patches to apply after checkout (see section below)\n ...\n strict-patches/ <-- list of extra patches to apply when building strictly confined snap\n ...\n```\n\n## Applying patches\n\nMost MicroK8s components are retrieved from an upstream source (specified in the `repository`), with a specific tag (specified in `version.sh`), have some patches applied to them (from the `patches/` and `strict-patches/` directories) and are then built (using `build.sh`).\n\n\nThis section explains the directory format for the `patches` and `strict-patches` directories. The same rules apply for both. Note that the `strict-patches` (if any) are applied **after** any `patches` have been applied.\n\nOur patches do not frequently change between versions, but they do have to be rebased from time to time, which breaks compatibility with older versions. For that reason, we maintain a set of patches for each version that introduces a breaking change. Consider the following directory structure for the Kubernetes component.\n\n```\npatches/default/0.patch\npatches/v1.27.0/a.patch\npatches/v1.27.0/b.patch\npatches/v1.27.4/c.patch\npatches/v1.28.0/d.patch\npatches/v1.28.0-beta.0/e.patch\n```\n\nThe Kubernetes version to build may be decided dynamically while building the snap, or be pinned to a specified version. The following table shows which patches we would apply depending on the Kubernetes version that we build:\n\n| Kubernetes version | Applied patches | Explanation |\n| ------------------ | ----------------------- | ------------------------------------------------------------------------------------------ |\n| `v1.27.0` | `a.patch` and `b.patch` | |\n| `v1.27.1` | `a.patch` and `b.patch` | In case there is no exact match, find the most recent older version |\n| `v1.27.4` | `c.patch` | Older patches are not applied |\n| `v1.27.12` | `c.patch` | In semver, `v1.27.12 > v1.27.4` so we again must get the most recent patches |\n| `v1.28.0-rc.0` | `d.patch` | Extra items from semver are ignored, so we can define the `v1.28.0` patch and be done |\n| `v1.28.0-beta.0` | `e.patch` | Extra items from semver are ignored, but due to exact match this patch is used instead |\n| `v1.28.0` | `d.patch` | Extra items from semver are ignored, so we can define the `v1.28.0` patch and be done |\n| `v1.28.4` | `d.patch` | Picks the patches from the stable versions only, not from beta |\n| `v1.29.1` | `d.patch` | Uses patches from most recent version, even if on a different minor |\n| `hack/branch` | `0.patch` | If not semver and no match, any patches from the `default/` directory are applied (if any) |\n\nSame logic applies for all other components as well.\n\n### Testing which patches would be applied\n\nYou can verify which set of patches would be applied in any case using the `print-patches-for.py` script directly:\n\n```bash\n$ ./build-scripts/print-patches-for.py kubernetes v1.27.4\n/home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.27.4/0000-Kubelite-integration.patch\n$ ./build-scripts/print-patches-for.py kubernetes v1.27.3\n/home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.27.0/0000-Kubelite-integration.patch\n/home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.27.0/0001-Unix-socket-skip-validation-in-component-status.patch\n$ ./build-scripts/print-patches-for.py kubernetes v1.28.1\n/home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.28.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch\n/home/ubuntu/microk8s/build-scripts/components/kubernetes/patches/v1.28.0/0000-Kubelite-integration.patch\n```\n\n### How to add support for newer versions\n\nWhen a new release comes out which is no longer compatible with the existing latest patches, simply create a new directory under `patches/` with the new version number. This ensures that previous versions will still work, and newer ones will pick up the fixed patches.\n" }, { "path": "build-scripts/components/cluster-agent/build.sh", "content": "#!/bin/bash\n\nexport INSTALL=\"${1}/bin\"\nmkdir -p \"${INSTALL}\"\n\nmake cluster-agent\ncp cluster-agent \"${INSTALL}\"\n" }, { "path": "build-scripts/components/cluster-agent/repository", "content": "https://github.com/canonical/microk8s-cluster-agent\n" }, { "path": "build-scripts/components/cluster-agent/version.sh", "content": "#!/bin/bash\n\necho \"main\"\n" }, { "path": "build-scripts/components/cni/build.sh", "content": "#!/bin/bash\n\nVERSION=\"${2}\"\n\nINSTALL=\"${1}/opt/cni/bin\"\nmkdir -p \"${INSTALL}\"\n\n# these would very tedious to apply with a patch\ngo get github.com/docker/docker/pkg/reexec\ngo mod vendor\nsed -i 's/^package main/package plugin_main/' plugins/*/*/*.go\nsed -i 's/^func main()/func Main()/' plugins/*/*/*.go\n\nexport CGO_ENABLED=0\n\ngo build -o cni -ldflags \"-s -w -extldflags -static -X github.com/containernetworking/plugins/pkg/utils/buildversion.BuildVersion=${VERSION}\" ./cni.go\n\ncp cni \"${INSTALL}/\"\nfor plugin in dhcp host-local static bridge host-device ipvlan loopback macvlan ptp vlan bandwidth firewall portmap sbr tuning vrf; do\n ln -f -s ./cni \"${INSTALL}/${plugin}\"\ndone\n" }, { "path": "build-scripts/components/cni/patches/default/0001-single-entrypoint-for-cni-tools.patch", "content": "From 3d0636d0ad86c9050da190b50bc01387d71dc80a Mon Sep 17 00:00:00 2001\nFrom: MicroK8s builder bot \nDate: Sun, 12 Feb 2023 13:34:45 +0000\nSubject: [PATCH] single entrypoint for cni tools\n\n---\n cni.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 54 insertions(+)\n create mode 100644 cni.go\n\ndiff --git a/cni.go b/cni.go\nnew file mode 100644\nindex 0000000..93828f9\n--- /dev/null\n+++ b/cni.go\n@@ -0,0 +1,54 @@\n+package main\n+\n+import (\n+\t\"os\"\n+\t\"path/filepath\"\n+\n+\t\"github.com/docker/docker/pkg/reexec\"\n+\n+\tipam_dhcp \"github.com/containernetworking/plugins/plugins/ipam/dhcp\"\n+\tipam_host_local \"github.com/containernetworking/plugins/plugins/ipam/host-local\"\n+\tipam_static \"github.com/containernetworking/plugins/plugins/ipam/static\"\n+\n+\tmain_bridge \"github.com/containernetworking/plugins/plugins/main/bridge\"\n+\tmain_host_device \"github.com/containernetworking/plugins/plugins/main/host-device\"\n+\tmain_ipvlan \"github.com/containernetworking/plugins/plugins/main/ipvlan\"\n+\tmain_loopback \"github.com/containernetworking/plugins/plugins/main/loopback\"\n+\tmain_macvlan \"github.com/containernetworking/plugins/plugins/main/macvlan\"\n+\tmain_ptp \"github.com/containernetworking/plugins/plugins/main/ptp\"\n+\tmain_vlan \"github.com/containernetworking/plugins/plugins/main/vlan\"\n+\n+\tmeta_bandwidth \"github.com/containernetworking/plugins/plugins/meta/bandwidth\"\n+\tmeta_firewall \"github.com/containernetworking/plugins/plugins/meta/firewall\"\n+\tmeta_portmap \"github.com/containernetworking/plugins/plugins/meta/portmap\"\n+\tmeta_sbr \"github.com/containernetworking/plugins/plugins/meta/sbr\"\n+\tmeta_tuning \"github.com/containernetworking/plugins/plugins/meta/tuning\"\n+\tmeta_vrf \"github.com/containernetworking/plugins/plugins/meta/vrf\"\n+)\n+\n+func main() {\n+\tos.Args[0] = filepath.Base(os.Args[0])\n+\tif reexec.Init() {\n+\t\treturn\n+\t}\n+\tpanic(\"invalid entrypoint name\")\n+}\n+\n+func init() {\n+\treexec.Register(\"dhcp\", ipam_dhcp.Main)\n+\treexec.Register(\"host-local\", ipam_host_local.Main)\n+\treexec.Register(\"static\", ipam_static.Main)\n+\treexec.Register(\"bridge\", main_bridge.Main)\n+\treexec.Register(\"host-device\", main_host_device.Main)\n+\treexec.Register(\"ipvlan\", main_ipvlan.Main)\n+\treexec.Register(\"loopback\", main_loopback.Main)\n+\treexec.Register(\"macvlan\", main_macvlan.Main)\n+\treexec.Register(\"ptp\", main_ptp.Main)\n+\treexec.Register(\"vlan\", main_vlan.Main)\n+\treexec.Register(\"bandwidth\", meta_bandwidth.Main)\n+\treexec.Register(\"firewall\", meta_firewall.Main)\n+\treexec.Register(\"portmap\", meta_portmap.Main)\n+\treexec.Register(\"sbr\", meta_sbr.Main)\n+\treexec.Register(\"tuning\", meta_tuning.Main)\n+\treexec.Register(\"vrf\", meta_vrf.Main)\n+}\n-- \n2.25.1\n" }, { "path": "build-scripts/components/cni/repository", "content": "https://github.com/containernetworking/plugins\n" }, { "path": "build-scripts/components/cni/version.sh", "content": "#!/bin/bash\n\n# Match https://github.com/kubernetes/kubernetes/blob/master/build/dependencies.yaml#L20\necho \"v1.8.0\"\n" }, { "path": "build-scripts/components/containerd/build.sh", "content": "#!/bin/bash\n\nINSTALL=\"${1}/bin\"\nmkdir -p \"${INSTALL}\"\n\nVERSION=\"${2}\"\nREVISION=$(git rev-parse HEAD)\n\nsed -i \"s,^VERSION.*$,VERSION=${VERSION},\" Makefile\nsed -i \"s,^REVISION.*$,REVISION=${REVISION},\" Makefile\n\nexport STATIC=1\nfor bin in ctr containerd containerd-shim-runc-v2; do\n make \"bin/${bin}\"\n cp \"bin/${bin}\" \"${INSTALL}/${bin}\"\ndone\n" }, { "path": "build-scripts/components/containerd/patches/default/0001-microk8s-sideload-images-plugin.patch", "content": "From d703811ab64963a6d52e6ac98b6a33b26b13e020 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Mon, 10 Jul 2023 12:15:34 +0300\nSubject: [PATCH] microk8s sideload images plugin\n\n---\n cmd/containerd/builtins_microk8s.go | 6 ++\n microk8s_plugins/sideload.go | 132 ++++++++++++++++++++++++++++\n 2 files changed, 138 insertions(+)\n create mode 100644 cmd/containerd/builtins_microk8s.go\n create mode 100644 microk8s_plugins/sideload.go\n\ndiff --git a/cmd/containerd/builtins_microk8s.go b/cmd/containerd/builtins_microk8s.go\nnew file mode 100644\nindex 0000000..d9afc6f\n--- /dev/null\n+++ b/cmd/containerd/builtins_microk8s.go\n@@ -0,0 +1,6 @@\n+package main\n+\n+// register containerd microk8s plugins here\n+import (\n+\t_ \"github.com/containerd/containerd/microk8s_plugins\"\n+)\ndiff --git a/microk8s_plugins/sideload.go b/microk8s_plugins/sideload.go\nnew file mode 100644\nindex 0000000..3ac632e\n--- /dev/null\n+++ b/microk8s_plugins/sideload.go\n@@ -0,0 +1,132 @@\n+package microk8s\n+\n+import (\n+\t\"fmt\"\n+\t\"os\"\n+\t\"path/filepath\"\n+\t\"time\"\n+\n+\t\"github.com/containerd/containerd\"\n+\t\"github.com/containerd/containerd/log\"\n+\t\"github.com/containerd/containerd/platforms\"\n+\t\"github.com/containerd/containerd/plugin\"\n+)\n+\n+const pluginName = \"sideload-images\"\n+\n+var logger = log.L.WithField(\"plugin\", pluginName)\n+\n+type Config struct {\n+\t// Interval configures how frequently the plugin will look for new images found\n+\t// in the sources. If set to zero, images are only loaded during initial start.\n+\tInterval *time.Duration `toml:\"interval\"`\n+\n+\t// Sources is a list of paths to look for .tar images.\n+\t// For example, `/var/snap/microk8s/common/etc/sideload`\n+\tSources []string `toml:\"sources\"`\n+\n+\t// Namespace the images will be loaded into, e.g. \"k8s.io\"\n+\tNamespace string `toml:\"namespace\"`\n+}\n+\n+func (c *Config) SetDefaults() {\n+\tif c.Namespace == \"\" {\n+\t\tc.Namespace = \"k8s.io\"\n+\t}\n+\tif len(c.Sources) == 0 {\n+\t\tsnapCommon := os.Getenv(\"SNAP_COMMON\")\n+\t\tif snapCommon == \"\" {\n+\t\t\tsnapCommon = \"/var/snap/microk8s/common\"\n+\t\t}\n+\t\tc.Sources = []string{filepath.Join(snapCommon, \"etc\", \"sideload\")}\n+\t}\n+\tif c.Interval == nil {\n+\t\tt := 5 * time.Second\n+\t\tc.Interval = &t\n+\t}\n+}\n+\n+func init() {\n+\tc := &Config{}\n+\tplugin.Register(&plugin.Registration{\n+\t\tType: plugin.ServicePlugin,\n+\t\tID: pluginName,\n+\t\tConfig: c,\n+\t\tInitFn: func(ic *plugin.InitContext) (interface{}, error) {\n+\t\t\tconfig := ic.Config.(*Config)\n+\t\t\tconfig.SetDefaults()\n+\n+\t\t\tlogger.Debugf(\"Loaded config %#v\", config)\n+\n+\t\t\tif len(config.Sources) == 0 {\n+\t\t\t\treturn nil, fmt.Errorf(\"no sources configured: %w\", plugin.ErrSkipPlugin)\n+\t\t\t}\n+\n+\t\t\tgo func() {\n+\t\t\t\t// get a containerd client\n+\t\t\t\tvar (\n+\t\t\t\t\tcl *containerd.Client\n+\t\t\t\t\terr error\n+\t\t\t\t)\n+\t\t\t\tfor cl == nil {\n+\t\t\t\t\tselect {\n+\t\t\t\t\tcase <-ic.Context.Done():\n+\t\t\t\t\t\treturn\n+\t\t\t\t\tdefault:\n+\t\t\t\t\t}\n+\n+\t\t\t\t\tcl, err = containerd.New(ic.Address, containerd.WithDefaultNamespace(config.Namespace), containerd.WithTimeout(2*time.Second))\n+\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\tlogger.Info(\"Failed to create containerd client\")\n+\t\t\t\t\t}\n+\t\t\t\t}\n+\n+\t\t\t\tfor {\n+\t\t\t\tnextDir:\n+\t\t\t\t\tfor _, dir := range c.Sources {\n+\t\t\t\t\t\tlogger := logger.WithField(\"dir\", dir)\n+\t\t\t\t\t\tlogger.Debug(\"Looking for images\")\n+\t\t\t\t\t\tfiles, err := filepath.Glob(filepath.Join(dir, \"*.tar\"))\n+\t\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\t\tlogger.WithError(err).Warn(\"Failed to look for images\")\n+\t\t\t\t\t\t\tcontinue nextDir\n+\t\t\t\t\t\t}\n+\n+\t\t\t\t\tnextFile:\n+\t\t\t\t\t\tfor _, file := range files {\n+\t\t\t\t\t\t\tlogger := logger.WithField(\"file\", file)\n+\t\t\t\t\t\t\tr, err := os.Open(file)\n+\t\t\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\t\t\tlogger.WithError(err).Warn(\"Failed to open file\")\n+\t\t\t\t\t\t\t\tcontinue nextFile\n+\t\t\t\t\t\t\t}\n+\t\t\t\t\t\t\timages, err := cl.Import(ic.Context, r, containerd.WithImportPlatform(platforms.Default()))\n+\t\t\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\t\t\tlogger.WithError(err).Error(\"Failed to import images\")\n+\t\t\t\t\t\t\t} else {\n+\t\t\t\t\t\t\t\tlogger.Infof(\"Imported %d images\", len(images))\n+\t\t\t\t\t\t\t\tos.Rename(file, file+\".loaded\")\n+\t\t\t\t\t\t\t}\n+\t\t\t\t\t\t\tif closeErr := r.Close(); closeErr != nil {\n+\t\t\t\t\t\t\t\tlogger.WithError(closeErr).Error(\"Failed to close reader\")\n+\t\t\t\t\t\t\t}\n+\t\t\t\t\t\t}\n+\t\t\t\t\t}\n+\n+\t\t\t\t\t// retry after interval, finish if interval is zero\n+\t\t\t\t\tif *c.Interval == 0 {\n+\t\t\t\t\t\tlogger.Info(\"Plugin terminating\")\n+\t\t\t\t\t\treturn\n+\t\t\t\t\t}\n+\t\t\t\t\tselect {\n+\t\t\t\t\tcase <-ic.Context.Done():\n+\t\t\t\t\t\treturn\n+\t\t\t\t\tcase <-time.After(*c.Interval):\n+\t\t\t\t\t}\n+\t\t\t\t}\n+\t\t\t}()\n+\n+\t\t\treturn nil, nil\n+\t\t},\n+\t})\n+}\n--\n2.34.1\n" }, { "path": "build-scripts/components/containerd/patches/v2.1.3/0001-microk8s-sideload-images-plugin.patch", "content": "From 7f26b3e013169510867383f09358b2d91641ad9f Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Mon, 10 Jul 2023 12:15:34 +0300\nSubject: [PATCH] microk8s sideload images plugin\n\n---\n cmd/containerd/builtins_microk8s.go | 6 ++\n microk8s_plugins/sideload.go | 142 ++++++++++++++++++++++++++++\n 2 files changed, 148 insertions(+)\n create mode 100644 cmd/containerd/builtins_microk8s.go\n create mode 100644 microk8s_plugins/sideload.go\n\ndiff --git a/cmd/containerd/builtins_microk8s.go b/cmd/containerd/builtins_microk8s.go\nnew file mode 100644\nindex 000000000..c215987fa\n--- /dev/null\n+++ b/cmd/containerd/builtins_microk8s.go\n@@ -0,0 +1,6 @@\n+package main\n+\n+// register containerd microk8s plugins here\n+import (\n+\t_ \"github.com/containerd/containerd/v2/microk8s_plugins\"\n+)\ndiff --git a/microk8s_plugins/sideload.go b/microk8s_plugins/sideload.go\nnew file mode 100644\nindex 000000000..a6d97c8a3\n--- /dev/null\n+++ b/microk8s_plugins/sideload.go\n@@ -0,0 +1,142 @@\n+package microk8s\n+\n+import (\n+\t\"fmt\"\n+\t\"os\"\n+\t\"path/filepath\"\n+\t\"time\"\n+\n+\tcontainerd \"github.com/containerd/containerd/v2/client\"\n+\t\"github.com/containerd/containerd/v2/pkg/namespaces\"\n+\t\"github.com/containerd/containerd/v2/plugins\"\n+\t\"github.com/containerd/log\"\n+\t\"github.com/containerd/platforms\"\n+\t\"github.com/containerd/plugin\"\n+\t\"github.com/containerd/plugin/registry\"\n+)\n+\n+const pluginName = \"sideload-images\"\n+\n+var logger = log.L.WithField(\"plugin\", pluginName)\n+\n+type Config struct {\n+\t// Interval configures how frequently the plugin will look for new images found\n+\t// in the sources. If set to zero, images are only loaded during initial start.\n+\tInterval *time.Duration `toml:\"interval\"`\n+\n+\t// Sources is a list of paths to look for .tar images.\n+\t// For example, `/var/snap/microk8s/common/etc/sideload`\n+\tSources []string `toml:\"sources\"`\n+\n+\t// Namespace the images will be loaded into, e.g. \"k8s.io\"\n+\tNamespace string `toml:\"namespace\"`\n+}\n+\n+func (c *Config) SetDefaults() {\n+\tif c.Namespace == \"\" {\n+\t\tc.Namespace = \"k8s.io\"\n+\t}\n+\tif len(c.Sources) == 0 {\n+\t\tsnapCommon := os.Getenv(\"SNAP_COMMON\")\n+\t\tif snapCommon == \"\" {\n+\t\t\tsnapCommon = \"/var/snap/microk8s/common\"\n+\t\t}\n+\t\tc.Sources = []string{filepath.Join(snapCommon, \"etc\", \"sideload\")}\n+\t}\n+\tif c.Interval == nil {\n+\t\tt := 5 * time.Second\n+\t\tc.Interval = &t\n+\t}\n+}\n+\n+func init() {\n+\tc := &Config{}\n+\tregistry.Register(&plugin.Registration{\n+\t\tType: plugins.ServicePlugin,\n+\t\tID: pluginName,\n+\t\tConfig: c,\n+\t\tInitFn: func(ic *plugin.InitContext) (interface{}, error) {\n+\t\t\tconfig := ic.Config.(*Config)\n+\t\t\tconfig.SetDefaults()\n+\n+\t\t\tlogger.Debugf(\"Loaded config %#v\", config)\n+\n+\t\t\tif len(config.Sources) == 0 {\n+\t\t\t\treturn nil, fmt.Errorf(\"no sources configured: %w\", plugin.ErrSkipPlugin)\n+\t\t\t}\n+\n+\t\t\tgo func() {\n+\t\t\t\t// get a containerd client\n+\t\t\t\tvar (\n+\t\t\t\t\tcl *containerd.Client\n+\t\t\t\t\terr error\n+\t\t\t\t)\n+\t\t\t\tfor cl == nil {\n+\t\t\t\t\tselect {\n+\t\t\t\t\tcase <-ic.Context.Done():\n+\t\t\t\t\t\treturn\n+\t\t\t\t\tdefault:\n+\t\t\t\t\t}\n+\n+\t\t\t\t\tcl, err = containerd.New(\n+\t\t\t\t\t\t\"\",\n+\t\t\t\t\t\tcontainerd.WithDefaultNamespace(config.Namespace),\n+\t\t\t\t\t\tcontainerd.WithDefaultPlatform(platforms.Default()),\n+\t\t\t\t\t\tcontainerd.WithInMemoryServices(ic),\n+\t\t\t\t\t\tcontainerd.WithTimeout(2*time.Second),\n+\t\t\t\t\t)\n+\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\tlogger.Info(\"Failed to create containerd client\")\n+\t\t\t\t\t}\n+\t\t\t\t}\n+\n+\t\t\t\tfor {\n+\t\t\t\tnextDir:\n+\t\t\t\t\tfor _, dir := range c.Sources {\n+\t\t\t\t\t\tlogger := logger.WithField(\"dir\", dir)\n+\t\t\t\t\t\tlogger.Debug(\"Looking for images\")\n+\t\t\t\t\t\tfiles, err := filepath.Glob(filepath.Join(dir, \"*.tar\"))\n+\t\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\t\tlogger.WithError(err).Warn(\"Failed to look for images\")\n+\t\t\t\t\t\t\tcontinue nextDir\n+\t\t\t\t\t\t}\n+\n+\t\t\t\t\tnextFile:\n+\t\t\t\t\t\tfor _, file := range files {\n+\t\t\t\t\t\t\tlogger := logger.WithField(\"file\", file)\n+\t\t\t\t\t\t\tr, err := os.Open(file)\n+\t\t\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\t\t\tlogger.WithError(err).Warn(\"Failed to open file\")\n+\t\t\t\t\t\t\t\tcontinue nextFile\n+\t\t\t\t\t\t\t}\n+\t\t\t\t\t\t\tctx := namespaces.WithNamespace(ic.Context, config.Namespace)\n+\t\t\t\t\t\t\timages, err := cl.Import(ctx, r, containerd.WithImportPlatform(platforms.DefaultStrict()))\n+\t\t\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\t\t\tlogger.WithError(err).Error(\"Failed to import images\")\n+\t\t\t\t\t\t\t} else {\n+\t\t\t\t\t\t\t\tlogger.Infof(\"Imported %d images\", len(images))\n+\t\t\t\t\t\t\t\tos.Rename(file, file+\".loaded\")\n+\t\t\t\t\t\t\t}\n+\t\t\t\t\t\t\tif closeErr := r.Close(); closeErr != nil {\n+\t\t\t\t\t\t\t\tlogger.WithError(closeErr).Error(\"Failed to close reader\")\n+\t\t\t\t\t\t\t}\n+\t\t\t\t\t\t}\n+\t\t\t\t\t}\n+\n+\t\t\t\t\t// retry after interval, finish if interval is zero\n+\t\t\t\t\tif *c.Interval == 0 {\n+\t\t\t\t\t\tlogger.Info(\"Plugin terminating\")\n+\t\t\t\t\t\treturn\n+\t\t\t\t\t}\n+\t\t\t\t\tselect {\n+\t\t\t\t\tcase <-ic.Context.Done():\n+\t\t\t\t\t\treturn\n+\t\t\t\t\tcase <-time.After(*c.Interval):\n+\t\t\t\t\t}\n+\t\t\t\t}\n+\t\t\t}()\n+\n+\t\t\treturn nil, nil\n+\t\t},\n+\t})\n+}\n-- \n2.43.0\n" }, { "path": "build-scripts/components/containerd/repository", "content": "https://github.com/containerd/containerd\n" }, { "path": "build-scripts/components/containerd/version.sh", "content": "#!/bin/bash\n\necho \"v2.1.3\"\n" }, { "path": "build-scripts/components/etcd/build.sh", "content": "#!/bin/bash\n\nexport INSTALL=\"${1}\"\nmkdir -p \"${INSTALL}\"\n\nGO_LDFLAGS=\"-s -w\" GO_BUILD_FLAGS=\"-v\" make build\n\nfor bin in etcd etcdctl; do\n cp \"bin/${bin}\" \"${INSTALL}/${bin}\"\ndone\n" }, { "path": "build-scripts/components/etcd/repository", "content": "https://github.com/etcd-io/etcd\n" }, { "path": "build-scripts/components/etcd/version.sh", "content": "#!/bin/bash\n\necho \"v3.6.6\"\n" }, { "path": "build-scripts/components/flannel-cni-plugin/build.sh", "content": "#!/bin/bash\n\nINSTALL=\"${1}/opt/cni/bin\"\nmkdir -p \"${INSTALL}\"\n\nVERSION=\"${2}\"\n\nexport CGO_ENABLED=0\ngo build -o dist/flannel -ldflags \"-s -w -X github.com/flannel-io/cni-plugin/version.Version=${VERSION} -extldflags -static\"\n\ncp dist/flannel \"${INSTALL}/flannel\"\n" }, { "path": "build-scripts/components/flannel-cni-plugin/repository", "content": "https://github.com/flannel-io/cni-plugin\n" }, { "path": "build-scripts/components/flannel-cni-plugin/version.sh", "content": "#!/bin/bash\n\necho \"v1.8.0-flannel2\"\n" }, { "path": "build-scripts/components/flanneld/build.sh", "content": "#!/bin/bash\n\nINSTALL=\"${1}/opt/cni/bin\"\nmkdir -p \"${INSTALL}\"\n\nVERSION=\"${2}\"\n\nexport CGO_ENABLED=0\ngo build -o dist/flanneld -ldflags \"-s -w -X github.com/flannel-io/flannel/version.Version=${VERSION} -extldflags -static\"\n\ncp dist/flanneld \"${INSTALL}/flanneld\"\n" }, { "path": "build-scripts/components/flanneld/patches/default/0001-disable-udp-backend.patch", "content": "From 45ec777a0d113089453eca7fd2f7cb195555c6c9 Mon Sep 17 00:00:00 2001\nFrom: MicroK8s builder bot \nDate: Wed, 15 Feb 2023 15:52:51 +0000\nSubject: [PATCH] disable udp backend\n\n---\n main.go | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n\ndiff --git a/main.go b/main.go\nindex 064f58d..b591458 100644\n--- a/main.go\n+++ b/main.go\n@@ -50,7 +50,7 @@ import (\n \t_ \"github.com/flannel-io/flannel/pkg/backend/ipip\"\n \t_ \"github.com/flannel-io/flannel/pkg/backend/ipsec\"\n \t_ \"github.com/flannel-io/flannel/pkg/backend/tencentvpc\"\n-\t_ \"github.com/flannel-io/flannel/pkg/backend/udp\"\n+\t// _ \"github.com/flannel-io/flannel/pkg/backend/udp\"\n \t_ \"github.com/flannel-io/flannel/pkg/backend/vxlan\"\n \t_ \"github.com/flannel-io/flannel/pkg/backend/wireguard\"\n )\n-- \n2.25.1\n" }, { "path": "build-scripts/components/flanneld/repository", "content": "https://github.com/flannel-io/flannel\n" }, { "path": "build-scripts/components/flanneld/version.sh", "content": "#!/bin/bash\n\necho \"v0.27.4\"\n" }, { "path": "build-scripts/components/helm/build.sh", "content": "#!/bin/bash\n\nVERSION=\"${2}\"\n\nINSTALL=\"${1}\"\nmkdir -p \"${INSTALL}/bin\"\n\nmake VERSION=\"${VERSION}\"\ncp bin/helm \"${INSTALL}/bin/helm\"\n\n./bin/helm completion bash \\\n | sed \"s/complete -o default -F __start_helm helm/complete -o default -F __start_helm microk8s.helm/g\" \\\n | sed \"s/complete -o default -o nospace -F __start_helm helm/complete -o default -o nospace -F __start_helm microk8s.helm/g\" \\\n > bin/helm.bash\n\n./bin/helm completion bash \\\n | sed \"s/complete -o default -F __start_helm helm/complete -o default -F __start_helm microk8s.helm3/g\" \\\n | sed \"s/complete -o default -o nospace -F __start_helm helm/complete -o default -o nospace -F __start_helm microk8s.helm3/g\" \\\n > bin/helm3.bash\n\ncp bin/helm.bash \"${INSTALL}/helm.bash\"\ncp bin/helm3.bash \"${INSTALL}/helm3.bash\"\n" }, { "path": "build-scripts/components/helm/repository", "content": "https://github.com/helm/helm\n" }, { "path": "build-scripts/components/helm/version.sh", "content": "#!/bin/bash\n\necho \"v3.19.2\"\n" }, { "path": "build-scripts/components/k8s-dqlite/build.sh", "content": "#!/bin/bash\n\nINSTALL=\"${1}/bin\"\nmkdir -p \"${INSTALL}\"\n\nmake static -j\n\ncp bin/static/dqlite \"${INSTALL}/dqlite\"\ncp bin/static/k8s-dqlite \"${INSTALL}/k8s-dqlite\"\n" }, { "path": "build-scripts/components/k8s-dqlite/repository", "content": "https://github.com/canonical/k8s-dqlite\n" }, { "path": "build-scripts/components/k8s-dqlite/version.sh", "content": "#!/bin/bash\n\necho \"v1.8.1\"\n" }, { "path": "build-scripts/components/kubernetes/build.sh", "content": "#!/bin/bash -x\n\nINSTALL=\"${1}\"\n\nexport KUBE_GIT_VERSION_FILE=\"${PWD}/.version.sh\"\n\nfor app in kubectl kubelite; do\n make WHAT=\"cmd/${app}\" KUBE_STATIC_OVERRIDES=kubelite\n cp _output/bin/\"${app}\" \"${INSTALL}/${app}\"\ndone\n\n_output/bin/kubectl completion bash \\\n | sed \"s/complete -o default -F __start_kubectl kubectl/complete -o default -F __start_kubectl microk8s.kubectl/g\" \\\n | sed \"s/complete -o default -o nospace -F __start_kubectl kubectl/complete -o default -o nospace -F __start_kubectl microk8s.kubectl/g\" \\\n > _output/kubectl.bash\n\ncp _output/kubectl.bash \"${INSTALL}/kubectl.bash\"\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.27.0/0000-Kubelite-integration.patch", "content": "From d0ae18d074db5ff361f363073f32b2f30c7a3686 Mon Sep 17 00:00:00 2001\nFrom: Konstantinos Tsakalozos \nDate: Wed, 3 Mar 2021 18:19:37 +0200\nSubject: [PATCH] Kubelite integration\n\n---\n cmd/kube-apiserver/app/server.go | 9 +++-\n cmd/kube-scheduler/app/server.go | 6 ++-\n cmd/kubelet/app/server.go | 13 +++--\n cmd/kubelite/app/daemons/daemon.go | 84 +++++++++++++++++++++++++++++\n cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/app/server.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/kubelite.go | 28 ++++++++++\n pkg/volume/csi/csi_plugin.go | 10 ++--\n 8 files changed, 297 insertions(+), 11 deletions(-)\n create mode 100644 cmd/kubelite/app/daemons/daemon.go\n create mode 100644 cmd/kubelite/app/options/options.go\n create mode 100644 cmd/kubelite/app/server.go\n create mode 100644 cmd/kubelite/kubelite.go\n\ndiff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go\nindex fc36d044dbe..cffb7c35a3c 100644\n--- a/cmd/kube-apiserver/app/server.go\n+++ b/cmd/kube-apiserver/app/server.go\n@@ -89,7 +89,7 @@ func init() {\n }\n\n // NewAPIServerCommand creates a *cobra.Command object with default parameters\n-func NewAPIServerCommand() *cobra.Command {\n+func NewAPIServerCommand(stopCh... <- chan struct{}) *cobra.Command {\n \ts := options.NewServerRunOptions()\n \tcmd := &cobra.Command{\n \t\tUse: \"kube-apiserver\",\n@@ -129,7 +129,12 @@ cluster's shared state through which all other components interact.`,\n \t\t\t}\n \t\t\t// add feature enablement metrics\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n-\t\t\treturn Run(completedOptions, genericapiserver.SetupSignalHandler())\n+\n+\t\t\tif len(stopCh) != 0 {\n+\t\t\t\treturn Run(completedOptions, stopCh[0])\n+\t\t\t} else {\n+\t\t\t\treturn Run(completedOptions, genericapiserver.SetupSignalHandler())\n+\t\t\t}\n \t\t},\n \t\tArgs: func(cmd *cobra.Command, args []string) error {\n \t\t\tfor _, arg := range args {\ndiff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go\nindex 8d01f3b7670..44ac7f69328 100644\n--- a/cmd/kube-scheduler/app/server.go\n+++ b/cmd/kube-scheduler/app/server.go\n@@ -132,7 +132,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op\n \tctx, cancel := context.WithCancel(context.Background())\n \tdefer cancel()\n \tgo func() {\n-\t\tstopCh := server.SetupSignalHandler()\n+ c := cmd.Context()\n+ if c == nil {\n+ c = server.SetupSignalContext()\n+ }\n+ stopCh := c.Done()\n \t\t<-stopCh\n \t\tcancel()\n \t}()\ndiff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go\nindex 9444f136866..8ca88f64d04 100644\n--- a/cmd/kubelet/app/server.go\n+++ b/cmd/kubelet/app/server.go\n@@ -120,7 +120,7 @@ const (\n )\n\n // NewKubeletCommand creates a *cobra.Command object with default parameters\n-func NewKubeletCommand() *cobra.Command {\n+func NewKubeletCommand(ctx ...context.Context) *cobra.Command {\n \tcleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)\n \tcleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n \tkubeletFlags := options.NewKubeletFlags()\n@@ -250,6 +250,12 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API\n \t\t\tif err := checkPermissions(); err != nil {\n \t\t\t\tklog.ErrorS(err, \"kubelet running with insufficient permissions\")\n \t\t\t}\n+\t\t\trunctx := context.Background()\n+\t\t\tif len(ctx) == 0 {\n+\t\t\t\trunctx = genericapiserver.SetupSignalContext()\n+\t\t\t} else {\n+\t\t\t\trunctx = ctx[0]\n+\t\t\t}\n\n \t\t\t// make the kubelet's config safe for logging\n \t\t\tconfig := kubeletServer.KubeletConfiguration.DeepCopy()\n@@ -259,12 +265,9 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API\n \t\t\t// log the kubelet's config for inspection\n \t\t\tklog.V(5).InfoS(\"KubeletConfiguration\", \"configuration\", config)\n\n-\t\t\t// set up signal context for kubelet shutdown\n-\t\t\tctx := genericapiserver.SetupSignalContext()\n-\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n \t\t\t// run the kubelet\n-\t\t\treturn Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n+\t\t\treturn Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n \t\t},\n \t}\n\ndiff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go\nnew file mode 100644\nindex 00000000000..dbef03cf07e\n--- /dev/null\n+++ b/cmd/kubelite/app/daemons/daemon.go\n@@ -0,0 +1,84 @@\n+package daemon\n+\n+import (\n+\t\"context\"\n+\t\"k8s.io/client-go/kubernetes\"\n+\t\"k8s.io/client-go/tools/clientcmd\"\n+\t\"k8s.io/klog/v2\"\n+\tgenericcontrollermanager \"k8s.io/controller-manager/app\"\n+\tapiserver \"k8s.io/kubernetes/cmd/kube-apiserver/app\"\n+\tcontroller \"k8s.io/kubernetes/cmd/kube-controller-manager/app\"\n+\tproxy \"k8s.io/kubernetes/cmd/kube-proxy/app\"\n+\tscheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n+\tkubelet \"k8s.io/kubernetes/cmd/kubelet/app\"\n+\n+\t\"time\"\n+)\n+\n+func StartControllerManager(args []string, ctx context.Context) {\n+\tcommand := controller.NewControllerManagerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Controller Manager\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Controller Manager exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Controller Manager\")\n+}\n+\n+func StartScheduler(args []string, ctx context.Context) {\n+\tcommand := scheduler.NewSchedulerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Scheduler\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Scheduler exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Scheduler\")\n+}\n+\n+func StartProxy(args []string) {\n+\tcommand := proxy.NewProxyCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Proxy\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Proxy exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Proxy\")\n+}\n+\n+func StartKubelet(args []string, ctx context.Context) {\n+\tcommand := kubelet.NewKubeletCommand(ctx)\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Kubelet\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Kubelet exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Kubelet\")\n+}\n+\n+func StartAPIServer(args []string, ctx <-chan struct{}) {\n+\tcommand := apiserver.NewAPIServerCommand(ctx)\n+\tcommand.SetArgs(args)\n+\tklog.Info(\"Starting API Server\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"API Server exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping API Server\")\n+}\n+\n+func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) {\n+\tklog.Info(\"Waiting for the API server\")\n+\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigpath)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not find the cluster's kubeconfig file %v\", err)\n+\t}\n+\t// create the client\n+\tclient, err := kubernetes.NewForConfig(config)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not create client to the cluster %v\", err)\n+\t}\n+\tgenericcontrollermanager.WaitForAPIServer(client, timeout)\n+}\n\\ No newline at end of file\ndiff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go\nnew file mode 100644\nindex 00000000000..80f1d8b09fc\n--- /dev/null\n+++ b/cmd/kubelite/app/options/options.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright 2018 The Kubernetes Authors.\n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+\n+package options\n+\n+import (\n+\t\"bufio\"\n+\t\"k8s.io/klog/v2\"\n+\t\"os\"\n+\t\"strings\"\n+)\n+\n+// Options has all the params needed to run a Kubelite\n+type Options struct {\n+\tSchedulerArgsFile string\n+\tControllerManagerArgsFile string\n+\tProxyArgsFile string\n+\tKubeletArgsFile string\n+\tAPIServerArgsFile string\n+\tKubeconfigFile \t\t string\n+\tStartControlPlane\t\t bool\n+}\n+\n+func NewOptions() (*Options){\n+\to := Options{\n+\t\t\"/var/snap/microk8s/current/args/kube-scheduler\",\n+\t\t\"/var/snap/microk8s/current/args/kube-controller-manager\",\n+\t\t\"/var/snap/microk8s/current/args/kube-proxy\",\n+\t\t\"/var/snap/microk8s/current/args/kubelet\",\n+\t\t\"/var/snap/microk8s/current/args/kube-apiserver\",\n+\t\t\"/var/snap/microk8s/current/credentials/client.config\",\n+\t\ttrue,\n+\t}\n+\treturn &o\n+}\n+\n+func ReadArgsFromFile(filename string) []string {\n+\tvar args []string\n+\tfile, err := os.Open(filename)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"Failed to open arguments file %v\", err)\n+\t}\n+\tdefer file.Close()\n+\n+\tscanner := bufio.NewScanner(file)\n+\tfor scanner.Scan() {\n+\t\tline := scanner.Text()\n+\t\tline = strings.TrimSpace(line)\n+\t\t// ignore lines with # and empty lines\n+\t\tif len(line) <= 0 || strings.HasPrefix(line, \"#\") {\n+\t\t\tcontinue\n+\t\t}\n+\t\t// remove \" and '\n+\t\tfor _, r := range \"\\\"'\" {\n+\t\t\tline = strings.ReplaceAll(line, string(r), \"\")\n+\t\t}\n+\t\tfor _, part := range strings.Split(line, \" \") {\n+\n+\t\t\targs = append(args, os.ExpandEnv(part))\n+\t\t}\n+\t}\n+\tif err := scanner.Err(); err != nil {\n+\t\tklog.Fatalf(\"Failed to read arguments file %v\", err)\n+\t}\n+\treturn args\n+}\ndiff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go\nnew file mode 100644\nindex 00000000000..e7452a09e3e\n--- /dev/null\n+++ b/cmd/kubelite/app/server.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright © 2020 NAME HERE \n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+package app\n+\n+import (\n+\t\"fmt\"\n+\t\"github.com/spf13/cobra\"\n+\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n+\tdaemon \"k8s.io/kubernetes/cmd/kubelite/app/daemons\"\n+\t\"k8s.io/kubernetes/cmd/kubelite/app/options\"\n+\t\"os\"\n+\t\"time\"\n+)\n+\n+var opts = options.NewOptions()\n+\n+// liteCmd represents the base command when called without any subcommands\n+var liteCmd = &cobra.Command{\n+\tUse: \"kubelite\",\n+\tShort: \"Single server kubernetes\",\n+\tLong: `A single server that spawns all other kubernetes servers as threads`,\n+\t// Uncomment the following line if your bare application\n+\t// has an action associated with it:\n+\tRun: func(cmd *cobra.Command, args []string) {\n+\t\tctx := genericapiserver.SetupSignalContext()\n+\n+\t\tif opts.StartControlPlane {\n+\t\t\tapiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile)\n+\t\t\tgo daemon.StartAPIServer(apiserverArgs, ctx.Done())\n+\t\t\tdaemon.WaitForAPIServer(opts.KubeconfigFile, 360 * time.Second)\n+\n+\t\t\tcontrollerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile)\n+\t\t\tgo daemon.StartControllerManager(controllerArgs, ctx)\n+\n+\t\t\tschedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile)\n+\t\t\tgo daemon.StartScheduler(schedulerArgs, ctx)\n+\t\t}\n+\n+\t\tproxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile)\n+\t\tgo daemon.StartProxy(proxyArgs)\n+\n+\t\tkubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile)\n+\t\tdaemon.StartKubelet(kubeletArgs, ctx)\n+\t},\n+}\n+\n+// Execute adds all child commands to the root command and sets flags appropriately.\n+// This is called by main.main(). It only needs to happen once to the liteCmd.\n+func Execute() {\n+\tif err := liteCmd.Execute(); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+}\n+\n+func init() {\n+\tcobra.OnInitialize()\n+\n+\tliteCmd.Flags().StringVar(&opts.SchedulerArgsFile, \"scheduler-args-file\", opts.SchedulerArgsFile, \"file with the arguments for the scheduler\")\n+\tliteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, \"controller-manager-args-file\", opts.ControllerManagerArgsFile, \"file with the arguments for the controller manager\")\n+\tliteCmd.Flags().StringVar(&opts.ProxyArgsFile, \"proxy-args-file\", opts.ProxyArgsFile , \"file with the arguments for kube-proxy\")\n+\tliteCmd.Flags().StringVar(&opts.KubeletArgsFile, \"kubelet-args-file\", opts.KubeletArgsFile, \"file with the arguments for kubelet\")\n+\tliteCmd.Flags().StringVar(&opts.APIServerArgsFile, \"apiserver-args-file\", opts.APIServerArgsFile, \"file with the arguments for the API server\")\n+\tliteCmd.Flags().StringVar(&opts.KubeconfigFile , \"kubeconfig-file\", opts.KubeconfigFile, \"the kubeconfig file to use to healthcheck the API server\")\n+\tliteCmd.Flags().BoolVar(&opts.StartControlPlane, \"start-control-plane\", opts.StartControlPlane, \"start the control plane (API server, scheduler and controller manager)\")\n+}\ndiff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go\nnew file mode 100644\nindex 00000000000..667b24f68e6\n--- /dev/null\n+++ b/cmd/kubelite/kubelite.go\n@@ -0,0 +1,28 @@\n+package main\n+\n+import (\n+\t\"github.com/spf13/pflag\"\n+\tcliflag \"k8s.io/component-base/cli/flag\"\n+\t\"math/rand\"\n+\t\"time\"\n+\n+\t\"k8s.io/component-base/logs\"\n+\t_ \"k8s.io/component-base/metrics/prometheus/clientgo\" // load all the prometheus client-go plugin\n+\t_ \"k8s.io/component-base/metrics/prometheus/version\" // for version metric registration\n+\t\"k8s.io/kubernetes/cmd/kubelite/app\"\n+)\n+\n+func main() {\n+\tprintln(\"Starting kubelite\")\n+\trand.Seed(time.Now().UnixNano())\n+\t// TODO: once we switch everything over to Cobra commands, we can go back to calling\n+\t// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the\n+\t// normalize func and add the go flag set by hand.\n+\tpflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n+\t// utilflag.InitFlags()\n+\tlogs.InitLogs()\n+\tdefer logs.FlushLogs()\n+\n+\tapp.Execute()\n+\tprintln(\"Stopping kubelite\")\n+}\ndiff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go\nindex ce7a543c94f..a8094f878d6 100644\n--- a/pkg/volume/csi/csi_plugin.go\n+++ b/pkg/volume/csi/csi_plugin.go\n@@ -240,18 +240,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {\n \t}\n\n \t// Initializing the label management channels\n-\tnim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n+\tlocalNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n\n \t// This function prevents Kubelet from posting Ready status until CSINode\n \t// is both installed and initialized\n-\tif err := initializeCSINode(host); err != nil {\n+\tif err := initializeCSINode(host, localNim); err != nil {\n \t\treturn errors.New(log(\"failed to initialize CSINode: %v\", err))\n \t}\n\n+\tif _, ok := host.(volume.KubeletVolumeHost); ok {\n+\t\tnim = localNim\n+\t}\n+\n \treturn nil\n }\n\n-func initializeCSINode(host volume.VolumeHost) error {\n+func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error {\n \tkvh, ok := host.(volume.KubeletVolumeHost)\n \tif !ok {\n \t\tklog.V(4).Info(\"Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet\")\n--\n2.34.1\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.27.0/0001-Unix-socket-skip-validation-in-component-status.patch", "content": "From dd1db952eab13912a55207c81a2ac267909677ac Mon Sep 17 00:00:00 2001\nFrom: Konstantinos Tsakalozos \nDate: Tue, 24 Aug 2021 11:17:19 +0300\nSubject: [PATCH] Unix socket skip validation in component status\n\n---\n pkg/registry/core/rest/storage_core.go | 6 ++++++\n 1 file changed, 6 insertions(+)\n\ndiff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go\nindex 1f915c32d4b..0bb7f1a9bf9 100644\n--- a/pkg/registry/core/rest/storage_core.go\n+++ b/pkg/registry/core/rest/storage_core.go\n@@ -350,6 +350,12 @@ func (s componentStatusStorage) serversToValidate() map[string]*componentstatus.\n \t\t\tklog.Errorf(\"Failed to parse etcd url for validation: %v\", err)\n \t\t\tcontinue\n \t\t}\n+\n+\t\tif etcdUrl.Scheme == \"unix\" {\n+\t\t\tklog.Infof(\"Socket etcd endpoint detected. Will not validate\")\n+\t\t\tcontinue\n+\t\t}\n+\n \t\tvar port int\n \t\tvar addr string\n \t\tif strings.Contains(etcdUrl.Host, \":\") {\n--\n2.25.1\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.27.4/0000-Kubelite-integration.patch", "content": "From 3162aa9df25819b60a3c0a3b044394639d55280c Mon Sep 17 00:00:00 2001\nFrom: Konstantinos Tsakalozos \nDate: Wed, 3 Mar 2021 18:19:37 +0200\nSubject: [PATCH] Kubelite integration\n\n---\n cmd/kube-apiserver/app/server.go | 9 +++-\n cmd/kube-scheduler/app/server.go | 6 ++-\n cmd/kubelet/app/server.go | 13 +++--\n cmd/kubelite/app/daemons/daemon.go | 84 +++++++++++++++++++++++++++++\n cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/app/server.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/kubelite.go | 28 ++++++++++\n pkg/volume/csi/csi_plugin.go | 10 ++--\n 8 files changed, 297 insertions(+), 11 deletions(-)\n create mode 100644 cmd/kubelite/app/daemons/daemon.go\n create mode 100644 cmd/kubelite/app/options/options.go\n create mode 100644 cmd/kubelite/app/server.go\n create mode 100644 cmd/kubelite/kubelite.go\n\ndiff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go\nindex b021bac2574..8d5a1bad8ed 100644\n--- a/cmd/kube-apiserver/app/server.go\n+++ b/cmd/kube-apiserver/app/server.go\n@@ -76,7 +76,7 @@ func init() {\n }\n\n // NewAPIServerCommand creates a *cobra.Command object with default parameters\n-func NewAPIServerCommand() *cobra.Command {\n+func NewAPIServerCommand(stopCh... <- chan struct{}) *cobra.Command {\n \ts := options.NewServerRunOptions()\n \tcmd := &cobra.Command{\n \t\tUse: \"kube-apiserver\",\n@@ -116,7 +116,12 @@ cluster's shared state through which all other components interact.`,\n \t\t\t}\n \t\t\t// add feature enablement metrics\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n-\t\t\treturn Run(completedOptions, genericapiserver.SetupSignalHandler())\n+\n+\t\t\tif len(stopCh) != 0 {\n+\t\t\t\treturn Run(completedOptions, stopCh[0])\n+\t\t\t} else {\n+\t\t\t\treturn Run(completedOptions, genericapiserver.SetupSignalHandler())\n+\t\t\t}\n \t\t},\n \t\tArgs: func(cmd *cobra.Command, args []string) error {\n \t\t\tfor _, arg := range args {\ndiff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go\nindex c48b09a420d..b7f273b02ac 100644\n--- a/cmd/kube-scheduler/app/server.go\n+++ b/cmd/kube-scheduler/app/server.go\n@@ -132,7 +132,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op\n \tctx, cancel := context.WithCancel(context.Background())\n \tdefer cancel()\n \tgo func() {\n-\t\tstopCh := server.SetupSignalHandler()\n+ c := cmd.Context()\n+ if c == nil {\n+ c = server.SetupSignalContext()\n+ }\n+ stopCh := c.Done()\n \t\t<-stopCh\n \t\tcancel()\n \t}()\ndiff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go\nindex 742524e325f..561cbf68868 100644\n--- a/cmd/kubelet/app/server.go\n+++ b/cmd/kubelet/app/server.go\n@@ -122,7 +122,7 @@ const (\n )\n\n // NewKubeletCommand creates a *cobra.Command object with default parameters\n-func NewKubeletCommand() *cobra.Command {\n+func NewKubeletCommand(ctx ...context.Context) *cobra.Command {\n \tcleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)\n \tcleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n \tkubeletFlags := options.NewKubeletFlags()\n@@ -249,6 +249,12 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\tif err := checkPermissions(); err != nil {\n \t\t\t\tklog.ErrorS(err, \"kubelet running with insufficient permissions\")\n \t\t\t}\n+\t\t\trunctx := context.Background()\n+\t\t\tif len(ctx) == 0 {\n+\t\t\t\trunctx = genericapiserver.SetupSignalContext()\n+\t\t\t} else {\n+\t\t\t\trunctx = ctx[0]\n+\t\t\t}\n\n \t\t\t// make the kubelet's config safe for logging\n \t\t\tconfig := kubeletServer.KubeletConfiguration.DeepCopy()\n@@ -258,12 +264,9 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\t// log the kubelet's config for inspection\n \t\t\tklog.V(5).InfoS(\"KubeletConfiguration\", \"configuration\", klog.Format(config))\n\n-\t\t\t// set up signal context for kubelet shutdown\n-\t\t\tctx := genericapiserver.SetupSignalContext()\n-\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n \t\t\t// run the kubelet\n-\t\t\treturn Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n+\t\t\treturn Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n \t\t},\n \t}\n\ndiff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go\nnew file mode 100644\nindex 00000000000..dbef03cf07e\n--- /dev/null\n+++ b/cmd/kubelite/app/daemons/daemon.go\n@@ -0,0 +1,84 @@\n+package daemon\n+\n+import (\n+\t\"context\"\n+\t\"k8s.io/client-go/kubernetes\"\n+\t\"k8s.io/client-go/tools/clientcmd\"\n+\t\"k8s.io/klog/v2\"\n+\tgenericcontrollermanager \"k8s.io/controller-manager/app\"\n+\tapiserver \"k8s.io/kubernetes/cmd/kube-apiserver/app\"\n+\tcontroller \"k8s.io/kubernetes/cmd/kube-controller-manager/app\"\n+\tproxy \"k8s.io/kubernetes/cmd/kube-proxy/app\"\n+\tscheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n+\tkubelet \"k8s.io/kubernetes/cmd/kubelet/app\"\n+\n+\t\"time\"\n+)\n+\n+func StartControllerManager(args []string, ctx context.Context) {\n+\tcommand := controller.NewControllerManagerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Controller Manager\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Controller Manager exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Controller Manager\")\n+}\n+\n+func StartScheduler(args []string, ctx context.Context) {\n+\tcommand := scheduler.NewSchedulerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Scheduler\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Scheduler exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Scheduler\")\n+}\n+\n+func StartProxy(args []string) {\n+\tcommand := proxy.NewProxyCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Proxy\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Proxy exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Proxy\")\n+}\n+\n+func StartKubelet(args []string, ctx context.Context) {\n+\tcommand := kubelet.NewKubeletCommand(ctx)\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Kubelet\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Kubelet exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Kubelet\")\n+}\n+\n+func StartAPIServer(args []string, ctx <-chan struct{}) {\n+\tcommand := apiserver.NewAPIServerCommand(ctx)\n+\tcommand.SetArgs(args)\n+\tklog.Info(\"Starting API Server\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"API Server exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping API Server\")\n+}\n+\n+func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) {\n+\tklog.Info(\"Waiting for the API server\")\n+\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigpath)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not find the cluster's kubeconfig file %v\", err)\n+\t}\n+\t// create the client\n+\tclient, err := kubernetes.NewForConfig(config)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not create client to the cluster %v\", err)\n+\t}\n+\tgenericcontrollermanager.WaitForAPIServer(client, timeout)\n+}\n\\ No newline at end of file\ndiff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go\nnew file mode 100644\nindex 00000000000..80f1d8b09fc\n--- /dev/null\n+++ b/cmd/kubelite/app/options/options.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright 2018 The Kubernetes Authors.\n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+\n+package options\n+\n+import (\n+\t\"bufio\"\n+\t\"k8s.io/klog/v2\"\n+\t\"os\"\n+\t\"strings\"\n+)\n+\n+// Options has all the params needed to run a Kubelite\n+type Options struct {\n+\tSchedulerArgsFile string\n+\tControllerManagerArgsFile string\n+\tProxyArgsFile string\n+\tKubeletArgsFile string\n+\tAPIServerArgsFile string\n+\tKubeconfigFile \t\t string\n+\tStartControlPlane\t\t bool\n+}\n+\n+func NewOptions() (*Options){\n+\to := Options{\n+\t\t\"/var/snap/microk8s/current/args/kube-scheduler\",\n+\t\t\"/var/snap/microk8s/current/args/kube-controller-manager\",\n+\t\t\"/var/snap/microk8s/current/args/kube-proxy\",\n+\t\t\"/var/snap/microk8s/current/args/kubelet\",\n+\t\t\"/var/snap/microk8s/current/args/kube-apiserver\",\n+\t\t\"/var/snap/microk8s/current/credentials/client.config\",\n+\t\ttrue,\n+\t}\n+\treturn &o\n+}\n+\n+func ReadArgsFromFile(filename string) []string {\n+\tvar args []string\n+\tfile, err := os.Open(filename)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"Failed to open arguments file %v\", err)\n+\t}\n+\tdefer file.Close()\n+\n+\tscanner := bufio.NewScanner(file)\n+\tfor scanner.Scan() {\n+\t\tline := scanner.Text()\n+\t\tline = strings.TrimSpace(line)\n+\t\t// ignore lines with # and empty lines\n+\t\tif len(line) <= 0 || strings.HasPrefix(line, \"#\") {\n+\t\t\tcontinue\n+\t\t}\n+\t\t// remove \" and '\n+\t\tfor _, r := range \"\\\"'\" {\n+\t\t\tline = strings.ReplaceAll(line, string(r), \"\")\n+\t\t}\n+\t\tfor _, part := range strings.Split(line, \" \") {\n+\n+\t\t\targs = append(args, os.ExpandEnv(part))\n+\t\t}\n+\t}\n+\tif err := scanner.Err(); err != nil {\n+\t\tklog.Fatalf(\"Failed to read arguments file %v\", err)\n+\t}\n+\treturn args\n+}\ndiff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go\nnew file mode 100644\nindex 00000000000..e7452a09e3e\n--- /dev/null\n+++ b/cmd/kubelite/app/server.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright © 2020 NAME HERE \n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+package app\n+\n+import (\n+\t\"fmt\"\n+\t\"github.com/spf13/cobra\"\n+\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n+\tdaemon \"k8s.io/kubernetes/cmd/kubelite/app/daemons\"\n+\t\"k8s.io/kubernetes/cmd/kubelite/app/options\"\n+\t\"os\"\n+\t\"time\"\n+)\n+\n+var opts = options.NewOptions()\n+\n+// liteCmd represents the base command when called without any subcommands\n+var liteCmd = &cobra.Command{\n+\tUse: \"kubelite\",\n+\tShort: \"Single server kubernetes\",\n+\tLong: `A single server that spawns all other kubernetes servers as threads`,\n+\t// Uncomment the following line if your bare application\n+\t// has an action associated with it:\n+\tRun: func(cmd *cobra.Command, args []string) {\n+\t\tctx := genericapiserver.SetupSignalContext()\n+\n+\t\tif opts.StartControlPlane {\n+\t\t\tapiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile)\n+\t\t\tgo daemon.StartAPIServer(apiserverArgs, ctx.Done())\n+\t\t\tdaemon.WaitForAPIServer(opts.KubeconfigFile, 360 * time.Second)\n+\n+\t\t\tcontrollerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile)\n+\t\t\tgo daemon.StartControllerManager(controllerArgs, ctx)\n+\n+\t\t\tschedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile)\n+\t\t\tgo daemon.StartScheduler(schedulerArgs, ctx)\n+\t\t}\n+\n+\t\tproxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile)\n+\t\tgo daemon.StartProxy(proxyArgs)\n+\n+\t\tkubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile)\n+\t\tdaemon.StartKubelet(kubeletArgs, ctx)\n+\t},\n+}\n+\n+// Execute adds all child commands to the root command and sets flags appropriately.\n+// This is called by main.main(). It only needs to happen once to the liteCmd.\n+func Execute() {\n+\tif err := liteCmd.Execute(); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+}\n+\n+func init() {\n+\tcobra.OnInitialize()\n+\n+\tliteCmd.Flags().StringVar(&opts.SchedulerArgsFile, \"scheduler-args-file\", opts.SchedulerArgsFile, \"file with the arguments for the scheduler\")\n+\tliteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, \"controller-manager-args-file\", opts.ControllerManagerArgsFile, \"file with the arguments for the controller manager\")\n+\tliteCmd.Flags().StringVar(&opts.ProxyArgsFile, \"proxy-args-file\", opts.ProxyArgsFile , \"file with the arguments for kube-proxy\")\n+\tliteCmd.Flags().StringVar(&opts.KubeletArgsFile, \"kubelet-args-file\", opts.KubeletArgsFile, \"file with the arguments for kubelet\")\n+\tliteCmd.Flags().StringVar(&opts.APIServerArgsFile, \"apiserver-args-file\", opts.APIServerArgsFile, \"file with the arguments for the API server\")\n+\tliteCmd.Flags().StringVar(&opts.KubeconfigFile , \"kubeconfig-file\", opts.KubeconfigFile, \"the kubeconfig file to use to healthcheck the API server\")\n+\tliteCmd.Flags().BoolVar(&opts.StartControlPlane, \"start-control-plane\", opts.StartControlPlane, \"start the control plane (API server, scheduler and controller manager)\")\n+}\ndiff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go\nnew file mode 100644\nindex 00000000000..667b24f68e6\n--- /dev/null\n+++ b/cmd/kubelite/kubelite.go\n@@ -0,0 +1,28 @@\n+package main\n+\n+import (\n+\t\"github.com/spf13/pflag\"\n+\tcliflag \"k8s.io/component-base/cli/flag\"\n+\t\"math/rand\"\n+\t\"time\"\n+\n+\t\"k8s.io/component-base/logs\"\n+\t_ \"k8s.io/component-base/metrics/prometheus/clientgo\" // load all the prometheus client-go plugin\n+\t_ \"k8s.io/component-base/metrics/prometheus/version\" // for version metric registration\n+\t\"k8s.io/kubernetes/cmd/kubelite/app\"\n+)\n+\n+func main() {\n+\tprintln(\"Starting kubelite\")\n+\trand.Seed(time.Now().UnixNano())\n+\t// TODO: once we switch everything over to Cobra commands, we can go back to calling\n+\t// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the\n+\t// normalize func and add the go flag set by hand.\n+\tpflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n+\t// utilflag.InitFlags()\n+\tlogs.InitLogs()\n+\tdefer logs.FlushLogs()\n+\n+\tapp.Execute()\n+\tprintln(\"Stopping kubelite\")\n+}\ndiff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go\nindex 2556517276e..0b5ef45d083 100644\n--- a/pkg/volume/csi/csi_plugin.go\n+++ b/pkg/volume/csi/csi_plugin.go\n@@ -243,18 +243,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {\n \t}\n\n \t// Initializing the label management channels\n-\tnim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n+\tlocalNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n\n \t// This function prevents Kubelet from posting Ready status until CSINode\n \t// is both installed and initialized\n-\tif err := initializeCSINode(host); err != nil {\n+\tif err := initializeCSINode(host, localNim); err != nil {\n \t\treturn errors.New(log(\"failed to initialize CSINode: %v\", err))\n \t}\n\n+\tif _, ok := host.(volume.KubeletVolumeHost); ok {\n+\t\tnim = localNim\n+\t}\n+\n \treturn nil\n }\n\n-func initializeCSINode(host volume.VolumeHost) error {\n+func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error {\n \tkvh, ok := host.(volume.KubeletVolumeHost)\n \tif !ok {\n \t\tklog.V(4).Info(\"Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet\")\n--\n2.34.1\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.28.0/0000-Kubelite-integration.patch", "content": "From 3162aa9df25819b60a3c0a3b044394639d55280c Mon Sep 17 00:00:00 2001\nFrom: Konstantinos Tsakalozos \nDate: Wed, 3 Mar 2021 18:19:37 +0200\nSubject: [PATCH] Kubelite integration\n\n---\n cmd/kube-apiserver/app/server.go | 9 +++-\n cmd/kube-scheduler/app/server.go | 6 ++-\n cmd/kubelet/app/server.go | 13 +++--\n cmd/kubelite/app/daemons/daemon.go | 84 +++++++++++++++++++++++++++++\n cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/app/server.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/kubelite.go | 28 ++++++++++\n pkg/volume/csi/csi_plugin.go | 10 ++--\n 8 files changed, 297 insertions(+), 11 deletions(-)\n create mode 100644 cmd/kubelite/app/daemons/daemon.go\n create mode 100644 cmd/kubelite/app/options/options.go\n create mode 100644 cmd/kubelite/app/server.go\n create mode 100644 cmd/kubelite/kubelite.go\n\ndiff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go\nindex b021bac2574..8d5a1bad8ed 100644\n--- a/cmd/kube-apiserver/app/server.go\n+++ b/cmd/kube-apiserver/app/server.go\n@@ -76,7 +76,7 @@ func init() {\n }\n\n // NewAPIServerCommand creates a *cobra.Command object with default parameters\n-func NewAPIServerCommand() *cobra.Command {\n+func NewAPIServerCommand(stopCh... <- chan struct{}) *cobra.Command {\n \ts := options.NewServerRunOptions()\n \tcmd := &cobra.Command{\n \t\tUse: \"kube-apiserver\",\n@@ -116,7 +116,12 @@ cluster's shared state through which all other components interact.`,\n \t\t\t}\n \t\t\t// add feature enablement metrics\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n-\t\t\treturn Run(completedOptions, genericapiserver.SetupSignalHandler())\n+\n+\t\t\tif len(stopCh) != 0 {\n+\t\t\t\treturn Run(completedOptions, stopCh[0])\n+\t\t\t} else {\n+\t\t\t\treturn Run(completedOptions, genericapiserver.SetupSignalHandler())\n+\t\t\t}\n \t\t},\n \t\tArgs: func(cmd *cobra.Command, args []string) error {\n \t\t\tfor _, arg := range args {\ndiff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go\nindex c48b09a420d..b7f273b02ac 100644\n--- a/cmd/kube-scheduler/app/server.go\n+++ b/cmd/kube-scheduler/app/server.go\n@@ -132,7 +132,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op\n \tctx, cancel := context.WithCancel(context.Background())\n \tdefer cancel()\n \tgo func() {\n-\t\tstopCh := server.SetupSignalHandler()\n+ c := cmd.Context()\n+ if c == nil {\n+ c = server.SetupSignalContext()\n+ }\n+ stopCh := c.Done()\n \t\t<-stopCh\n \t\tcancel()\n \t}()\ndiff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go\nindex 742524e325f..561cbf68868 100644\n--- a/cmd/kubelet/app/server.go\n+++ b/cmd/kubelet/app/server.go\n@@ -122,7 +122,7 @@ const (\n )\n\n // NewKubeletCommand creates a *cobra.Command object with default parameters\n-func NewKubeletCommand() *cobra.Command {\n+func NewKubeletCommand(ctx ...context.Context) *cobra.Command {\n \tcleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)\n \tcleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n \tkubeletFlags := options.NewKubeletFlags()\n@@ -249,6 +249,12 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\tif err := checkPermissions(); err != nil {\n \t\t\t\tklog.ErrorS(err, \"kubelet running with insufficient permissions\")\n \t\t\t}\n+\t\t\trunctx := context.Background()\n+\t\t\tif len(ctx) == 0 {\n+\t\t\t\trunctx = genericapiserver.SetupSignalContext()\n+\t\t\t} else {\n+\t\t\t\trunctx = ctx[0]\n+\t\t\t}\n\n \t\t\t// make the kubelet's config safe for logging\n \t\t\tconfig := kubeletServer.KubeletConfiguration.DeepCopy()\n@@ -258,12 +264,9 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\t// log the kubelet's config for inspection\n \t\t\tklog.V(5).InfoS(\"KubeletConfiguration\", \"configuration\", klog.Format(config))\n\n-\t\t\t// set up signal context for kubelet shutdown\n-\t\t\tctx := genericapiserver.SetupSignalContext()\n-\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n \t\t\t// run the kubelet\n-\t\t\treturn Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n+\t\t\treturn Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n \t\t},\n \t}\n\ndiff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go\nnew file mode 100644\nindex 00000000000..dbef03cf07e\n--- /dev/null\n+++ b/cmd/kubelite/app/daemons/daemon.go\n@@ -0,0 +1,84 @@\n+package daemon\n+\n+import (\n+\t\"context\"\n+\t\"k8s.io/client-go/kubernetes\"\n+\t\"k8s.io/client-go/tools/clientcmd\"\n+\t\"k8s.io/klog/v2\"\n+\tgenericcontrollermanager \"k8s.io/controller-manager/app\"\n+\tapiserver \"k8s.io/kubernetes/cmd/kube-apiserver/app\"\n+\tcontroller \"k8s.io/kubernetes/cmd/kube-controller-manager/app\"\n+\tproxy \"k8s.io/kubernetes/cmd/kube-proxy/app\"\n+\tscheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n+\tkubelet \"k8s.io/kubernetes/cmd/kubelet/app\"\n+\n+\t\"time\"\n+)\n+\n+func StartControllerManager(args []string, ctx context.Context) {\n+\tcommand := controller.NewControllerManagerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Controller Manager\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Controller Manager exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Controller Manager\")\n+}\n+\n+func StartScheduler(args []string, ctx context.Context) {\n+\tcommand := scheduler.NewSchedulerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Scheduler\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Scheduler exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Scheduler\")\n+}\n+\n+func StartProxy(args []string) {\n+\tcommand := proxy.NewProxyCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Proxy\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Proxy exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Proxy\")\n+}\n+\n+func StartKubelet(args []string, ctx context.Context) {\n+\tcommand := kubelet.NewKubeletCommand(ctx)\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Kubelet\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Kubelet exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Kubelet\")\n+}\n+\n+func StartAPIServer(args []string, ctx <-chan struct{}) {\n+\tcommand := apiserver.NewAPIServerCommand(ctx)\n+\tcommand.SetArgs(args)\n+\tklog.Info(\"Starting API Server\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"API Server exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping API Server\")\n+}\n+\n+func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) {\n+\tklog.Info(\"Waiting for the API server\")\n+\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigpath)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not find the cluster's kubeconfig file %v\", err)\n+\t}\n+\t// create the client\n+\tclient, err := kubernetes.NewForConfig(config)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not create client to the cluster %v\", err)\n+\t}\n+\tgenericcontrollermanager.WaitForAPIServer(client, timeout)\n+}\n\\ No newline at end of file\ndiff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go\nnew file mode 100644\nindex 00000000000..80f1d8b09fc\n--- /dev/null\n+++ b/cmd/kubelite/app/options/options.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright 2018 The Kubernetes Authors.\n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+\n+package options\n+\n+import (\n+\t\"bufio\"\n+\t\"k8s.io/klog/v2\"\n+\t\"os\"\n+\t\"strings\"\n+)\n+\n+// Options has all the params needed to run a Kubelite\n+type Options struct {\n+\tSchedulerArgsFile string\n+\tControllerManagerArgsFile string\n+\tProxyArgsFile string\n+\tKubeletArgsFile string\n+\tAPIServerArgsFile string\n+\tKubeconfigFile \t\t string\n+\tStartControlPlane\t\t bool\n+}\n+\n+func NewOptions() (*Options){\n+\to := Options{\n+\t\t\"/var/snap/microk8s/current/args/kube-scheduler\",\n+\t\t\"/var/snap/microk8s/current/args/kube-controller-manager\",\n+\t\t\"/var/snap/microk8s/current/args/kube-proxy\",\n+\t\t\"/var/snap/microk8s/current/args/kubelet\",\n+\t\t\"/var/snap/microk8s/current/args/kube-apiserver\",\n+\t\t\"/var/snap/microk8s/current/credentials/client.config\",\n+\t\ttrue,\n+\t}\n+\treturn &o\n+}\n+\n+func ReadArgsFromFile(filename string) []string {\n+\tvar args []string\n+\tfile, err := os.Open(filename)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"Failed to open arguments file %v\", err)\n+\t}\n+\tdefer file.Close()\n+\n+\tscanner := bufio.NewScanner(file)\n+\tfor scanner.Scan() {\n+\t\tline := scanner.Text()\n+\t\tline = strings.TrimSpace(line)\n+\t\t// ignore lines with # and empty lines\n+\t\tif len(line) <= 0 || strings.HasPrefix(line, \"#\") {\n+\t\t\tcontinue\n+\t\t}\n+\t\t// remove \" and '\n+\t\tfor _, r := range \"\\\"'\" {\n+\t\t\tline = strings.ReplaceAll(line, string(r), \"\")\n+\t\t}\n+\t\tfor _, part := range strings.Split(line, \" \") {\n+\n+\t\t\targs = append(args, os.ExpandEnv(part))\n+\t\t}\n+\t}\n+\tif err := scanner.Err(); err != nil {\n+\t\tklog.Fatalf(\"Failed to read arguments file %v\", err)\n+\t}\n+\treturn args\n+}\ndiff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go\nnew file mode 100644\nindex 00000000000..e7452a09e3e\n--- /dev/null\n+++ b/cmd/kubelite/app/server.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright © 2020 NAME HERE \n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+package app\n+\n+import (\n+\t\"fmt\"\n+\t\"github.com/spf13/cobra\"\n+\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n+\tdaemon \"k8s.io/kubernetes/cmd/kubelite/app/daemons\"\n+\t\"k8s.io/kubernetes/cmd/kubelite/app/options\"\n+\t\"os\"\n+\t\"time\"\n+)\n+\n+var opts = options.NewOptions()\n+\n+// liteCmd represents the base command when called without any subcommands\n+var liteCmd = &cobra.Command{\n+\tUse: \"kubelite\",\n+\tShort: \"Single server kubernetes\",\n+\tLong: `A single server that spawns all other kubernetes servers as threads`,\n+\t// Uncomment the following line if your bare application\n+\t// has an action associated with it:\n+\tRun: func(cmd *cobra.Command, args []string) {\n+\t\tctx := genericapiserver.SetupSignalContext()\n+\n+\t\tif opts.StartControlPlane {\n+\t\t\tapiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile)\n+\t\t\tgo daemon.StartAPIServer(apiserverArgs, ctx.Done())\n+\t\t\tdaemon.WaitForAPIServer(opts.KubeconfigFile, 360 * time.Second)\n+\n+\t\t\tcontrollerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile)\n+\t\t\tgo daemon.StartControllerManager(controllerArgs, ctx)\n+\n+\t\t\tschedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile)\n+\t\t\tgo daemon.StartScheduler(schedulerArgs, ctx)\n+\t\t}\n+\n+\t\tproxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile)\n+\t\tgo daemon.StartProxy(proxyArgs)\n+\n+\t\tkubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile)\n+\t\tdaemon.StartKubelet(kubeletArgs, ctx)\n+\t},\n+}\n+\n+// Execute adds all child commands to the root command and sets flags appropriately.\n+// This is called by main.main(). It only needs to happen once to the liteCmd.\n+func Execute() {\n+\tif err := liteCmd.Execute(); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+}\n+\n+func init() {\n+\tcobra.OnInitialize()\n+\n+\tliteCmd.Flags().StringVar(&opts.SchedulerArgsFile, \"scheduler-args-file\", opts.SchedulerArgsFile, \"file with the arguments for the scheduler\")\n+\tliteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, \"controller-manager-args-file\", opts.ControllerManagerArgsFile, \"file with the arguments for the controller manager\")\n+\tliteCmd.Flags().StringVar(&opts.ProxyArgsFile, \"proxy-args-file\", opts.ProxyArgsFile , \"file with the arguments for kube-proxy\")\n+\tliteCmd.Flags().StringVar(&opts.KubeletArgsFile, \"kubelet-args-file\", opts.KubeletArgsFile, \"file with the arguments for kubelet\")\n+\tliteCmd.Flags().StringVar(&opts.APIServerArgsFile, \"apiserver-args-file\", opts.APIServerArgsFile, \"file with the arguments for the API server\")\n+\tliteCmd.Flags().StringVar(&opts.KubeconfigFile , \"kubeconfig-file\", opts.KubeconfigFile, \"the kubeconfig file to use to healthcheck the API server\")\n+\tliteCmd.Flags().BoolVar(&opts.StartControlPlane, \"start-control-plane\", opts.StartControlPlane, \"start the control plane (API server, scheduler and controller manager)\")\n+}\ndiff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go\nnew file mode 100644\nindex 00000000000..667b24f68e6\n--- /dev/null\n+++ b/cmd/kubelite/kubelite.go\n@@ -0,0 +1,28 @@\n+package main\n+\n+import (\n+\t\"github.com/spf13/pflag\"\n+\tcliflag \"k8s.io/component-base/cli/flag\"\n+\t\"math/rand\"\n+\t\"time\"\n+\n+\t\"k8s.io/component-base/logs\"\n+\t_ \"k8s.io/component-base/metrics/prometheus/clientgo\" // load all the prometheus client-go plugin\n+\t_ \"k8s.io/component-base/metrics/prometheus/version\" // for version metric registration\n+\t\"k8s.io/kubernetes/cmd/kubelite/app\"\n+)\n+\n+func main() {\n+\tprintln(\"Starting kubelite\")\n+\trand.Seed(time.Now().UnixNano())\n+\t// TODO: once we switch everything over to Cobra commands, we can go back to calling\n+\t// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the\n+\t// normalize func and add the go flag set by hand.\n+\tpflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n+\t// utilflag.InitFlags()\n+\tlogs.InitLogs()\n+\tdefer logs.FlushLogs()\n+\n+\tapp.Execute()\n+\tprintln(\"Stopping kubelite\")\n+}\ndiff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go\nindex 2556517276e..0b5ef45d083 100644\n--- a/pkg/volume/csi/csi_plugin.go\n+++ b/pkg/volume/csi/csi_plugin.go\n@@ -243,18 +243,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {\n \t}\n\n \t// Initializing the label management channels\n-\tnim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n+\tlocalNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n\n \t// This function prevents Kubelet from posting Ready status until CSINode\n \t// is both installed and initialized\n-\tif err := initializeCSINode(host); err != nil {\n+\tif err := initializeCSINode(host, localNim); err != nil {\n \t\treturn errors.New(log(\"failed to initialize CSINode: %v\", err))\n \t}\n\n+\tif _, ok := host.(volume.KubeletVolumeHost); ok {\n+\t\tnim = localNim\n+\t}\n+\n \treturn nil\n }\n\n-func initializeCSINode(host volume.VolumeHost) error {\n+func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error {\n \tkvh, ok := host.(volume.KubeletVolumeHost)\n \tif !ok {\n \t\tklog.V(4).Info(\"Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet\")\n--\n2.34.1\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.28.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch", "content": "From 55f4864d816c8e7ca0ebb39571dc88dbdf05eff2 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 27 Jul 2023 18:08:00 +0300\nSubject: [PATCH] Set log reapply handling to ignore unchanged\n\n---\n staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n\ndiff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go\nindex 2db9b1f5382..e0824dcdc4e 100644\n--- a/staging/src/k8s.io/component-base/logs/api/v1/options.go\n+++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go\n@@ -64,7 +64,7 @@ func NewLoggingConfiguration() *LoggingConfiguration {\n // are no goroutines which might call logging functions. The default for ValidateAndApply\n // and ValidateAndApplyWithOptions is to return an error when called more than once.\n // Binaries and unit tests can override that behavior.\n-var ReapplyHandling = ReapplyHandlingError\n+var ReapplyHandling = ReapplyHandlingIgnoreUnchanged\n\n type ReapplyHandlingType int\n\n--\n2.34.1\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.31.0/0000-Kubelite-integration.patch", "content": "From d261b947963b9e808725a21e3d55e52c20826e22 Mon Sep 17 00:00:00 2001\nFrom: Konstantinos Tsakalozos \nDate: Wed, 3 Mar 2021 18:19:37 +0200\nSubject: [PATCH] Kubelite integration\n\n---\n cmd/kube-apiserver/app/server.go | 9 ++-\n cmd/kube-scheduler/app/server.go | 6 +-\n cmd/kubelet/app/server.go | 13 +++--\n cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++++++++++++++\n cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/app/server.go | 80 +++++++++++++++++++++++++++\n cmd/kubelite/kubelite.go | 25 +++++++++\n pkg/volume/csi/csi_plugin.go | 10 +++-\n 8 files changed, 296 insertions(+), 11 deletions(-)\n create mode 100644 cmd/kubelite/app/daemons/daemon.go\n create mode 100644 cmd/kubelite/app/options/options.go\n create mode 100644 cmd/kubelite/app/server.go\n create mode 100644 cmd/kubelite/kubelite.go\n\ndiff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go\nindex fad0f1e9579..5e291005671 100644\n--- a/cmd/kube-apiserver/app/server.go\n+++ b/cmd/kube-apiserver/app/server.go\n@@ -63,7 +63,7 @@ func init() {\n }\n \n // NewAPIServerCommand creates a *cobra.Command object with default parameters\n-func NewAPIServerCommand() *cobra.Command {\n+func NewAPIServerCommand(ctx ...context.Context) *cobra.Command {\n \t_, featureGate := utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(\n \t\tutilversion.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)\n \ts := options.NewServerRunOptions()\n@@ -119,7 +119,12 @@ cluster's shared state through which all other components interact.`,\n \t\t\treturn nil\n \t\t},\n \t}\n-\tcmd.SetContext(genericapiserver.SetupSignalContext())\n+\n+\tif len(ctx) == 0 {\n+\t\tcmd.SetContext(genericapiserver.SetupSignalContext())\n+\t} else {\n+\t\tcmd.SetContext(ctx[0])\n+\t}\n \n \tfs := cmd.Flags()\n \tnamedFlagSets := s.Flags()\ndiff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go\nindex 7b08b119b4b..5d2a5a5f51a 100644\n--- a/cmd/kube-scheduler/app/server.go\n+++ b/cmd/kube-scheduler/app/server.go\n@@ -145,7 +145,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op\n \tctx, cancel := context.WithCancel(context.Background())\n \tdefer cancel()\n \tgo func() {\n-\t\tstopCh := server.SetupSignalHandler()\n+ c := cmd.Context()\n+ if c == nil {\n+ c = server.SetupSignalContext()\n+ }\n+ stopCh := c.Done()\n \t\t<-stopCh\n \t\tcancel()\n \t}()\ndiff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go\nindex 2d90030d210..666996e2ecb 100644\n--- a/cmd/kubelet/app/server.go\n+++ b/cmd/kubelet/app/server.go\n@@ -129,7 +129,7 @@ const (\n )\n \n // NewKubeletCommand creates a *cobra.Command object with default parameters\n-func NewKubeletCommand() *cobra.Command {\n+func NewKubeletCommand(ctx ...context.Context) *cobra.Command {\n \tcleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)\n \tcleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n \tkubeletFlags := options.NewKubeletFlags()\n@@ -266,6 +266,12 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\tif err := checkPermissions(); err != nil {\n \t\t\t\tklog.ErrorS(err, \"kubelet running with insufficient permissions\")\n \t\t\t}\n+\t\t\trunctx := context.Background()\n+\t\t\tif len(ctx) == 0 {\n+\t\t\t\trunctx = genericapiserver.SetupSignalContext()\n+\t\t\t} else {\n+\t\t\t\trunctx = ctx[0]\n+\t\t\t}\n \n \t\t\t// make the kubelet's config safe for logging\n \t\t\tconfig := kubeletServer.KubeletConfiguration.DeepCopy()\n@@ -275,12 +281,9 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\t// log the kubelet's config for inspection\n \t\t\tklog.V(5).InfoS(\"KubeletConfiguration\", \"configuration\", klog.Format(config))\n \n-\t\t\t// set up signal context for kubelet shutdown\n-\t\t\tctx := genericapiserver.SetupSignalContext()\n-\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n \t\t\t// run the kubelet\n-\t\t\treturn Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n+\t\t\treturn Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n \t\t},\n \t}\n \ndiff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go\nnew file mode 100644\nindex 00000000000..46c1af7fdb9\n--- /dev/null\n+++ b/cmd/kubelite/app/daemons/daemon.go\n@@ -0,0 +1,85 @@\n+package daemon\n+\n+import (\n+\t\"context\"\n+\n+\t\"k8s.io/client-go/kubernetes\"\n+\t\"k8s.io/client-go/tools/clientcmd\"\n+\tgenericcontrollermanager \"k8s.io/controller-manager/app\"\n+\t\"k8s.io/klog/v2\"\n+\tapiserver \"k8s.io/kubernetes/cmd/kube-apiserver/app\"\n+\tcontroller \"k8s.io/kubernetes/cmd/kube-controller-manager/app\"\n+\tproxy \"k8s.io/kubernetes/cmd/kube-proxy/app\"\n+\tscheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n+\tkubelet \"k8s.io/kubernetes/cmd/kubelet/app\"\n+\n+\t\"time\"\n+)\n+\n+func StartControllerManager(args []string, ctx context.Context) {\n+\tcommand := controller.NewControllerManagerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Controller Manager\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Controller Manager exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Controller Manager\")\n+}\n+\n+func StartScheduler(args []string, ctx context.Context) {\n+\tcommand := scheduler.NewSchedulerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Scheduler\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Scheduler exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Scheduler\")\n+}\n+\n+func StartProxy(args []string) {\n+\tcommand := proxy.NewProxyCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Proxy\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Proxy exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Proxy\")\n+}\n+\n+func StartKubelet(args []string, ctx context.Context) {\n+\tcommand := kubelet.NewKubeletCommand(ctx)\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Kubelet\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Kubelet exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Kubelet\")\n+}\n+\n+func StartAPIServer(args []string, ctx context.Context) {\n+\tcommand := apiserver.NewAPIServerCommand(ctx)\n+\tcommand.SetArgs(args)\n+\tklog.Info(\"Starting API Server\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"API Server exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping API Server\")\n+}\n+\n+func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) {\n+\tklog.Info(\"Waiting for the API server\")\n+\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigpath)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not find the cluster's kubeconfig file %v\", err)\n+\t}\n+\t// create the client\n+\tclient, err := kubernetes.NewForConfig(config)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not create client to the cluster %v\", err)\n+\t}\n+\tgenericcontrollermanager.WaitForAPIServer(client, timeout)\n+}\ndiff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go\nnew file mode 100644\nindex 00000000000..80f1d8b09fc\n--- /dev/null\n+++ b/cmd/kubelite/app/options/options.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright 2018 The Kubernetes Authors.\n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+\n+package options\n+\n+import (\n+\t\"bufio\"\n+\t\"k8s.io/klog/v2\"\n+\t\"os\"\n+\t\"strings\"\n+)\n+\n+// Options has all the params needed to run a Kubelite\n+type Options struct {\n+\tSchedulerArgsFile string\n+\tControllerManagerArgsFile string\n+\tProxyArgsFile string\n+\tKubeletArgsFile string\n+\tAPIServerArgsFile string\n+\tKubeconfigFile \t\t string\n+\tStartControlPlane\t\t bool\n+}\n+\n+func NewOptions() (*Options){\n+\to := Options{\n+\t\t\"/var/snap/microk8s/current/args/kube-scheduler\",\n+\t\t\"/var/snap/microk8s/current/args/kube-controller-manager\",\n+\t\t\"/var/snap/microk8s/current/args/kube-proxy\",\n+\t\t\"/var/snap/microk8s/current/args/kubelet\",\n+\t\t\"/var/snap/microk8s/current/args/kube-apiserver\",\n+\t\t\"/var/snap/microk8s/current/credentials/client.config\",\n+\t\ttrue,\n+\t}\n+\treturn &o\n+}\n+\n+func ReadArgsFromFile(filename string) []string {\n+\tvar args []string\n+\tfile, err := os.Open(filename)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"Failed to open arguments file %v\", err)\n+\t}\n+\tdefer file.Close()\n+\n+\tscanner := bufio.NewScanner(file)\n+\tfor scanner.Scan() {\n+\t\tline := scanner.Text()\n+\t\tline = strings.TrimSpace(line)\n+\t\t// ignore lines with # and empty lines\n+\t\tif len(line) <= 0 || strings.HasPrefix(line, \"#\") {\n+\t\t\tcontinue\n+\t\t}\n+\t\t// remove \" and '\n+\t\tfor _, r := range \"\\\"'\" {\n+\t\t\tline = strings.ReplaceAll(line, string(r), \"\")\n+\t\t}\n+\t\tfor _, part := range strings.Split(line, \" \") {\n+\n+\t\t\targs = append(args, os.ExpandEnv(part))\n+\t\t}\n+\t}\n+\tif err := scanner.Err(); err != nil {\n+\t\tklog.Fatalf(\"Failed to read arguments file %v\", err)\n+\t}\n+\treturn args\n+}\ndiff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go\nnew file mode 100644\nindex 00000000000..4ff36cd6432\n--- /dev/null\n+++ b/cmd/kubelite/app/server.go\n@@ -0,0 +1,80 @@\n+/*\n+Copyright © 2020 NAME HERE \n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+\thttp://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+package app\n+\n+import (\n+\t\"fmt\"\n+\t\"os\"\n+\t\"time\"\n+\n+\t\"github.com/spf13/cobra\"\n+\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n+\tdaemon \"k8s.io/kubernetes/cmd/kubelite/app/daemons\"\n+\t\"k8s.io/kubernetes/cmd/kubelite/app/options\"\n+)\n+\n+var opts = options.NewOptions()\n+\n+// liteCmd represents the base command when called without any subcommands\n+var liteCmd = &cobra.Command{\n+\tUse: \"kubelite\",\n+\tShort: \"Single server kubernetes\",\n+\tLong: `A single server that spawns all other kubernetes servers as threads`,\n+\t// Uncomment the following line if your bare application\n+\t// has an action associated with it:\n+\tRun: func(cmd *cobra.Command, args []string) {\n+\t\tctx := genericapiserver.SetupSignalContext()\n+\n+\t\tif opts.StartControlPlane {\n+\t\t\tapiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile)\n+\t\t\tgo daemon.StartAPIServer(apiserverArgs, ctx)\n+\t\t\tdaemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second)\n+\n+\t\t\tcontrollerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile)\n+\t\t\tgo daemon.StartControllerManager(controllerArgs, ctx)\n+\n+\t\t\tschedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile)\n+\t\t\tgo daemon.StartScheduler(schedulerArgs, ctx)\n+\t\t}\n+\n+\t\tproxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile)\n+\t\tgo daemon.StartProxy(proxyArgs)\n+\n+\t\tkubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile)\n+\t\tdaemon.StartKubelet(kubeletArgs, ctx)\n+\t},\n+}\n+\n+// Execute adds all child commands to the root command and sets flags appropriately.\n+// This is called by main.main(). It only needs to happen once to the liteCmd.\n+func Execute() {\n+\tif err := liteCmd.Execute(); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+}\n+\n+func init() {\n+\tcobra.OnInitialize()\n+\n+\tliteCmd.Flags().StringVar(&opts.SchedulerArgsFile, \"scheduler-args-file\", opts.SchedulerArgsFile, \"file with the arguments for the scheduler\")\n+\tliteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, \"controller-manager-args-file\", opts.ControllerManagerArgsFile, \"file with the arguments for the controller manager\")\n+\tliteCmd.Flags().StringVar(&opts.ProxyArgsFile, \"proxy-args-file\", opts.ProxyArgsFile, \"file with the arguments for kube-proxy\")\n+\tliteCmd.Flags().StringVar(&opts.KubeletArgsFile, \"kubelet-args-file\", opts.KubeletArgsFile, \"file with the arguments for kubelet\")\n+\tliteCmd.Flags().StringVar(&opts.APIServerArgsFile, \"apiserver-args-file\", opts.APIServerArgsFile, \"file with the arguments for the API server\")\n+\tliteCmd.Flags().StringVar(&opts.KubeconfigFile, \"kubeconfig-file\", opts.KubeconfigFile, \"the kubeconfig file to use to healthcheck the API server\")\n+\tliteCmd.Flags().BoolVar(&opts.StartControlPlane, \"start-control-plane\", opts.StartControlPlane, \"start the control plane (API server, scheduler and controller manager)\")\n+}\ndiff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go\nnew file mode 100644\nindex 00000000000..30ab604f480\n--- /dev/null\n+++ b/cmd/kubelite/kubelite.go\n@@ -0,0 +1,25 @@\n+package main\n+\n+import (\n+\t\"github.com/spf13/pflag\"\n+\tcliflag \"k8s.io/component-base/cli/flag\"\n+\n+\t\"k8s.io/component-base/logs\"\n+\t_ \"k8s.io/component-base/metrics/prometheus/clientgo\" // load all the prometheus client-go plugin\n+\t_ \"k8s.io/component-base/metrics/prometheus/version\" // for version metric registration\n+\t\"k8s.io/kubernetes/cmd/kubelite/app\"\n+)\n+\n+func main() {\n+\tprintln(\"Starting kubelite\")\n+\t// TODO: once we switch everything over to Cobra commands, we can go back to calling\n+\t// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the\n+\t// normalize func and add the go flag set by hand.\n+\tpflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n+\t// utilflag.InitFlags()\n+\tlogs.InitLogs()\n+\tdefer logs.FlushLogs()\n+\n+\tapp.Execute()\n+\tprintln(\"Stopping kubelite\")\n+}\ndiff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go\nindex fec8a34b4d3..523eb78f05c 100644\n--- a/pkg/volume/csi/csi_plugin.go\n+++ b/pkg/volume/csi/csi_plugin.go\n@@ -253,18 +253,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {\n \t}\n \n \t// Initializing the label management channels\n-\tnim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n+\tlocalNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n \n \t// This function prevents Kubelet from posting Ready status until CSINode\n \t// is both installed and initialized\n-\tif err := initializeCSINode(host); err != nil {\n+\tif err := initializeCSINode(host, localNim); err != nil {\n \t\treturn errors.New(log(\"failed to initialize CSINode: %v\", err))\n \t}\n \n+\tif _, ok := host.(volume.KubeletVolumeHost); ok {\n+\t\tnim = localNim\n+\t}\n+\n \treturn nil\n }\n \n-func initializeCSINode(host volume.VolumeHost) error {\n+func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error {\n \tkvh, ok := host.(volume.KubeletVolumeHost)\n \tif !ok {\n \t\tklog.V(4).Info(\"Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet\")\n-- \n2.34.1\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.31.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch", "content": "From 55f4864d816c8e7ca0ebb39571dc88dbdf05eff2 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 27 Jul 2023 18:08:00 +0300\nSubject: [PATCH] Set log reapply handling to ignore unchanged\n\n---\n staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n\ndiff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go\nindex 2db9b1f5382..e0824dcdc4e 100644\n--- a/staging/src/k8s.io/component-base/logs/api/v1/options.go\n+++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go\n@@ -64,7 +64,7 @@ func NewLoggingConfiguration() *LoggingConfiguration {\n // are no goroutines which might call logging functions. The default for ValidateAndApply\n // and ValidateAndApplyWithOptions is to return an error when called more than once.\n // Binaries and unit tests can override that behavior.\n-var ReapplyHandling = ReapplyHandlingError\n+var ReapplyHandling = ReapplyHandlingIgnoreUnchanged\n\n type ReapplyHandlingType int\n\n--\n2.34.1\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.32.0/0000-Kubelite-integration.patch", "content": "From 819b718ecfee8c4f6fb503d0dea80a43e86cdb6f Mon Sep 17 00:00:00 2001\nFrom: Konstantinos Tsakalozos \nDate: Wed, 3 Mar 2021 18:19:37 +0200\nSubject: [PATCH] Kubelite integration\n\n---\n cmd/kube-apiserver/app/server.go | 9 ++-\n cmd/kube-scheduler/app/server.go | 6 +-\n cmd/kubelet/app/server.go | 13 +++--\n cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++++++++++++++\n cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/app/server.go | 80 +++++++++++++++++++++++++++\n cmd/kubelite/kubelite.go | 25 +++++++++\n pkg/volume/csi/csi_plugin.go | 10 +++-\n 8 files changed, 296 insertions(+), 11 deletions(-)\n create mode 100644 cmd/kubelite/app/daemons/daemon.go\n create mode 100644 cmd/kubelite/app/options/options.go\n create mode 100644 cmd/kubelite/app/server.go\n create mode 100644 cmd/kubelite/kubelite.go\n\ndiff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go\nindex 8aa05a4d8f8..75b982aaff9 100644\n--- a/cmd/kube-apiserver/app/server.go\n+++ b/cmd/kube-apiserver/app/server.go\n@@ -63,11 +63,16 @@ func init() {\n }\n \n // NewAPIServerCommand creates a *cobra.Command object with default parameters\n-func NewAPIServerCommand() *cobra.Command {\n+func NewAPIServerCommand(ctxs ...context.Context) *cobra.Command {\n \t_, featureGate := featuregate.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(\n \t\tfeaturegate.DefaultKubeComponent, utilversion.DefaultBuildEffectiveVersion(), utilfeature.DefaultMutableFeatureGate)\n \ts := options.NewServerRunOptions()\n-\tctx := genericapiserver.SetupSignalContext()\n+\tctx := context.Background()\n+\tif len(ctxs) == 0 {\n+\t\tctx = genericapiserver.SetupSignalContext()\n+\t} else {\n+\t\tctx = ctxs[0]\n+\t}\n \n \tcmd := &cobra.Command{\n \t\tUse: \"kube-apiserver\",\ndiff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go\nindex 1785bbdcc91..64f01ba5c93 100644\n--- a/cmd/kube-scheduler/app/server.go\n+++ b/cmd/kube-scheduler/app/server.go\n@@ -144,7 +144,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op\n \tctx, cancel := context.WithCancel(context.Background())\n \tdefer cancel()\n \tgo func() {\n-\t\tstopCh := server.SetupSignalHandler()\n+ c := cmd.Context()\n+ if c == nil {\n+ c = server.SetupSignalContext()\n+ }\n+ stopCh := c.Done()\n \t\t<-stopCh\n \t\tcancel()\n \t}()\ndiff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go\nindex cfcf6e7d5cd..9c3853173ff 100644\n--- a/cmd/kubelet/app/server.go\n+++ b/cmd/kubelet/app/server.go\n@@ -138,7 +138,7 @@ const (\n )\n \n // NewKubeletCommand creates a *cobra.Command object with default parameters\n-func NewKubeletCommand() *cobra.Command {\n+func NewKubeletCommand(ctx ...context.Context) *cobra.Command {\n \tcleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)\n \tcleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n \tkubeletFlags := options.NewKubeletFlags()\n@@ -275,6 +275,12 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\tif err := checkPermissions(); err != nil {\n \t\t\t\tklog.ErrorS(err, \"kubelet running with insufficient permissions\")\n \t\t\t}\n+\t\t\trunctx := context.Background()\n+\t\t\tif len(ctx) == 0 {\n+\t\t\t\trunctx = genericapiserver.SetupSignalContext()\n+\t\t\t} else {\n+\t\t\t\trunctx = ctx[0]\n+\t\t\t}\n \n \t\t\t// make the kubelet's config safe for logging\n \t\t\tconfig := kubeletServer.KubeletConfiguration.DeepCopy()\n@@ -284,12 +290,9 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\t// log the kubelet's config for inspection\n \t\t\tklog.V(5).InfoS(\"KubeletConfiguration\", \"configuration\", klog.Format(config))\n \n-\t\t\t// set up signal context for kubelet shutdown\n-\t\t\tctx := genericapiserver.SetupSignalContext()\n-\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n \t\t\t// run the kubelet\n-\t\t\treturn Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n+\t\t\treturn Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n \t\t},\n \t}\n \ndiff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go\nnew file mode 100644\nindex 00000000000..46c1af7fdb9\n--- /dev/null\n+++ b/cmd/kubelite/app/daemons/daemon.go\n@@ -0,0 +1,85 @@\n+package daemon\n+\n+import (\n+\t\"context\"\n+\n+\t\"k8s.io/client-go/kubernetes\"\n+\t\"k8s.io/client-go/tools/clientcmd\"\n+\tgenericcontrollermanager \"k8s.io/controller-manager/app\"\n+\t\"k8s.io/klog/v2\"\n+\tapiserver \"k8s.io/kubernetes/cmd/kube-apiserver/app\"\n+\tcontroller \"k8s.io/kubernetes/cmd/kube-controller-manager/app\"\n+\tproxy \"k8s.io/kubernetes/cmd/kube-proxy/app\"\n+\tscheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n+\tkubelet \"k8s.io/kubernetes/cmd/kubelet/app\"\n+\n+\t\"time\"\n+)\n+\n+func StartControllerManager(args []string, ctx context.Context) {\n+\tcommand := controller.NewControllerManagerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Controller Manager\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Controller Manager exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Controller Manager\")\n+}\n+\n+func StartScheduler(args []string, ctx context.Context) {\n+\tcommand := scheduler.NewSchedulerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Scheduler\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Scheduler exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Scheduler\")\n+}\n+\n+func StartProxy(args []string) {\n+\tcommand := proxy.NewProxyCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Proxy\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Proxy exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Proxy\")\n+}\n+\n+func StartKubelet(args []string, ctx context.Context) {\n+\tcommand := kubelet.NewKubeletCommand(ctx)\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Kubelet\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Kubelet exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Kubelet\")\n+}\n+\n+func StartAPIServer(args []string, ctx context.Context) {\n+\tcommand := apiserver.NewAPIServerCommand(ctx)\n+\tcommand.SetArgs(args)\n+\tklog.Info(\"Starting API Server\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"API Server exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping API Server\")\n+}\n+\n+func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) {\n+\tklog.Info(\"Waiting for the API server\")\n+\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigpath)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not find the cluster's kubeconfig file %v\", err)\n+\t}\n+\t// create the client\n+\tclient, err := kubernetes.NewForConfig(config)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not create client to the cluster %v\", err)\n+\t}\n+\tgenericcontrollermanager.WaitForAPIServer(client, timeout)\n+}\ndiff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go\nnew file mode 100644\nindex 00000000000..80f1d8b09fc\n--- /dev/null\n+++ b/cmd/kubelite/app/options/options.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright 2018 The Kubernetes Authors.\n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+\n+package options\n+\n+import (\n+\t\"bufio\"\n+\t\"k8s.io/klog/v2\"\n+\t\"os\"\n+\t\"strings\"\n+)\n+\n+// Options has all the params needed to run a Kubelite\n+type Options struct {\n+\tSchedulerArgsFile string\n+\tControllerManagerArgsFile string\n+\tProxyArgsFile string\n+\tKubeletArgsFile string\n+\tAPIServerArgsFile string\n+\tKubeconfigFile \t\t string\n+\tStartControlPlane\t\t bool\n+}\n+\n+func NewOptions() (*Options){\n+\to := Options{\n+\t\t\"/var/snap/microk8s/current/args/kube-scheduler\",\n+\t\t\"/var/snap/microk8s/current/args/kube-controller-manager\",\n+\t\t\"/var/snap/microk8s/current/args/kube-proxy\",\n+\t\t\"/var/snap/microk8s/current/args/kubelet\",\n+\t\t\"/var/snap/microk8s/current/args/kube-apiserver\",\n+\t\t\"/var/snap/microk8s/current/credentials/client.config\",\n+\t\ttrue,\n+\t}\n+\treturn &o\n+}\n+\n+func ReadArgsFromFile(filename string) []string {\n+\tvar args []string\n+\tfile, err := os.Open(filename)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"Failed to open arguments file %v\", err)\n+\t}\n+\tdefer file.Close()\n+\n+\tscanner := bufio.NewScanner(file)\n+\tfor scanner.Scan() {\n+\t\tline := scanner.Text()\n+\t\tline = strings.TrimSpace(line)\n+\t\t// ignore lines with # and empty lines\n+\t\tif len(line) <= 0 || strings.HasPrefix(line, \"#\") {\n+\t\t\tcontinue\n+\t\t}\n+\t\t// remove \" and '\n+\t\tfor _, r := range \"\\\"'\" {\n+\t\t\tline = strings.ReplaceAll(line, string(r), \"\")\n+\t\t}\n+\t\tfor _, part := range strings.Split(line, \" \") {\n+\n+\t\t\targs = append(args, os.ExpandEnv(part))\n+\t\t}\n+\t}\n+\tif err := scanner.Err(); err != nil {\n+\t\tklog.Fatalf(\"Failed to read arguments file %v\", err)\n+\t}\n+\treturn args\n+}\ndiff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go\nnew file mode 100644\nindex 00000000000..4ff36cd6432\n--- /dev/null\n+++ b/cmd/kubelite/app/server.go\n@@ -0,0 +1,80 @@\n+/*\n+Copyright © 2020 NAME HERE \n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+\thttp://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+package app\n+\n+import (\n+\t\"fmt\"\n+\t\"os\"\n+\t\"time\"\n+\n+\t\"github.com/spf13/cobra\"\n+\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n+\tdaemon \"k8s.io/kubernetes/cmd/kubelite/app/daemons\"\n+\t\"k8s.io/kubernetes/cmd/kubelite/app/options\"\n+)\n+\n+var opts = options.NewOptions()\n+\n+// liteCmd represents the base command when called without any subcommands\n+var liteCmd = &cobra.Command{\n+\tUse: \"kubelite\",\n+\tShort: \"Single server kubernetes\",\n+\tLong: `A single server that spawns all other kubernetes servers as threads`,\n+\t// Uncomment the following line if your bare application\n+\t// has an action associated with it:\n+\tRun: func(cmd *cobra.Command, args []string) {\n+\t\tctx := genericapiserver.SetupSignalContext()\n+\n+\t\tif opts.StartControlPlane {\n+\t\t\tapiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile)\n+\t\t\tgo daemon.StartAPIServer(apiserverArgs, ctx)\n+\t\t\tdaemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second)\n+\n+\t\t\tcontrollerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile)\n+\t\t\tgo daemon.StartControllerManager(controllerArgs, ctx)\n+\n+\t\t\tschedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile)\n+\t\t\tgo daemon.StartScheduler(schedulerArgs, ctx)\n+\t\t}\n+\n+\t\tproxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile)\n+\t\tgo daemon.StartProxy(proxyArgs)\n+\n+\t\tkubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile)\n+\t\tdaemon.StartKubelet(kubeletArgs, ctx)\n+\t},\n+}\n+\n+// Execute adds all child commands to the root command and sets flags appropriately.\n+// This is called by main.main(). It only needs to happen once to the liteCmd.\n+func Execute() {\n+\tif err := liteCmd.Execute(); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+}\n+\n+func init() {\n+\tcobra.OnInitialize()\n+\n+\tliteCmd.Flags().StringVar(&opts.SchedulerArgsFile, \"scheduler-args-file\", opts.SchedulerArgsFile, \"file with the arguments for the scheduler\")\n+\tliteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, \"controller-manager-args-file\", opts.ControllerManagerArgsFile, \"file with the arguments for the controller manager\")\n+\tliteCmd.Flags().StringVar(&opts.ProxyArgsFile, \"proxy-args-file\", opts.ProxyArgsFile, \"file with the arguments for kube-proxy\")\n+\tliteCmd.Flags().StringVar(&opts.KubeletArgsFile, \"kubelet-args-file\", opts.KubeletArgsFile, \"file with the arguments for kubelet\")\n+\tliteCmd.Flags().StringVar(&opts.APIServerArgsFile, \"apiserver-args-file\", opts.APIServerArgsFile, \"file with the arguments for the API server\")\n+\tliteCmd.Flags().StringVar(&opts.KubeconfigFile, \"kubeconfig-file\", opts.KubeconfigFile, \"the kubeconfig file to use to healthcheck the API server\")\n+\tliteCmd.Flags().BoolVar(&opts.StartControlPlane, \"start-control-plane\", opts.StartControlPlane, \"start the control plane (API server, scheduler and controller manager)\")\n+}\ndiff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go\nnew file mode 100644\nindex 00000000000..30ab604f480\n--- /dev/null\n+++ b/cmd/kubelite/kubelite.go\n@@ -0,0 +1,25 @@\n+package main\n+\n+import (\n+\t\"github.com/spf13/pflag\"\n+\tcliflag \"k8s.io/component-base/cli/flag\"\n+\n+\t\"k8s.io/component-base/logs\"\n+\t_ \"k8s.io/component-base/metrics/prometheus/clientgo\" // load all the prometheus client-go plugin\n+\t_ \"k8s.io/component-base/metrics/prometheus/version\" // for version metric registration\n+\t\"k8s.io/kubernetes/cmd/kubelite/app\"\n+)\n+\n+func main() {\n+\tprintln(\"Starting kubelite\")\n+\t// TODO: once we switch everything over to Cobra commands, we can go back to calling\n+\t// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the\n+\t// normalize func and add the go flag set by hand.\n+\tpflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n+\t// utilflag.InitFlags()\n+\tlogs.InitLogs()\n+\tdefer logs.FlushLogs()\n+\n+\tapp.Execute()\n+\tprintln(\"Stopping kubelite\")\n+}\ndiff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go\nindex 59fa6245b7f..6443dffd695 100644\n--- a/pkg/volume/csi/csi_plugin.go\n+++ b/pkg/volume/csi/csi_plugin.go\n@@ -256,18 +256,22 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {\n \t}\n \n \t// Initializing the label management channels\n-\tnim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n+\tlocalNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n \n \t// This function prevents Kubelet from posting Ready status until CSINode\n \t// is both installed and initialized\n-\tif err := initializeCSINode(host); err != nil {\n+\tif err := initializeCSINode(host, localNim); err != nil {\n \t\treturn errors.New(log(\"failed to initialize CSINode: %v\", err))\n \t}\n \n+\tif _, ok := host.(volume.KubeletVolumeHost); ok {\n+\t\tnim = localNim\n+\t}\n+\n \treturn nil\n }\n \n-func initializeCSINode(host volume.VolumeHost) error {\n+func initializeCSINode(host volume.VolumeHost, nim nodeinfomanager.Interface) error {\n \tkvh, ok := host.(volume.KubeletVolumeHost)\n \tif !ok {\n \t\tklog.V(4).Info(\"Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet\")\n-- \n2.43.0\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.32.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch", "content": "From 55f4864d816c8e7ca0ebb39571dc88dbdf05eff2 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 27 Jul 2023 18:08:00 +0300\nSubject: [PATCH] Set log reapply handling to ignore unchanged\n\n---\n staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n\ndiff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go\nindex 2db9b1f5382..e0824dcdc4e 100644\n--- a/staging/src/k8s.io/component-base/logs/api/v1/options.go\n+++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go\n@@ -64,7 +64,7 @@ func NewLoggingConfiguration() *LoggingConfiguration {\n // are no goroutines which might call logging functions. The default for ValidateAndApply\n // and ValidateAndApplyWithOptions is to return an error when called more than once.\n // Binaries and unit tests can override that behavior.\n-var ReapplyHandling = ReapplyHandlingError\n+var ReapplyHandling = ReapplyHandlingIgnoreUnchanged\n\n type ReapplyHandlingType int\n\n--\n2.34.1\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.33.0/0000-Kubelite-integration.patch", "content": "From aa3c3a427082e2a1ddd9f72790a8977efefee1cf Mon Sep 17 00:00:00 2001\nFrom: Konstantinos Tsakalozos \nDate: Wed, 3 Mar 2021 18:19:37 +0200\nSubject: [PATCH 1/2] Kubelite integration\n\n---\n cmd/kube-apiserver/app/server.go | 9 ++-\n cmd/kube-scheduler/app/server.go | 6 +-\n cmd/kubelet/app/server.go | 13 +++--\n cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++++++++++++++\n cmd/kubelite/app/options/options.go | 79 +++++++++++++++++++++++++++\n cmd/kubelite/app/server.go | 80 +++++++++++++++++++++++++++\n cmd/kubelite/kubelite.go | 25 +++++++++\n pkg/volume/csi/csi_plugin.go | 11 +++-\n 8 files changed, 297 insertions(+), 11 deletions(-)\n create mode 100644 cmd/kubelite/app/daemons/daemon.go\n create mode 100644 cmd/kubelite/app/options/options.go\n create mode 100644 cmd/kubelite/app/server.go\n create mode 100644 cmd/kubelite/kubelite.go\n\ndiff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go\nindex 042ddc9714a..6a6d4b07f40 100644\n--- a/cmd/kube-apiserver/app/server.go\n+++ b/cmd/kube-apiserver/app/server.go\n@@ -65,9 +65,14 @@ func init() {\n }\n \n // NewAPIServerCommand creates a *cobra.Command object with default parameters\n-func NewAPIServerCommand() *cobra.Command {\n+func NewAPIServerCommand(ctxs ...context.Context) *cobra.Command {\n \ts := options.NewServerRunOptions()\n-\tctx := genericapiserver.SetupSignalContext()\n+\tctx := context.Background()\n+\tif len(ctxs) == 0 {\n+\t\tctx = genericapiserver.SetupSignalContext()\n+\t} else {\n+\t\tctx = ctxs[0]\n+\t}\n \tfeatureGate := s.GenericServerRunOptions.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)\n \n \tcmd := &cobra.Command{\ndiff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go\nindex c9ab27027cf..aade3f81dc4 100644\n--- a/cmd/kube-scheduler/app/server.go\n+++ b/cmd/kube-scheduler/app/server.go\n@@ -149,7 +149,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op\n \tctx, cancel := context.WithCancel(context.Background())\n \tdefer cancel()\n \tgo func() {\n-\t\tstopCh := server.SetupSignalHandler()\n+ c := cmd.Context()\n+ if c == nil {\n+ c = server.SetupSignalContext()\n+ }\n+ stopCh := c.Done()\n \t\t<-stopCh\n \t\tcancel()\n \t}()\ndiff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go\nindex 3bd3ac16eb7..70fa7fd86b7 100644\n--- a/cmd/kubelet/app/server.go\n+++ b/cmd/kubelet/app/server.go\n@@ -135,7 +135,7 @@ func init() {\n }\n \n // NewKubeletCommand creates a *cobra.Command object with default parameters\n-func NewKubeletCommand() *cobra.Command {\n+func NewKubeletCommand(ctx ...context.Context) *cobra.Command {\n \tcleanFlagSet := pflag.NewFlagSet(server.ComponentKubelet, pflag.ContinueOnError)\n \tcleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n \tkubeletFlags := options.NewKubeletFlags()\n@@ -279,6 +279,12 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\tif err := checkPermissions(); err != nil {\n \t\t\t\tklog.ErrorS(err, \"kubelet running with insufficient permissions\")\n \t\t\t}\n+\t\t\trunctx := context.Background()\n+\t\t\tif len(ctx) == 0 {\n+\t\t\t\trunctx = genericapiserver.SetupSignalContext()\n+\t\t\t} else {\n+\t\t\t\trunctx = ctx[0]\n+\t\t\t}\n \n \t\t\t// make the kubelet's config safe for logging\n \t\t\tconfig := kubeletServer.KubeletConfiguration.DeepCopy()\n@@ -288,12 +294,9 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\t// log the kubelet's config for inspection\n \t\t\tklog.V(5).InfoS(\"KubeletConfiguration\", \"configuration\", klog.Format(config))\n \n-\t\t\t// set up signal context for kubelet shutdown\n-\t\t\tctx := genericapiserver.SetupSignalContext()\n-\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n \t\t\t// run the kubelet\n-\t\t\treturn Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n+\t\t\treturn Run(runctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\n \t\t},\n \t}\n \ndiff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go\nnew file mode 100644\nindex 00000000000..46c1af7fdb9\n--- /dev/null\n+++ b/cmd/kubelite/app/daemons/daemon.go\n@@ -0,0 +1,85 @@\n+package daemon\n+\n+import (\n+\t\"context\"\n+\n+\t\"k8s.io/client-go/kubernetes\"\n+\t\"k8s.io/client-go/tools/clientcmd\"\n+\tgenericcontrollermanager \"k8s.io/controller-manager/app\"\n+\t\"k8s.io/klog/v2\"\n+\tapiserver \"k8s.io/kubernetes/cmd/kube-apiserver/app\"\n+\tcontroller \"k8s.io/kubernetes/cmd/kube-controller-manager/app\"\n+\tproxy \"k8s.io/kubernetes/cmd/kube-proxy/app\"\n+\tscheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n+\tkubelet \"k8s.io/kubernetes/cmd/kubelet/app\"\n+\n+\t\"time\"\n+)\n+\n+func StartControllerManager(args []string, ctx context.Context) {\n+\tcommand := controller.NewControllerManagerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Controller Manager\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Controller Manager exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Controller Manager\")\n+}\n+\n+func StartScheduler(args []string, ctx context.Context) {\n+\tcommand := scheduler.NewSchedulerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Scheduler\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Scheduler exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Scheduler\")\n+}\n+\n+func StartProxy(args []string) {\n+\tcommand := proxy.NewProxyCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Proxy\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Proxy exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Proxy\")\n+}\n+\n+func StartKubelet(args []string, ctx context.Context) {\n+\tcommand := kubelet.NewKubeletCommand(ctx)\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Kubelet\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Kubelet exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Kubelet\")\n+}\n+\n+func StartAPIServer(args []string, ctx context.Context) {\n+\tcommand := apiserver.NewAPIServerCommand(ctx)\n+\tcommand.SetArgs(args)\n+\tklog.Info(\"Starting API Server\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"API Server exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping API Server\")\n+}\n+\n+func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) {\n+\tklog.Info(\"Waiting for the API server\")\n+\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigpath)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not find the cluster's kubeconfig file %v\", err)\n+\t}\n+\t// create the client\n+\tclient, err := kubernetes.NewForConfig(config)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not create client to the cluster %v\", err)\n+\t}\n+\tgenericcontrollermanager.WaitForAPIServer(client, timeout)\n+}\ndiff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go\nnew file mode 100644\nindex 00000000000..80f1d8b09fc\n--- /dev/null\n+++ b/cmd/kubelite/app/options/options.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright 2018 The Kubernetes Authors.\n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+\n+package options\n+\n+import (\n+\t\"bufio\"\n+\t\"k8s.io/klog/v2\"\n+\t\"os\"\n+\t\"strings\"\n+)\n+\n+// Options has all the params needed to run a Kubelite\n+type Options struct {\n+\tSchedulerArgsFile string\n+\tControllerManagerArgsFile string\n+\tProxyArgsFile string\n+\tKubeletArgsFile string\n+\tAPIServerArgsFile string\n+\tKubeconfigFile \t\t string\n+\tStartControlPlane\t\t bool\n+}\n+\n+func NewOptions() (*Options){\n+\to := Options{\n+\t\t\"/var/snap/microk8s/current/args/kube-scheduler\",\n+\t\t\"/var/snap/microk8s/current/args/kube-controller-manager\",\n+\t\t\"/var/snap/microk8s/current/args/kube-proxy\",\n+\t\t\"/var/snap/microk8s/current/args/kubelet\",\n+\t\t\"/var/snap/microk8s/current/args/kube-apiserver\",\n+\t\t\"/var/snap/microk8s/current/credentials/client.config\",\n+\t\ttrue,\n+\t}\n+\treturn &o\n+}\n+\n+func ReadArgsFromFile(filename string) []string {\n+\tvar args []string\n+\tfile, err := os.Open(filename)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"Failed to open arguments file %v\", err)\n+\t}\n+\tdefer file.Close()\n+\n+\tscanner := bufio.NewScanner(file)\n+\tfor scanner.Scan() {\n+\t\tline := scanner.Text()\n+\t\tline = strings.TrimSpace(line)\n+\t\t// ignore lines with # and empty lines\n+\t\tif len(line) <= 0 || strings.HasPrefix(line, \"#\") {\n+\t\t\tcontinue\n+\t\t}\n+\t\t// remove \" and '\n+\t\tfor _, r := range \"\\\"'\" {\n+\t\t\tline = strings.ReplaceAll(line, string(r), \"\")\n+\t\t}\n+\t\tfor _, part := range strings.Split(line, \" \") {\n+\n+\t\t\targs = append(args, os.ExpandEnv(part))\n+\t\t}\n+\t}\n+\tif err := scanner.Err(); err != nil {\n+\t\tklog.Fatalf(\"Failed to read arguments file %v\", err)\n+\t}\n+\treturn args\n+}\ndiff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go\nnew file mode 100644\nindex 00000000000..4ff36cd6432\n--- /dev/null\n+++ b/cmd/kubelite/app/server.go\n@@ -0,0 +1,80 @@\n+/*\n+Copyright © 2020 NAME HERE \n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+\thttp://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+package app\n+\n+import (\n+\t\"fmt\"\n+\t\"os\"\n+\t\"time\"\n+\n+\t\"github.com/spf13/cobra\"\n+\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n+\tdaemon \"k8s.io/kubernetes/cmd/kubelite/app/daemons\"\n+\t\"k8s.io/kubernetes/cmd/kubelite/app/options\"\n+)\n+\n+var opts = options.NewOptions()\n+\n+// liteCmd represents the base command when called without any subcommands\n+var liteCmd = &cobra.Command{\n+\tUse: \"kubelite\",\n+\tShort: \"Single server kubernetes\",\n+\tLong: `A single server that spawns all other kubernetes servers as threads`,\n+\t// Uncomment the following line if your bare application\n+\t// has an action associated with it:\n+\tRun: func(cmd *cobra.Command, args []string) {\n+\t\tctx := genericapiserver.SetupSignalContext()\n+\n+\t\tif opts.StartControlPlane {\n+\t\t\tapiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile)\n+\t\t\tgo daemon.StartAPIServer(apiserverArgs, ctx)\n+\t\t\tdaemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second)\n+\n+\t\t\tcontrollerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile)\n+\t\t\tgo daemon.StartControllerManager(controllerArgs, ctx)\n+\n+\t\t\tschedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile)\n+\t\t\tgo daemon.StartScheduler(schedulerArgs, ctx)\n+\t\t}\n+\n+\t\tproxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile)\n+\t\tgo daemon.StartProxy(proxyArgs)\n+\n+\t\tkubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile)\n+\t\tdaemon.StartKubelet(kubeletArgs, ctx)\n+\t},\n+}\n+\n+// Execute adds all child commands to the root command and sets flags appropriately.\n+// This is called by main.main(). It only needs to happen once to the liteCmd.\n+func Execute() {\n+\tif err := liteCmd.Execute(); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+}\n+\n+func init() {\n+\tcobra.OnInitialize()\n+\n+\tliteCmd.Flags().StringVar(&opts.SchedulerArgsFile, \"scheduler-args-file\", opts.SchedulerArgsFile, \"file with the arguments for the scheduler\")\n+\tliteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, \"controller-manager-args-file\", opts.ControllerManagerArgsFile, \"file with the arguments for the controller manager\")\n+\tliteCmd.Flags().StringVar(&opts.ProxyArgsFile, \"proxy-args-file\", opts.ProxyArgsFile, \"file with the arguments for kube-proxy\")\n+\tliteCmd.Flags().StringVar(&opts.KubeletArgsFile, \"kubelet-args-file\", opts.KubeletArgsFile, \"file with the arguments for kubelet\")\n+\tliteCmd.Flags().StringVar(&opts.APIServerArgsFile, \"apiserver-args-file\", opts.APIServerArgsFile, \"file with the arguments for the API server\")\n+\tliteCmd.Flags().StringVar(&opts.KubeconfigFile, \"kubeconfig-file\", opts.KubeconfigFile, \"the kubeconfig file to use to healthcheck the API server\")\n+\tliteCmd.Flags().BoolVar(&opts.StartControlPlane, \"start-control-plane\", opts.StartControlPlane, \"start the control plane (API server, scheduler and controller manager)\")\n+}\ndiff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go\nnew file mode 100644\nindex 00000000000..30ab604f480\n--- /dev/null\n+++ b/cmd/kubelite/kubelite.go\n@@ -0,0 +1,25 @@\n+package main\n+\n+import (\n+\t\"github.com/spf13/pflag\"\n+\tcliflag \"k8s.io/component-base/cli/flag\"\n+\n+\t\"k8s.io/component-base/logs\"\n+\t_ \"k8s.io/component-base/metrics/prometheus/clientgo\" // load all the prometheus client-go plugin\n+\t_ \"k8s.io/component-base/metrics/prometheus/version\" // for version metric registration\n+\t\"k8s.io/kubernetes/cmd/kubelite/app\"\n+)\n+\n+func main() {\n+\tprintln(\"Starting kubelite\")\n+\t// TODO: once we switch everything over to Cobra commands, we can go back to calling\n+\t// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the\n+\t// normalize func and add the go flag set by hand.\n+\tpflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n+\t// utilflag.InitFlags()\n+\tlogs.InitLogs()\n+\tdefer logs.FlushLogs()\n+\n+\tapp.Execute()\n+\tprintln(\"Stopping kubelite\")\n+}\ndiff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go\nindex 902f31e9911..95d20ba48d8 100644\n--- a/pkg/volume/csi/csi_plugin.go\n+++ b/pkg/volume/csi/csi_plugin.go\n@@ -345,18 +345,23 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {\n \t}\n \n \t// Initializing the label management channels\n-\tnim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n+\tlocalNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n \tPluginHandler.csiPlugin = p\n \n \t// This function prevents Kubelet from posting Ready status until CSINode\n \t// is both installed and initialized\n-\tif err := initializeCSINode(host, p.csiDriverInformer); err != nil {\n+\tif err := initializeCSINode(host, p.csiDriverInformer, localNim); err != nil {\n \t\treturn errors.New(log(\"failed to initialize CSINode: %v\", err))\n \t}\n+\n+\tif _, ok := host.(volume.KubeletVolumeHost); ok {\n+\t\tnim = localNim\n+\t}\n+\n \treturn nil\n }\n \n-func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer) error {\n+func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer, nim nodeinfomanager.Interface) error {\n \tkvh, ok := host.(volume.KubeletVolumeHost)\n \tif !ok {\n \t\tklog.V(4).Info(\"Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet\")\n-- \n2.49.0\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.33.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch", "content": "From beaeb73228b7e4a836b3bd5d73e49402279a47a6 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 27 Jul 2023 18:08:00 +0300\nSubject: [PATCH 2/2] Set log reapply handling to ignore unchanged\n\n---\n staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n\ndiff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go\nindex 95c0a2cba98..4482564ad95 100644\n--- a/staging/src/k8s.io/component-base/logs/api/v1/options.go\n+++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go\n@@ -65,7 +65,7 @@ func NewLoggingConfiguration() *LoggingConfiguration {\n // are no goroutines which might call logging functions. The default for ValidateAndApply\n // and ValidateAndApplyWithOptions is to return an error when called more than once.\n // Binaries and unit tests can override that behavior.\n-var ReapplyHandling = ReapplyHandlingError\n+var ReapplyHandling = ReapplyHandlingIgnoreUnchanged\n \n type ReapplyHandlingType int\n \n-- \n2.49.0\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.34.0/0000-Kubelite-integration.patch", "content": "From 241fc2a00be6fde3f7059ae61842cba44dff82a5 Mon Sep 17 00:00:00 2001\nFrom: Konstantinos Tsakalozos \nDate: Wed, 3 Mar 2021 18:19:37 +0200\nSubject: [PATCH] Kubelite integration\n\n---\n cmd/kube-apiserver/app/server.go | 9 +-\n .../app/options/options.go | 10 --\n cmd/kube-scheduler/app/server.go | 6 +-\n cmd/kubelet/app/server.go | 4 -\n cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++\n cmd/kubelite/app/options/options.go | 79 ++++++++++++++++\n cmd/kubelite/app/server.go | 91 +++++++++++++++++++\n cmd/kubelite/kubelite.go | 25 +++++\n pkg/volume/csi/csi_plugin.go | 11 ++-\n 9 files changed, 300 insertions(+), 20 deletions(-)\n create mode 100644 cmd/kubelite/app/daemons/daemon.go\n create mode 100644 cmd/kubelite/app/options/options.go\n create mode 100644 cmd/kubelite/app/server.go\n create mode 100644 cmd/kubelite/kubelite.go\n\ndiff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go\nindex 71ebb317461..86dc50ce671 100644\n--- a/cmd/kube-apiserver/app/server.go\n+++ b/cmd/kube-apiserver/app/server.go\n@@ -67,9 +67,14 @@ func init() {\n }\n \n // NewAPIServerCommand creates a *cobra.Command object with default parameters\n-func NewAPIServerCommand() *cobra.Command {\n+func NewAPIServerCommand(ctxs ...context.Context) *cobra.Command {\n \ts := options.NewServerRunOptions()\n-\tctx := genericapiserver.SetupSignalContext()\n+\tctx := context.Background()\n+\tif len(ctxs) == 0 {\n+\t\tctx = genericapiserver.SetupSignalContext()\n+\t} else {\n+\t\tctx = ctxs[0]\n+\t}\n \tfeatureGate := s.GenericServerRunOptions.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)\n \n \tcmd := &cobra.Command{\ndiff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go\nindex 176b2a14bc1..5048b04014e 100644\n--- a/cmd/kube-controller-manager/app/options/options.go\n+++ b/cmd/kube-controller-manager/app/options/options.go\n@@ -25,11 +25,9 @@ import (\n \tv1 \"k8s.io/api/core/v1\"\n \tutilerrors \"k8s.io/apimachinery/pkg/util/errors\"\n \tutilruntime \"k8s.io/apimachinery/pkg/util/runtime\"\n-\t\"k8s.io/apimachinery/pkg/util/version\"\n \tapiserveroptions \"k8s.io/apiserver/pkg/server/options\"\n \t\"k8s.io/apiserver/pkg/util/compatibility\"\n \tutilfeature \"k8s.io/apiserver/pkg/util/feature\"\n-\tclientgofeaturegate \"k8s.io/client-go/features\"\n \tclientset \"k8s.io/client-go/kubernetes\"\n \tclientgokubescheme \"k8s.io/client-go/kubernetes/scheme\"\n \trestclient \"k8s.io/client-go/rest\"\n@@ -40,7 +38,6 @@ import (\n \tcpoptions \"k8s.io/cloud-provider/options\"\n \tcliflag \"k8s.io/component-base/cli/flag\"\n \tbasecompatibility \"k8s.io/component-base/compatibility\"\n-\t\"k8s.io/component-base/featuregate\"\n \t\"k8s.io/component-base/logs\"\n \tlogsapi \"k8s.io/component-base/logs/api/v1\"\n \t\"k8s.io/component-base/metrics\"\n@@ -298,13 +295,6 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy\n \tfs.StringVar(&s.Master, \"master\", s.Master, \"The address of the Kubernetes API server (overrides any value in kubeconfig).\")\n \tfs.StringVar(&s.Generic.ClientConnection.Kubeconfig, \"kubeconfig\", s.Generic.ClientConnection.Kubeconfig, \"Path to kubeconfig file with authorization and master location information (the master location can be overridden by the master flag).\")\n \n-\tif !utilfeature.DefaultFeatureGate.Enabled(featuregate.Feature(clientgofeaturegate.WatchListClient)) {\n-\t\tver := version.MustParse(\"1.34\")\n-\t\tif err := utilfeature.DefaultMutableFeatureGate.OverrideDefaultAtVersion(featuregate.Feature(clientgofeaturegate.WatchListClient), true, ver); err != nil {\n-\t\t\tpanic(fmt.Sprintf(\"unable to set %s feature gate, err: %v\", clientgofeaturegate.WatchListClient, err))\n-\t\t}\n-\t}\n-\n \ts.ComponentGlobalsRegistry.AddFlags(fss.FlagSet(\"generic\"))\n \n \treturn fss\ndiff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go\nindex 803e0c37861..5b58d437d30 100644\n--- a/cmd/kube-scheduler/app/server.go\n+++ b/cmd/kube-scheduler/app/server.go\n@@ -150,7 +150,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op\n \tctx, cancel := context.WithCancel(context.Background())\n \tdefer cancel()\n \tgo func() {\n-\t\tstopCh := server.SetupSignalHandler()\n+ c := cmd.Context()\n+ if c == nil {\n+ c = server.SetupSignalContext()\n+ }\n+ stopCh := c.Done()\n \t\t<-stopCh\n \t\tcancel()\n \t}()\ndiff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go\nindex f069f6b08b9..62f3ca58dc7 100644\n--- a/cmd/kubelet/app/server.go\n+++ b/cmd/kubelet/app/server.go\n@@ -63,7 +63,6 @@ import (\n \t\"k8s.io/apimachinery/pkg/util/sets\"\n \t\"k8s.io/apimachinery/pkg/util/validation/field\"\n \t\"k8s.io/apimachinery/pkg/util/wait\"\n-\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n \t\"k8s.io/apiserver/pkg/server/healthz\"\n \tutilfeature \"k8s.io/apiserver/pkg/util/feature\"\n \tclientset \"k8s.io/client-go/kubernetes\"\n@@ -291,9 +290,6 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\t// log the kubelet's config for inspection\n \t\t\tlogger.V(5).Info(\"KubeletConfiguration\", \"configuration\", klog.Format(config))\n \n-\t\t\t// set up signal context for kubelet shutdown\n-\t\t\tctx := genericapiserver.SetupSignalContext()\n-\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n \t\t\t// run the kubelet\n \t\t\treturn Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\ndiff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go\nnew file mode 100644\nindex 00000000000..46c1af7fdb9\n--- /dev/null\n+++ b/cmd/kubelite/app/daemons/daemon.go\n@@ -0,0 +1,85 @@\n+package daemon\n+\n+import (\n+\t\"context\"\n+\n+\t\"k8s.io/client-go/kubernetes\"\n+\t\"k8s.io/client-go/tools/clientcmd\"\n+\tgenericcontrollermanager \"k8s.io/controller-manager/app\"\n+\t\"k8s.io/klog/v2\"\n+\tapiserver \"k8s.io/kubernetes/cmd/kube-apiserver/app\"\n+\tcontroller \"k8s.io/kubernetes/cmd/kube-controller-manager/app\"\n+\tproxy \"k8s.io/kubernetes/cmd/kube-proxy/app\"\n+\tscheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n+\tkubelet \"k8s.io/kubernetes/cmd/kubelet/app\"\n+\n+\t\"time\"\n+)\n+\n+func StartControllerManager(args []string, ctx context.Context) {\n+\tcommand := controller.NewControllerManagerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Controller Manager\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Controller Manager exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Controller Manager\")\n+}\n+\n+func StartScheduler(args []string, ctx context.Context) {\n+\tcommand := scheduler.NewSchedulerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Scheduler\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Scheduler exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Scheduler\")\n+}\n+\n+func StartProxy(args []string) {\n+\tcommand := proxy.NewProxyCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Proxy\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Proxy exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Proxy\")\n+}\n+\n+func StartKubelet(args []string, ctx context.Context) {\n+\tcommand := kubelet.NewKubeletCommand(ctx)\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Kubelet\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Kubelet exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Kubelet\")\n+}\n+\n+func StartAPIServer(args []string, ctx context.Context) {\n+\tcommand := apiserver.NewAPIServerCommand(ctx)\n+\tcommand.SetArgs(args)\n+\tklog.Info(\"Starting API Server\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"API Server exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping API Server\")\n+}\n+\n+func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) {\n+\tklog.Info(\"Waiting for the API server\")\n+\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigpath)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not find the cluster's kubeconfig file %v\", err)\n+\t}\n+\t// create the client\n+\tclient, err := kubernetes.NewForConfig(config)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not create client to the cluster %v\", err)\n+\t}\n+\tgenericcontrollermanager.WaitForAPIServer(client, timeout)\n+}\ndiff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go\nnew file mode 100644\nindex 00000000000..80f1d8b09fc\n--- /dev/null\n+++ b/cmd/kubelite/app/options/options.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright 2018 The Kubernetes Authors.\n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+\n+package options\n+\n+import (\n+\t\"bufio\"\n+\t\"k8s.io/klog/v2\"\n+\t\"os\"\n+\t\"strings\"\n+)\n+\n+// Options has all the params needed to run a Kubelite\n+type Options struct {\n+\tSchedulerArgsFile string\n+\tControllerManagerArgsFile string\n+\tProxyArgsFile string\n+\tKubeletArgsFile string\n+\tAPIServerArgsFile string\n+\tKubeconfigFile \t\t string\n+\tStartControlPlane\t\t bool\n+}\n+\n+func NewOptions() (*Options){\n+\to := Options{\n+\t\t\"/var/snap/microk8s/current/args/kube-scheduler\",\n+\t\t\"/var/snap/microk8s/current/args/kube-controller-manager\",\n+\t\t\"/var/snap/microk8s/current/args/kube-proxy\",\n+\t\t\"/var/snap/microk8s/current/args/kubelet\",\n+\t\t\"/var/snap/microk8s/current/args/kube-apiserver\",\n+\t\t\"/var/snap/microk8s/current/credentials/client.config\",\n+\t\ttrue,\n+\t}\n+\treturn &o\n+}\n+\n+func ReadArgsFromFile(filename string) []string {\n+\tvar args []string\n+\tfile, err := os.Open(filename)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"Failed to open arguments file %v\", err)\n+\t}\n+\tdefer file.Close()\n+\n+\tscanner := bufio.NewScanner(file)\n+\tfor scanner.Scan() {\n+\t\tline := scanner.Text()\n+\t\tline = strings.TrimSpace(line)\n+\t\t// ignore lines with # and empty lines\n+\t\tif len(line) <= 0 || strings.HasPrefix(line, \"#\") {\n+\t\t\tcontinue\n+\t\t}\n+\t\t// remove \" and '\n+\t\tfor _, r := range \"\\\"'\" {\n+\t\t\tline = strings.ReplaceAll(line, string(r), \"\")\n+\t\t}\n+\t\tfor _, part := range strings.Split(line, \" \") {\n+\n+\t\t\targs = append(args, os.ExpandEnv(part))\n+\t\t}\n+\t}\n+\tif err := scanner.Err(); err != nil {\n+\t\tklog.Fatalf(\"Failed to read arguments file %v\", err)\n+\t}\n+\treturn args\n+}\ndiff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go\nnew file mode 100644\nindex 00000000000..858a7c1094c\n--- /dev/null\n+++ b/cmd/kubelite/app/server.go\n@@ -0,0 +1,91 @@\n+/*\n+Copyright © 2020 NAME HERE \n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+\thttp://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+package app\n+\n+import (\n+\t\"fmt\"\n+\t\"os\"\n+\t\"time\"\n+\n+\t\"github.com/spf13/cobra\"\n+\t\"k8s.io/apimachinery/pkg/util/version\"\n+\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n+\tutilfeature \"k8s.io/apiserver/pkg/util/feature\"\n+\tclientgofeaturegate \"k8s.io/client-go/features\"\n+\t\"k8s.io/component-base/featuregate\"\n+\tdaemon \"k8s.io/kubernetes/cmd/kubelite/app/daemons\"\n+\t\"k8s.io/kubernetes/cmd/kubelite/app/options\"\n+)\n+\n+var opts = options.NewOptions()\n+\n+// liteCmd represents the base command when called without any subcommands\n+var liteCmd = &cobra.Command{\n+\tUse: \"kubelite\",\n+\tShort: \"Single server kubernetes\",\n+\tLong: `A single server that spawns all other kubernetes servers as threads`,\n+\t// Uncomment the following line if your bare application\n+\t// has an action associated with it:\n+\tRun: func(cmd *cobra.Command, args []string) {\n+\t\tctx := genericapiserver.SetupSignalContext()\n+\n+\t\tif opts.StartControlPlane {\n+\t\t\tif !utilfeature.DefaultFeatureGate.Enabled(featuregate.Feature(clientgofeaturegate.WatchListClient)) {\n+\t\t\t\tver := version.MustParse(\"1.34\")\n+\t\t\t\tif err := utilfeature.DefaultMutableFeatureGate.OverrideDefaultAtVersion(featuregate.Feature(clientgofeaturegate.WatchListClient), true, ver); err != nil {\n+\t\t\t\t\tpanic(fmt.Sprintf(\"unable to set %s feature gate, err: %v\", clientgofeaturegate.WatchListClient, err))\n+\t\t\t\t}\n+\t\t\t}\n+\n+\t\t\tapiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile)\n+\t\t\tgo daemon.StartAPIServer(apiserverArgs, ctx)\n+\t\t\tdaemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second)\n+\n+\t\t\tcontrollerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile)\n+\t\t\tgo daemon.StartControllerManager(controllerArgs, ctx)\n+\n+\t\t\tschedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile)\n+\t\t\tgo daemon.StartScheduler(schedulerArgs, ctx)\n+\t\t}\n+\n+\t\tproxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile)\n+\t\tgo daemon.StartProxy(proxyArgs)\n+\n+\t\tkubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile)\n+\t\tdaemon.StartKubelet(kubeletArgs, ctx)\n+\t},\n+}\n+\n+// Execute adds all child commands to the root command and sets flags appropriately.\n+// This is called by main.main(). It only needs to happen once to the liteCmd.\n+func Execute() {\n+\tif err := liteCmd.Execute(); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+}\n+\n+func init() {\n+\tcobra.OnInitialize()\n+\n+\tliteCmd.Flags().StringVar(&opts.SchedulerArgsFile, \"scheduler-args-file\", opts.SchedulerArgsFile, \"file with the arguments for the scheduler\")\n+\tliteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, \"controller-manager-args-file\", opts.ControllerManagerArgsFile, \"file with the arguments for the controller manager\")\n+\tliteCmd.Flags().StringVar(&opts.ProxyArgsFile, \"proxy-args-file\", opts.ProxyArgsFile, \"file with the arguments for kube-proxy\")\n+\tliteCmd.Flags().StringVar(&opts.KubeletArgsFile, \"kubelet-args-file\", opts.KubeletArgsFile, \"file with the arguments for kubelet\")\n+\tliteCmd.Flags().StringVar(&opts.APIServerArgsFile, \"apiserver-args-file\", opts.APIServerArgsFile, \"file with the arguments for the API server\")\n+\tliteCmd.Flags().StringVar(&opts.KubeconfigFile, \"kubeconfig-file\", opts.KubeconfigFile, \"the kubeconfig file to use to healthcheck the API server\")\n+\tliteCmd.Flags().BoolVar(&opts.StartControlPlane, \"start-control-plane\", opts.StartControlPlane, \"start the control plane (API server, scheduler and controller manager)\")\n+}\ndiff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go\nnew file mode 100644\nindex 00000000000..30ab604f480\n--- /dev/null\n+++ b/cmd/kubelite/kubelite.go\n@@ -0,0 +1,25 @@\n+package main\n+\n+import (\n+\t\"github.com/spf13/pflag\"\n+\tcliflag \"k8s.io/component-base/cli/flag\"\n+\n+\t\"k8s.io/component-base/logs\"\n+\t_ \"k8s.io/component-base/metrics/prometheus/clientgo\" // load all the prometheus client-go plugin\n+\t_ \"k8s.io/component-base/metrics/prometheus/version\" // for version metric registration\n+\t\"k8s.io/kubernetes/cmd/kubelite/app\"\n+)\n+\n+func main() {\n+\tprintln(\"Starting kubelite\")\n+\t// TODO: once we switch everything over to Cobra commands, we can go back to calling\n+\t// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the\n+\t// normalize func and add the go flag set by hand.\n+\tpflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n+\t// utilflag.InitFlags()\n+\tlogs.InitLogs()\n+\tdefer logs.FlushLogs()\n+\n+\tapp.Execute()\n+\tprintln(\"Stopping kubelite\")\n+}\ndiff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go\nindex d2d830376da..fa6dbc91fef 100644\n--- a/pkg/volume/csi/csi_plugin.go\n+++ b/pkg/volume/csi/csi_plugin.go\n@@ -347,18 +347,23 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {\n \t}\n \n \t// Initializing the label management channels\n-\tnim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n+\tlocalNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n \tPluginHandler.csiPlugin = p\n \n \t// This function prevents Kubelet from posting Ready status until CSINode\n \t// is both installed and initialized\n-\tif err := initializeCSINode(host, p.csiDriverInformer); err != nil {\n+\tif err := initializeCSINode(host, p.csiDriverInformer, localNim); err != nil {\n \t\treturn errors.New(log(\"failed to initialize CSINode: %v\", err))\n \t}\n+\n+\tif _, ok := host.(volume.KubeletVolumeHost); ok {\n+\t\tnim = localNim\n+\t}\n+\n \treturn nil\n }\n \n-func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer) error {\n+func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer, nim nodeinfomanager.Interface) error {\n \tkvh, ok := host.(volume.KubeletVolumeHost)\n \tif !ok {\n \t\tklog.V(4).Info(\"Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet\")\n-- \n2.43.0\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.34.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch", "content": "From e2e2155d05b3ddd16640272bd1360425a3883c78 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 27 Jul 2023 18:08:00 +0300\nSubject: [PATCH 2/2] Set log reapply handling to ignore unchanged\n\n---\n staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n\ndiff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go\nindex 4c8a0d2c53f..6cb6ae1f42d 100644\n--- a/staging/src/k8s.io/component-base/logs/api/v1/options.go\n+++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go\n@@ -65,7 +65,7 @@ func NewLoggingConfiguration() *LoggingConfiguration {\n // are no goroutines which might call logging functions. The default for ValidateAndApply\n // and ValidateAndApplyWithOptions is to return an error when called more than once.\n // Binaries and unit tests can override that behavior.\n-var ReapplyHandling = ReapplyHandlingError\n+var ReapplyHandling = ReapplyHandlingIgnoreUnchanged\n \n type ReapplyHandlingType int\n \n-- \n2.43.0\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.34.0/0002-fix-allow-node-to-get-endpointslices.patch", "content": "From dea2abd80878be1eff519216c0bad5a0e35462ec Mon Sep 17 00:00:00 2001\nFrom: Mateo Florido \nDate: Thu, 11 Sep 2025 17:36:10 -0500\nSubject: [PATCH] fix: allow node to get endpointslices\n\n---\n plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go | 1 +\n 1 file changed, 1 insertion(+)\n\ndiff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go\nindex 447b0bc2e99..daa3bde6b1c 100644\n--- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go\n+++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go\n@@ -228,6 +228,7 @@ func NodeRules() []rbacv1.PolicyRule {\n \t\t// TODO: add to the Node authorizer and restrict to endpoints referenced by pods or PVs bound to the node\n \t\t// Needed for glusterfs volumes\n \t\trbacv1helpers.NewRule(\"get\").Groups(legacyGroup).Resources(\"endpoints\").RuleOrDie(),\n+\t\trbacv1helpers.NewRule(\"get\", \"list\", \"watch\").Groups(discoveryGroup).Resources(\"endpointslices\").RuleOrDie(),\n \t\t// Used to create a certificatesigningrequest for a node-specific client certificate, and watch\n \t\t// for it to be signed. This allows the kubelet to rotate it's own certificate.\n \t\trbacv1helpers.NewRule(\"create\", \"get\", \"list\", \"watch\").Groups(certificatesGroup).Resources(\"certificatesigningrequests\").RuleOrDie(),\n-- \n2.48.1\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.35.0/0000-Kubelite-integration.patch", "content": "From afd6e0c5da37c3ae02c22c91ef898c1169e8d657 Mon Sep 17 00:00:00 2001\nFrom: Homayoon Alimohammadi \nDate: Thu, 27 Nov 2025 18:47:43 +0400\nSubject: [PATCH] Kubelite integration\n\nSigned-off-by: Homayoon Alimohammadi \n---\n cmd/kube-apiserver/app/server.go | 9 +-\n .../app/options/options.go | 10 --\n cmd/kube-scheduler/app/server.go | 6 +-\n cmd/kubelet/app/server.go | 4 -\n cmd/kubelite/app/daemons/daemon.go | 85 +++++++++++++++++\n cmd/kubelite/app/options/options.go | 79 ++++++++++++++++\n cmd/kubelite/app/server.go | 91 +++++++++++++++++++\n cmd/kubelite/kubelite.go | 25 +++++\n pkg/volume/csi/csi_plugin.go | 11 ++-\n 9 files changed, 300 insertions(+), 20 deletions(-)\n create mode 100644 cmd/kubelite/app/daemons/daemon.go\n create mode 100644 cmd/kubelite/app/options/options.go\n create mode 100644 cmd/kubelite/app/server.go\n create mode 100644 cmd/kubelite/kubelite.go\n\ndiff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go\nindex a0ba37e0103..32ae1a53496 100644\n--- a/cmd/kube-apiserver/app/server.go\n+++ b/cmd/kube-apiserver/app/server.go\n@@ -67,9 +67,14 @@ func init() {\n }\n \n // NewAPIServerCommand creates a *cobra.Command object with default parameters\n-func NewAPIServerCommand() *cobra.Command {\n+func NewAPIServerCommand(ctxs ...context.Context) *cobra.Command {\n \ts := options.NewServerRunOptions()\n-\tctx := genericapiserver.SetupSignalContext()\n+\tctx := context.Background()\n+\tif len(ctxs) == 0 {\n+\t\tctx = genericapiserver.SetupSignalContext()\n+\t} else {\n+\t\tctx = ctxs[0]\n+\t}\n \tfeatureGate := s.GenericServerRunOptions.ComponentGlobalsRegistry.FeatureGateFor(basecompatibility.DefaultKubeComponent)\n \n \tcmd := &cobra.Command{\ndiff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go\nindex 867cd8d3250..8c4f50a923e 100644\n--- a/cmd/kube-controller-manager/app/options/options.go\n+++ b/cmd/kube-controller-manager/app/options/options.go\n@@ -26,12 +26,10 @@ import (\n \tv1 \"k8s.io/api/core/v1\"\n \tutilerrors \"k8s.io/apimachinery/pkg/util/errors\"\n \tutilruntime \"k8s.io/apimachinery/pkg/util/runtime\"\n-\t\"k8s.io/apimachinery/pkg/util/version\"\n \t\"k8s.io/apiserver/pkg/server/flagz\"\n \tapiserveroptions \"k8s.io/apiserver/pkg/server/options\"\n \t\"k8s.io/apiserver/pkg/util/compatibility\"\n \tutilfeature \"k8s.io/apiserver/pkg/util/feature\"\n-\tclientgofeaturegate \"k8s.io/client-go/features\"\n \tclientset \"k8s.io/client-go/kubernetes\"\n \tclientgokubescheme \"k8s.io/client-go/kubernetes/scheme\"\n \trestclient \"k8s.io/client-go/rest\"\n@@ -42,7 +40,6 @@ import (\n \tcpoptions \"k8s.io/cloud-provider/options\"\n \tcliflag \"k8s.io/component-base/cli/flag\"\n \tbasecompatibility \"k8s.io/component-base/compatibility\"\n-\t\"k8s.io/component-base/featuregate\"\n \t\"k8s.io/component-base/logs\"\n \tlogsapi \"k8s.io/component-base/logs/api/v1\"\n \t\"k8s.io/component-base/metrics\"\n@@ -311,13 +308,6 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy\n \tfss.FlagSet(\"generic\").DurationVar(&s.ControllerShutdownTimeout, \"controller-shutdown-timeout\",\n \t\ts.ControllerShutdownTimeout, \"Time to wait for the controllers to shut down before terminating the executable\")\n \n-\tif !utilfeature.DefaultFeatureGate.Enabled(featuregate.Feature(clientgofeaturegate.WatchListClient)) {\n-\t\tver := version.MustParse(\"1.34\")\n-\t\tif err := utilfeature.DefaultMutableFeatureGate.OverrideDefaultAtVersion(featuregate.Feature(clientgofeaturegate.WatchListClient), true, ver); err != nil {\n-\t\t\tpanic(fmt.Sprintf(\"unable to set %s feature gate, err: %v\", clientgofeaturegate.WatchListClient, err))\n-\t\t}\n-\t}\n-\n \ts.ComponentGlobalsRegistry.AddFlags(fss.FlagSet(\"generic\"))\n \n \treturn fss\ndiff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go\nindex 834213e337f..1eb9c630220 100644\n--- a/cmd/kube-scheduler/app/server.go\n+++ b/cmd/kube-scheduler/app/server.go\n@@ -150,7 +150,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op\n \tctx, cancel := context.WithCancel(context.Background())\n \tdefer cancel()\n \tgo func() {\n-\t\tstopCh := server.SetupSignalHandler()\n+ c := cmd.Context()\n+ if c == nil {\n+ c = server.SetupSignalContext()\n+ }\n+ stopCh := c.Done()\n \t\t<-stopCh\n \t\tcancel()\n \t}()\ndiff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go\nindex 54ff8d6a26b..203fe982a9a 100644\n--- a/cmd/kubelet/app/server.go\n+++ b/cmd/kubelet/app/server.go\n@@ -64,7 +64,6 @@ import (\n \t\"k8s.io/apimachinery/pkg/util/sets\"\n \t\"k8s.io/apimachinery/pkg/util/validation/field\"\n \t\"k8s.io/apimachinery/pkg/util/wait\"\n-\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n \t\"k8s.io/apiserver/pkg/server/flagz\"\n \t\"k8s.io/apiserver/pkg/server/healthz\"\n \tutilfeature \"k8s.io/apiserver/pkg/util/feature\"\n@@ -287,9 +286,6 @@ is checked every 20 seconds (also configurable with a flag).`,\n \t\t\t// log the kubelet's config for inspection\n \t\t\tlogger.V(5).Info(\"KubeletConfiguration\", \"configuration\", klog.Format(config))\n \n-\t\t\t// set up signal context for kubelet shutdown\n-\t\t\tctx := genericapiserver.SetupSignalContext()\n-\n \t\t\tutilfeature.DefaultMutableFeatureGate.AddMetrics()\n \t\t\t// run the kubelet\n \t\t\treturn Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)\ndiff --git a/cmd/kubelite/app/daemons/daemon.go b/cmd/kubelite/app/daemons/daemon.go\nnew file mode 100644\nindex 00000000000..46c1af7fdb9\n--- /dev/null\n+++ b/cmd/kubelite/app/daemons/daemon.go\n@@ -0,0 +1,85 @@\n+package daemon\n+\n+import (\n+\t\"context\"\n+\n+\t\"k8s.io/client-go/kubernetes\"\n+\t\"k8s.io/client-go/tools/clientcmd\"\n+\tgenericcontrollermanager \"k8s.io/controller-manager/app\"\n+\t\"k8s.io/klog/v2\"\n+\tapiserver \"k8s.io/kubernetes/cmd/kube-apiserver/app\"\n+\tcontroller \"k8s.io/kubernetes/cmd/kube-controller-manager/app\"\n+\tproxy \"k8s.io/kubernetes/cmd/kube-proxy/app\"\n+\tscheduler \"k8s.io/kubernetes/cmd/kube-scheduler/app\"\n+\tkubelet \"k8s.io/kubernetes/cmd/kubelet/app\"\n+\n+\t\"time\"\n+)\n+\n+func StartControllerManager(args []string, ctx context.Context) {\n+\tcommand := controller.NewControllerManagerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Controller Manager\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Controller Manager exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Controller Manager\")\n+}\n+\n+func StartScheduler(args []string, ctx context.Context) {\n+\tcommand := scheduler.NewSchedulerCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Scheduler\")\n+\tif err := command.ExecuteContext(ctx); err != nil {\n+\t\tklog.Fatalf(\"Scheduler exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Scheduler\")\n+}\n+\n+func StartProxy(args []string) {\n+\tcommand := proxy.NewProxyCommand()\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Proxy\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Proxy exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Proxy\")\n+}\n+\n+func StartKubelet(args []string, ctx context.Context) {\n+\tcommand := kubelet.NewKubeletCommand(ctx)\n+\tcommand.SetArgs(args)\n+\n+\tklog.Info(\"Starting Kubelet\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"Kubelet exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping Kubelet\")\n+}\n+\n+func StartAPIServer(args []string, ctx context.Context) {\n+\tcommand := apiserver.NewAPIServerCommand(ctx)\n+\tcommand.SetArgs(args)\n+\tklog.Info(\"Starting API Server\")\n+\tif err := command.Execute(); err != nil {\n+\t\tklog.Fatalf(\"API Server exited %v\", err)\n+\t}\n+\tklog.Info(\"Stopping API Server\")\n+}\n+\n+func WaitForAPIServer(kubeconfigpath string, timeout time.Duration) {\n+\tklog.Info(\"Waiting for the API server\")\n+\tconfig, err := clientcmd.BuildConfigFromFlags(\"\", kubeconfigpath)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not find the cluster's kubeconfig file %v\", err)\n+\t}\n+\t// create the client\n+\tclient, err := kubernetes.NewForConfig(config)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"could not create client to the cluster %v\", err)\n+\t}\n+\tgenericcontrollermanager.WaitForAPIServer(client, timeout)\n+}\ndiff --git a/cmd/kubelite/app/options/options.go b/cmd/kubelite/app/options/options.go\nnew file mode 100644\nindex 00000000000..80f1d8b09fc\n--- /dev/null\n+++ b/cmd/kubelite/app/options/options.go\n@@ -0,0 +1,79 @@\n+/*\n+Copyright 2018 The Kubernetes Authors.\n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+\n+package options\n+\n+import (\n+\t\"bufio\"\n+\t\"k8s.io/klog/v2\"\n+\t\"os\"\n+\t\"strings\"\n+)\n+\n+// Options has all the params needed to run a Kubelite\n+type Options struct {\n+\tSchedulerArgsFile string\n+\tControllerManagerArgsFile string\n+\tProxyArgsFile string\n+\tKubeletArgsFile string\n+\tAPIServerArgsFile string\n+\tKubeconfigFile \t\t string\n+\tStartControlPlane\t\t bool\n+}\n+\n+func NewOptions() (*Options){\n+\to := Options{\n+\t\t\"/var/snap/microk8s/current/args/kube-scheduler\",\n+\t\t\"/var/snap/microk8s/current/args/kube-controller-manager\",\n+\t\t\"/var/snap/microk8s/current/args/kube-proxy\",\n+\t\t\"/var/snap/microk8s/current/args/kubelet\",\n+\t\t\"/var/snap/microk8s/current/args/kube-apiserver\",\n+\t\t\"/var/snap/microk8s/current/credentials/client.config\",\n+\t\ttrue,\n+\t}\n+\treturn &o\n+}\n+\n+func ReadArgsFromFile(filename string) []string {\n+\tvar args []string\n+\tfile, err := os.Open(filename)\n+\tif err != nil {\n+\t\tklog.Fatalf(\"Failed to open arguments file %v\", err)\n+\t}\n+\tdefer file.Close()\n+\n+\tscanner := bufio.NewScanner(file)\n+\tfor scanner.Scan() {\n+\t\tline := scanner.Text()\n+\t\tline = strings.TrimSpace(line)\n+\t\t// ignore lines with # and empty lines\n+\t\tif len(line) <= 0 || strings.HasPrefix(line, \"#\") {\n+\t\t\tcontinue\n+\t\t}\n+\t\t// remove \" and '\n+\t\tfor _, r := range \"\\\"'\" {\n+\t\t\tline = strings.ReplaceAll(line, string(r), \"\")\n+\t\t}\n+\t\tfor _, part := range strings.Split(line, \" \") {\n+\n+\t\t\targs = append(args, os.ExpandEnv(part))\n+\t\t}\n+\t}\n+\tif err := scanner.Err(); err != nil {\n+\t\tklog.Fatalf(\"Failed to read arguments file %v\", err)\n+\t}\n+\treturn args\n+}\ndiff --git a/cmd/kubelite/app/server.go b/cmd/kubelite/app/server.go\nnew file mode 100644\nindex 00000000000..858a7c1094c\n--- /dev/null\n+++ b/cmd/kubelite/app/server.go\n@@ -0,0 +1,91 @@\n+/*\n+Copyright © 2020 NAME HERE \n+\n+Licensed under the Apache License, Version 2.0 (the \"License\");\n+you may not use this file except in compliance with the License.\n+You may obtain a copy of the License at\n+\n+\thttp://www.apache.org/licenses/LICENSE-2.0\n+\n+Unless required by applicable law or agreed to in writing, software\n+distributed under the License is distributed on an \"AS IS\" BASIS,\n+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+See the License for the specific language governing permissions and\n+limitations under the License.\n+*/\n+package app\n+\n+import (\n+\t\"fmt\"\n+\t\"os\"\n+\t\"time\"\n+\n+\t\"github.com/spf13/cobra\"\n+\t\"k8s.io/apimachinery/pkg/util/version\"\n+\tgenericapiserver \"k8s.io/apiserver/pkg/server\"\n+\tutilfeature \"k8s.io/apiserver/pkg/util/feature\"\n+\tclientgofeaturegate \"k8s.io/client-go/features\"\n+\t\"k8s.io/component-base/featuregate\"\n+\tdaemon \"k8s.io/kubernetes/cmd/kubelite/app/daemons\"\n+\t\"k8s.io/kubernetes/cmd/kubelite/app/options\"\n+)\n+\n+var opts = options.NewOptions()\n+\n+// liteCmd represents the base command when called without any subcommands\n+var liteCmd = &cobra.Command{\n+\tUse: \"kubelite\",\n+\tShort: \"Single server kubernetes\",\n+\tLong: `A single server that spawns all other kubernetes servers as threads`,\n+\t// Uncomment the following line if your bare application\n+\t// has an action associated with it:\n+\tRun: func(cmd *cobra.Command, args []string) {\n+\t\tctx := genericapiserver.SetupSignalContext()\n+\n+\t\tif opts.StartControlPlane {\n+\t\t\tif !utilfeature.DefaultFeatureGate.Enabled(featuregate.Feature(clientgofeaturegate.WatchListClient)) {\n+\t\t\t\tver := version.MustParse(\"1.34\")\n+\t\t\t\tif err := utilfeature.DefaultMutableFeatureGate.OverrideDefaultAtVersion(featuregate.Feature(clientgofeaturegate.WatchListClient), true, ver); err != nil {\n+\t\t\t\t\tpanic(fmt.Sprintf(\"unable to set %s feature gate, err: %v\", clientgofeaturegate.WatchListClient, err))\n+\t\t\t\t}\n+\t\t\t}\n+\n+\t\t\tapiserverArgs := options.ReadArgsFromFile(opts.APIServerArgsFile)\n+\t\t\tgo daemon.StartAPIServer(apiserverArgs, ctx)\n+\t\t\tdaemon.WaitForAPIServer(opts.KubeconfigFile, 360*time.Second)\n+\n+\t\t\tcontrollerArgs := options.ReadArgsFromFile(opts.ControllerManagerArgsFile)\n+\t\t\tgo daemon.StartControllerManager(controllerArgs, ctx)\n+\n+\t\t\tschedulerArgs := options.ReadArgsFromFile(opts.SchedulerArgsFile)\n+\t\t\tgo daemon.StartScheduler(schedulerArgs, ctx)\n+\t\t}\n+\n+\t\tproxyArgs := options.ReadArgsFromFile(opts.ProxyArgsFile)\n+\t\tgo daemon.StartProxy(proxyArgs)\n+\n+\t\tkubeletArgs := options.ReadArgsFromFile(opts.KubeletArgsFile)\n+\t\tdaemon.StartKubelet(kubeletArgs, ctx)\n+\t},\n+}\n+\n+// Execute adds all child commands to the root command and sets flags appropriately.\n+// This is called by main.main(). It only needs to happen once to the liteCmd.\n+func Execute() {\n+\tif err := liteCmd.Execute(); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+}\n+\n+func init() {\n+\tcobra.OnInitialize()\n+\n+\tliteCmd.Flags().StringVar(&opts.SchedulerArgsFile, \"scheduler-args-file\", opts.SchedulerArgsFile, \"file with the arguments for the scheduler\")\n+\tliteCmd.Flags().StringVar(&opts.ControllerManagerArgsFile, \"controller-manager-args-file\", opts.ControllerManagerArgsFile, \"file with the arguments for the controller manager\")\n+\tliteCmd.Flags().StringVar(&opts.ProxyArgsFile, \"proxy-args-file\", opts.ProxyArgsFile, \"file with the arguments for kube-proxy\")\n+\tliteCmd.Flags().StringVar(&opts.KubeletArgsFile, \"kubelet-args-file\", opts.KubeletArgsFile, \"file with the arguments for kubelet\")\n+\tliteCmd.Flags().StringVar(&opts.APIServerArgsFile, \"apiserver-args-file\", opts.APIServerArgsFile, \"file with the arguments for the API server\")\n+\tliteCmd.Flags().StringVar(&opts.KubeconfigFile, \"kubeconfig-file\", opts.KubeconfigFile, \"the kubeconfig file to use to healthcheck the API server\")\n+\tliteCmd.Flags().BoolVar(&opts.StartControlPlane, \"start-control-plane\", opts.StartControlPlane, \"start the control plane (API server, scheduler and controller manager)\")\n+}\ndiff --git a/cmd/kubelite/kubelite.go b/cmd/kubelite/kubelite.go\nnew file mode 100644\nindex 00000000000..30ab604f480\n--- /dev/null\n+++ b/cmd/kubelite/kubelite.go\n@@ -0,0 +1,25 @@\n+package main\n+\n+import (\n+\t\"github.com/spf13/pflag\"\n+\tcliflag \"k8s.io/component-base/cli/flag\"\n+\n+\t\"k8s.io/component-base/logs\"\n+\t_ \"k8s.io/component-base/metrics/prometheus/clientgo\" // load all the prometheus client-go plugin\n+\t_ \"k8s.io/component-base/metrics/prometheus/version\" // for version metric registration\n+\t\"k8s.io/kubernetes/cmd/kubelite/app\"\n+)\n+\n+func main() {\n+\tprintln(\"Starting kubelite\")\n+\t// TODO: once we switch everything over to Cobra commands, we can go back to calling\n+\t// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the\n+\t// normalize func and add the go flag set by hand.\n+\tpflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)\n+\t// utilflag.InitFlags()\n+\tlogs.InitLogs()\n+\tdefer logs.FlushLogs()\n+\n+\tapp.Execute()\n+\tprintln(\"Stopping kubelite\")\n+}\ndiff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go\nindex d2d830376da..fa6dbc91fef 100644\n--- a/pkg/volume/csi/csi_plugin.go\n+++ b/pkg/volume/csi/csi_plugin.go\n@@ -347,18 +347,23 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {\n \t}\n \n \t// Initializing the label management channels\n-\tnim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n+\tlocalNim := nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins)\n \tPluginHandler.csiPlugin = p\n \n \t// This function prevents Kubelet from posting Ready status until CSINode\n \t// is both installed and initialized\n-\tif err := initializeCSINode(host, p.csiDriverInformer); err != nil {\n+\tif err := initializeCSINode(host, p.csiDriverInformer, localNim); err != nil {\n \t\treturn errors.New(log(\"failed to initialize CSINode: %v\", err))\n \t}\n+\n+\tif _, ok := host.(volume.KubeletVolumeHost); ok {\n+\t\tnim = localNim\n+\t}\n+\n \treturn nil\n }\n \n-func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer) error {\n+func initializeCSINode(host volume.VolumeHost, csiDriverInformer cache.SharedIndexInformer, nim nodeinfomanager.Interface) error {\n \tkvh, ok := host.(volume.KubeletVolumeHost)\n \tif !ok {\n \t\tklog.V(4).Info(\"Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet\")\n-- \n2.48.1\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.35.0/0001-Set-log-reapply-handling-to-ignore-unchanged.patch", "content": "From e2e2155d05b3ddd16640272bd1360425a3883c78 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 27 Jul 2023 18:08:00 +0300\nSubject: [PATCH 2/2] Set log reapply handling to ignore unchanged\n\n---\n staging/src/k8s.io/component-base/logs/api/v1/options.go | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n\ndiff --git a/staging/src/k8s.io/component-base/logs/api/v1/options.go b/staging/src/k8s.io/component-base/logs/api/v1/options.go\nindex 4c8a0d2c53f..6cb6ae1f42d 100644\n--- a/staging/src/k8s.io/component-base/logs/api/v1/options.go\n+++ b/staging/src/k8s.io/component-base/logs/api/v1/options.go\n@@ -65,7 +65,7 @@ func NewLoggingConfiguration() *LoggingConfiguration {\n // are no goroutines which might call logging functions. The default for ValidateAndApply\n // and ValidateAndApplyWithOptions is to return an error when called more than once.\n // Binaries and unit tests can override that behavior.\n-var ReapplyHandling = ReapplyHandlingError\n+var ReapplyHandling = ReapplyHandlingIgnoreUnchanged\n \n type ReapplyHandlingType int\n \n-- \n2.43.0\n\n" }, { "path": "build-scripts/components/kubernetes/patches/v1.35.0/0002-fix-allow-node-to-get-endpointslices.patch", "content": "From dea2abd80878be1eff519216c0bad5a0e35462ec Mon Sep 17 00:00:00 2001\nFrom: Mateo Florido \nDate: Thu, 11 Sep 2025 17:36:10 -0500\nSubject: [PATCH] fix: allow node to get endpointslices\n\n---\n plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go | 1 +\n 1 file changed, 1 insertion(+)\n\ndiff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go\nindex 447b0bc2e99..daa3bde6b1c 100644\n--- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go\n+++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go\n@@ -228,6 +228,7 @@ func NodeRules() []rbacv1.PolicyRule {\n \t\t// TODO: add to the Node authorizer and restrict to endpoints referenced by pods or PVs bound to the node\n \t\t// Needed for glusterfs volumes\n \t\trbacv1helpers.NewRule(\"get\").Groups(legacyGroup).Resources(\"endpoints\").RuleOrDie(),\n+\t\trbacv1helpers.NewRule(\"get\", \"list\", \"watch\").Groups(discoveryGroup).Resources(\"endpointslices\").RuleOrDie(),\n \t\t// Used to create a certificatesigningrequest for a node-specific client certificate, and watch\n \t\t// for it to be signed. This allows the kubelet to rotate it's own certificate.\n \t\trbacv1helpers.NewRule(\"create\", \"get\", \"list\", \"watch\").Groups(certificatesGroup).Resources(\"certificatesigningrequests\").RuleOrDie(),\n-- \n2.48.1\n\n" }, { "path": "build-scripts/components/kubernetes/pre-patch.sh", "content": "#!/bin/bash -x\n\n# Ensure clean Kubernetes version\nKUBE_ROOT=\"${PWD}\"\nsource \"${KUBE_ROOT}/hack/lib/version.sh\"\nkube::version::get_version_vars\nkube::version::save_version_vars \"${PWD}/.version.sh\"\n" }, { "path": "build-scripts/components/kubernetes/repository", "content": "https://github.com/kubernetes/kubernetes\n" }, { "path": "build-scripts/components/kubernetes/version.sh", "content": "#!/bin/bash\n\nKUBE_TRACK=\"${KUBE_TRACK:-}\" # example: \"1.24\"\nKUBE_VERSION=\"${KUBE_VERSION:-}\" # example: \"v1.24.2\"\n\nif [ -z \"${KUBE_VERSION}\" ]; then\n if [ -z \"${KUBE_TRACK}\" ]; then\n KUBE_VERSION=\"$(curl -L --silent \"https://dl.k8s.io/release/stable.txt\")\"\n else\n KUBE_TRACK=\"${KUBE_TRACK#v}\"\n KUBE_VERSION=\"$(curl -L --silent \"https://dl.k8s.io/release/stable-${KUBE_TRACK}.txt\")\"\n fi\nfi\n\necho \"${KUBE_VERSION}\"\n" }, { "path": "build-scripts/components/microk8s-completion/build.sh", "content": "#!/bin/bash\n\nINSTALL=\"${1}\"\n\ngo mod tidy -compat=1.17\ngo run -tags microk8s_hack ./cmd/helm 2> /dev/null\n\ncp microk8s.bash \"${INSTALL}/microk8s.bash\"\n" }, { "path": "build-scripts/components/microk8s-completion/patches/default/0001-microk8s-autocompleter-script.patch", "content": "From 18e46f3fe5fdb6bb90c5c39b923c57d255733e1e Mon Sep 17 00:00:00 2001\nFrom: Homayoon Alimohammadi \nDate: Thu, 27 Nov 2025 11:39:44 +0400\nSubject: [PATCH] microk8s autocompleter script\n\nSigned-off-by: Homayoon Alimohammadi \n---\n cmd/helm/hack_microk8s_autocompleter.go | 51 +++++++++++++++++++++++++\n cmd/helm/helm.go | 2 +-\n go.mod | 6 +++\n go.sum | 12 ++++++\n 4 files changed, 70 insertions(+), 1 deletion(-)\n create mode 100644 cmd/helm/hack_microk8s_autocompleter.go\n\ndiff --git a/cmd/helm/hack_microk8s_autocompleter.go b/cmd/helm/hack_microk8s_autocompleter.go\nnew file mode 100644\nindex 000000000..e819f44be\n--- /dev/null\n+++ b/cmd/helm/hack_microk8s_autocompleter.go\n@@ -0,0 +1,51 @@\n+// go:build microk8s_hack\n+\n+package main\n+\n+import (\n+\t\"bytes\"\n+\t\"log\"\n+\t\"os\"\n+\t\"strings\"\n+\n+\t\"github.com/spf13/cobra\"\n+\t\"helm.sh/helm/v3/pkg/action\"\n+\t\"k8s.io/kubectl/pkg/cmd\"\n+)\n+\n+func main() {\n+\tcommand := &cobra.Command{Use: \"microk8s\"}\n+\tcommand.AddCommand(cmd.NewDefaultKubectlCommand())\n+\thelmCmd, _ := newRootCmd(&action.Configuration{}, nil, nil)\n+\thelmCmd.Use = \"helm\"\n+\thelm3Cmd, _ := newRootCmd(&action.Configuration{}, nil, nil)\n+\thelm3Cmd.Use = \"helm3\"\n+\tcommand.AddCommand(&cobra.Command{Use: \"addons\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"add-node\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"config\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"enable\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"disable\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(helmCmd)\n+\tcommand.AddCommand(helm3Cmd)\n+\tcommand.AddCommand(&cobra.Command{Use: \"helm3\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"images\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"inspect\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"join\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"refresh-certs\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"remove-node\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"reset\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"start\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"status\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"stop\", Run: func(cmd *cobra.Command, args []string) {}})\n+\tcommand.AddCommand(&cobra.Command{Use: \"version\", Run: func(cmd *cobra.Command, args []string) {}})\n+\n+\tvar b bytes.Buffer\n+\tif err := command.GenBashCompletion(&b); err != nil {\n+\t\tlog.Fatalf(\"failed to generate completion script: %v\", err)\n+\t}\n+\tcompletionScript := b.String()\n+\tcompletionScript = strings.ReplaceAll(completionScript, `args=(\"${words[@]:1}\")`, `args=(\"${words[@]:2}\")`)\n+\tcompletionScript = strings.ReplaceAll(completionScript, `requestComp=\"${words[0]} __completeNoDesc ${args[*]}\"`, `requestComp=\"${words[0]} ${words[1]} __completeNoDesc ${args[*]}\"`)\n+\n+\tos.WriteFile(\"microk8s.bash\", []byte(completionScript), 0644)\n+}\ndiff --git a/cmd/helm/helm.go b/cmd/helm/helm.go\nindex 7bca93358..2008c1b08 100644\n--- a/cmd/helm/helm.go\n+++ b/cmd/helm/helm.go\n@@ -62,7 +62,7 @@ func hookOutputWriter(_, _, _ string) io.Writer {\n \treturn log.Writer()\n }\n \n-func main() {\n+func Main() {\n \t// Setting the name of the app for managedFields in the Kubernetes client.\n \t// It is set here to the full name of \"helm\" so that renaming of helm to\n \t// another name (e.g., helm2 or helm3) does not change the name of the\ndiff --git a/go.mod b/go.mod\nindex f2cecf187..534dfe524 100644\n--- a/go.mod\n+++ b/go.mod\n@@ -73,6 +73,7 @@ require (\n \tgithub.com/docker/go-metrics v0.0.1 // indirect\n \tgithub.com/emicklei/go-restful/v3 v3.12.2 // indirect\n \tgithub.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect\n+\tgithub.com/fatih/camelcase v1.0.0 // indirect\n \tgithub.com/fatih/color v1.13.0 // indirect\n \tgithub.com/felixge/httpsnoop v1.0.4 // indirect\n \tgithub.com/fxamacker/cbor/v2 v2.9.0 // indirect\n@@ -98,12 +99,14 @@ require (\n \tgithub.com/hashicorp/golang-lru/v2 v2.0.5 // indirect\n \tgithub.com/huandu/xstrings v1.5.0 // indirect\n \tgithub.com/inconshreveable/mousetrap v1.1.0 // indirect\n+\tgithub.com/jonboulle/clockwork v0.5.0 // indirect\n \tgithub.com/josharian/intern v1.0.0 // indirect\n \tgithub.com/json-iterator/go v1.1.12 // indirect\n \tgithub.com/klauspost/compress v1.18.0 // indirect\n \tgithub.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect\n \tgithub.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect\n \tgithub.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect\n+\tgithub.com/lithammer/dedent v1.1.0 // indirect\n \tgithub.com/mailru/easyjson v0.7.7 // indirect\n \tgithub.com/mattn/go-colorable v0.1.13 // indirect\n \tgithub.com/mattn/go-isatty v0.0.17 // indirect\n@@ -173,10 +176,13 @@ require (\n \tgopkg.in/inf.v0 v0.9.1 // indirect\n \tgopkg.in/yaml.v2 v2.4.0 // indirect\n \tk8s.io/component-base v0.34.0 // indirect\n+\tk8s.io/component-helpers v0.34.0 // indirect\n \tk8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect\n+\tk8s.io/metrics v0.34.0 // indirect\n \tk8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect\n \tsigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect\n \tsigs.k8s.io/kustomize/api v0.20.1 // indirect\n+\tsigs.k8s.io/kustomize/kustomize/v5 v5.7.1 // indirect\n \tsigs.k8s.io/kustomize/kyaml v0.20.1 // indirect\n \tsigs.k8s.io/randfill v1.0.0 // indirect\n \tsigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect\ndiff --git a/go.sum b/go.sum\nindex 217119fcb..bd7f38567 100644\n--- a/go.sum\n+++ b/go.sum\n@@ -90,6 +90,8 @@ github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb\n github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=\n github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4=\n github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc=\n+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=\n+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=\n github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=\n github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=\n github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=\n@@ -177,6 +179,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2\n github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=\n github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=\n github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=\n+github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=\n+github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=\n github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=\n github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=\n github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=\n@@ -208,6 +212,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=\n github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=\n github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=\n github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=\n+github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=\n+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=\n github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=\n github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=\n github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=\n@@ -533,12 +539,16 @@ k8s.io/client-go v0.34.0 h1:YoWv5r7bsBfb0Hs2jh8SOvFbKzzxyNo0nSb0zC19KZo=\n k8s.io/client-go v0.34.0/go.mod h1:ozgMnEKXkRjeMvBZdV1AijMHLTh3pbACPvK7zFR+QQY=\n k8s.io/component-base v0.34.0 h1:bS8Ua3zlJzapklsB1dZgjEJuJEeHjj8yTu1gxE2zQX8=\n k8s.io/component-base v0.34.0/go.mod h1:RSCqUdvIjjrEm81epPcjQ/DS+49fADvGSCkIP3IC6vg=\n+k8s.io/component-helpers v0.34.0 h1:5T7P9XGMoUy1JDNKzHf0p/upYbeUf8ZaSf9jbx0QlIo=\n+k8s.io/component-helpers v0.34.0/go.mod h1:kaOyl5tdtnymriYcVZg4uwDBe2d1wlIpXyDkt6sVnt4=\n k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=\n k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=\n k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA=\n k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts=\n k8s.io/kubectl v0.34.0 h1:NcXz4TPTaUwhiX4LU+6r6udrlm0NsVnSkP3R9t0dmxs=\n k8s.io/kubectl v0.34.0/go.mod h1:bmd0W5i+HuG7/p5sqicr0Li0rR2iIhXL0oUyLF3OjR4=\n+k8s.io/metrics v0.34.0 h1:nYSfG2+tnL6/MRC2I+sGHjtNEGoEoM/KktgGOoQFwws=\n+k8s.io/metrics v0.34.0/go.mod h1:KCuXmotE0v4AvoARKUP8NC4lUnbK/Du1mluGdor5h4M=\n k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y=\n k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=\n oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc=\n@@ -547,6 +557,8 @@ sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7np\n sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=\n sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I=\n sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM=\n+sigs.k8s.io/kustomize/kustomize/v5 v5.7.1 h1:sYJsarwy/SDJfjjLMUqwFDGPwzUtMOQ1i1Ed49+XSbw=\n+sigs.k8s.io/kustomize/kustomize/v5 v5.7.1/go.mod h1:+5/SrBcJ4agx1SJknGuR/c9thwRSKLxnKoI5BzXFaLU=\n sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78=\n sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po=\n sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=\n-- \n2.48.1\n\n" }, { "path": "build-scripts/components/microk8s-completion/repository", "content": "https://github.com/helm/helm\n" }, { "path": "build-scripts/components/microk8s-completion/version.sh", "content": "#!/bin/bash\n\necho \"v3.19.2\"\n" }, { "path": "build-scripts/components/python/requirements.txt", "content": "PyYAML==6.0.1\nnetifaces==0.10.9\njsonschema==4.0.0\n" }, { "path": "build-scripts/components/runc/build.sh", "content": "#!/bin/bash\n\nVERSION=\"${2}\"\n\nexport INSTALL=\"${1}/bin\"\nmkdir -p \"${INSTALL}\"\n\n# Ensure `runc --version` prints the correct release commit\nexport COMMIT=\"$(git describe --always --long \"${VERSION}\")\"\n\nmake BUILDTAGS=\"seccomp apparmor\" EXTRA_LDFLAGS=\"-s -w\" static\ncp runc \"${INSTALL}/runc\"\n" }, { "path": "build-scripts/components/runc/patches/default/0001-Disable-static-PIE-on-arm64.patch", "content": "From 7b7171f0f5048225e0d914cbffd47295af1fbfc5 Mon Sep 17 00:00:00 2001\nFrom: Lucian Petrut \nDate: Fri, 28 Mar 2025 13:14:34 +0000\nSubject: [PATCH] Disable static PIE on arm64\n\nUbuntu does not currently have the rcrt1.o file on arm64\n---\n Makefile | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n\ndiff --git a/Makefile b/Makefile\nindex 0a15fd90..d7e6209a 100644\n--- a/Makefile\n+++ b/Makefile\n@@ -45,7 +45,7 @@ LDFLAGS_STATIC := -extldflags -static\n # Enable static PIE executables on supported platforms.\n # This (among the other things) requires libc support (rcrt1.o), which seems\n # to be available only for arm64 and amd64 (Debian Bullseye).\n-ifneq (,$(filter $(GOARCH),arm64 amd64))\n+ifneq (,$(filter $(GOARCH),amd64))\n \tifeq (,$(findstring -race,$(EXTRA_FLAGS)))\n \t\tGO_BUILDMODE_STATIC := -buildmode=pie\n \t\tLDFLAGS_STATIC := -linkmode external -extldflags -static-pie\n-- \n2.43.0\n\n" }, { "path": "build-scripts/components/runc/repository", "content": "https://github.com/opencontainers/runc\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.1.12/0001-apparmor-change-profile-immediately-not-on-exec.patch", "content": "From a367e391600dfab0d9eb3deaec4db300a2fb1fa1 Mon Sep 17 00:00:00 2001\nFrom: Alberto Mardegan \nDate: Wed, 16 Jun 2021 15:04:16 +0300\nSubject: [PATCH 1/3] apparmor: change profile immediately, not on exec\n\n---\n libcontainer/apparmor/apparmor_linux.go | 8 ++++----\n 1 file changed, 4 insertions(+), 4 deletions(-)\n\ndiff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go\nindex 8b1483c..292cfa6 100644\n--- a/libcontainer/apparmor/apparmor_linux.go\n+++ b/libcontainer/apparmor/apparmor_linux.go\n@@ -48,9 +48,9 @@ func setProcAttr(attr, value string) error {\n \treturn err\n }\n\n-// changeOnExec reimplements aa_change_onexec from libapparmor in Go\n-func changeOnExec(name string) error {\n-\tif err := setProcAttr(\"exec\", \"exec \"+name); err != nil {\n+// changeProfile reimplements aa_change_profile from libapparmor in Go\n+func changeProfile(name string) error {\n+\tif err := setProcAttr(\"current\", \"changeprofile \"+name); err != nil {\n \t\treturn fmt.Errorf(\"apparmor failed to apply profile: %w\", err)\n \t}\n \treturn nil\n@@ -64,5 +64,5 @@ func applyProfile(name string) error {\n \t\treturn nil\n \t}\n\n-\treturn changeOnExec(name)\n+\treturn changeProfile(name)\n }\n--\n2.34.1\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.1.12/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch", "content": "From 5351ef6f5b592472e077512714b2516cdbae1b51 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 1 Feb 2024 11:23:08 +0200\nSubject: [PATCH 2/3] setns_init_linux: set the NNP flag after changing the\n apparmor profile\n\nWith the current version of the AppArmor kernel module, it's not\npossible to switch the AppArmor profile if the NoNewPrivileges flag is\nset. So, we invert the order of the two operations.\n\nAdjusts the previous patch for runc version v1.1.12\n\nCo-Authored-By: Alberto Mardegan \n---\n libcontainer/setns_init_linux.go | 10 +++++-----\n 1 file changed, 5 insertions(+), 5 deletions(-)\n\ndiff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go\nindex d1bb122..00407ce 100644\n--- a/libcontainer/setns_init_linux.go\n+++ b/libcontainer/setns_init_linux.go\n@@ -56,11 +56,6 @@ func (l *linuxSetnsInit) Init() error {\n \t\t\treturn err\n \t\t}\n \t}\n-\tif l.config.NoNewPrivileges {\n-\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n-\t\t\treturn err\n-\t\t}\n-\t}\n \tif err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {\n \t\treturn err\n \t}\n@@ -84,6 +79,11 @@ func (l *linuxSetnsInit) Init() error {\n \tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n \t\treturn err\n \t}\n+\tif l.config.NoNewPrivileges {\n+\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n+\t\t\treturn err\n+\t\t}\n+\t}\n\n \t// Check for the arg before waiting to make sure it exists and it is\n \t// returned as a create time error.\n--\n2.34.1" }, { "path": "build-scripts/components/runc/strict-patches/v1.1.12/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch", "content": "From 103a94a51ea334d25bf573f2f20cd4d9a099d827 Mon Sep 17 00:00:00 2001\nFrom: Alberto Mardegan \nDate: Thu, 17 Jun 2021 14:31:35 +0300\nSubject: [PATCH 3/3] standard_init_linux: change AppArmor profile as late as\n possible\n\n---\n libcontainer/standard_init_linux.go | 18 +++++++++---------\n 1 file changed, 9 insertions(+), 9 deletions(-)\n\ndiff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go\nindex d1d9435..7097571 100644\n--- a/libcontainer/standard_init_linux.go\n+++ b/libcontainer/standard_init_linux.go\n@@ -127,10 +127,6 @@ func (l *linuxStandardInit) Init() error {\n \t\t\treturn &os.SyscallError{Syscall: \"sethostname\", Err: err}\n \t\t}\n \t}\n-\tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n-\t\treturn fmt.Errorf(\"unable to apply apparmor profile: %w\", err)\n-\t}\n-\n \tfor key, value := range l.config.Config.Sysctl {\n \t\tif err := writeSystemProperty(key, value); err != nil {\n \t\t\treturn err\n@@ -150,17 +146,21 @@ func (l *linuxStandardInit) Init() error {\n \tif err != nil {\n \t\treturn fmt.Errorf(\"can't get pdeath signal: %w\", err)\n \t}\n-\tif l.config.NoNewPrivileges {\n-\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n-\t\t\treturn &os.SyscallError{Syscall: \"prctl(SET_NO_NEW_PRIVS)\", Err: err}\n-\t\t}\n-\t}\n \t// Tell our parent that we're ready to Execv. This must be done before the\n \t// Seccomp rules have been applied, because we need to be able to read and\n \t// write to a socket.\n \tif err := syncParentReady(l.pipe); err != nil {\n \t\treturn fmt.Errorf(\"sync ready: %w\", err)\n \t}\n+\tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n+\t\treturn fmt.Errorf(\"apply apparmor profile: %w\", err)\n+\t}\n+\tif l.config.NoNewPrivileges {\n+\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n+\t\t\treturn fmt.Errorf(\"set nonewprivileges: %w\", err)\n+\t\t}\n+\t}\n+\n \tif err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {\n \t\treturn fmt.Errorf(\"can't set process label: %w\", err)\n \t}\n--\n2.34.1" }, { "path": "build-scripts/components/runc/strict-patches/v1.1.15/0001-apparmor-change-profile-immediately-not-on-exec.patch", "content": "From a367e391600dfab0d9eb3deaec4db300a2fb1fa1 Mon Sep 17 00:00:00 2001\nFrom: Alberto Mardegan \nDate: Wed, 16 Jun 2021 15:04:16 +0300\nSubject: [PATCH 1/3] apparmor: change profile immediately, not on exec\n\n---\n libcontainer/apparmor/apparmor_linux.go | 8 ++++----\n 1 file changed, 4 insertions(+), 4 deletions(-)\n\ndiff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go\nindex 8b1483c..292cfa6 100644\n--- a/libcontainer/apparmor/apparmor_linux.go\n+++ b/libcontainer/apparmor/apparmor_linux.go\n@@ -48,9 +48,9 @@ func setProcAttr(attr, value string) error {\n \treturn err\n }\n\n-// changeOnExec reimplements aa_change_onexec from libapparmor in Go\n-func changeOnExec(name string) error {\n-\tif err := setProcAttr(\"exec\", \"exec \"+name); err != nil {\n+// changeProfile reimplements aa_change_profile from libapparmor in Go\n+func changeProfile(name string) error {\n+\tif err := setProcAttr(\"current\", \"changeprofile \"+name); err != nil {\n \t\treturn fmt.Errorf(\"apparmor failed to apply profile: %w\", err)\n \t}\n \treturn nil\n@@ -64,5 +64,5 @@ func applyProfile(name string) error {\n \t\treturn nil\n \t}\n\n-\treturn changeOnExec(name)\n+\treturn changeProfile(name)\n }\n--\n2.34.1\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.1.15/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch", "content": "From b145a4ac9e9cd09e82d35e0998c6ddee80854275 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 1 Feb 2024 11:23:08 +0200\nSubject: [PATCH] setns_init_linux: set the NNP flag after changing the\n apparmor profile\n\nWith the current version of the AppArmor kernel module, it's not\npossible to switch the AppArmor profile if the NoNewPrivileges flag is\nset. So, we invert the order of the two operations.\n\nAdjusts the previous patch for runc version v1.1.12\n\nCo-Authored-By: Alberto Mardegan \n---\n libcontainer/setns_init_linux.go | 10 +++++-----\n 1 file changed, 5 insertions(+), 5 deletions(-)\n\ndiff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go\nindex bb358901..6c1b16bd 100644\n--- a/libcontainer/setns_init_linux.go\n+++ b/libcontainer/setns_init_linux.go\n@@ -57,11 +57,6 @@ func (l *linuxSetnsInit) Init() error {\n \t\t\treturn err\n \t\t}\n \t}\n-\tif l.config.NoNewPrivileges {\n-\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n-\t\t\treturn err\n-\t\t}\n-\t}\n \n \t// Tell our parent that we're ready to exec. This must be done before the\n \t// Seccomp rules have been applied, because we need to be able to read and\n@@ -93,6 +88,11 @@ func (l *linuxSetnsInit) Init() error {\n \tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n \t\treturn err\n \t}\n+\tif l.config.NoNewPrivileges {\n+\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n+\t\t\treturn err\n+\t\t}\n+\t}\n \n \t// Check for the arg before waiting to make sure it exists and it is\n \t// returned as a create time error.\n-- \n2.43.0\n\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.1.15/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch", "content": "From f9e0ca2f29c6c77ea9bc9c52929dac3915545dd9 Mon Sep 17 00:00:00 2001\nFrom: Alberto Mardegan \nDate: Thu, 17 Jun 2021 14:31:35 +0300\nSubject: [PATCH] standard_init_linux: change AppArmor profile as late as\n possible\n\n---\n libcontainer/standard_init_linux.go | 18 +++++++++---------\n 1 file changed, 9 insertions(+), 9 deletions(-)\n\ndiff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go\nindex d9a6a224..e4d603e4 100644\n--- a/libcontainer/standard_init_linux.go\n+++ b/libcontainer/standard_init_linux.go\n@@ -127,10 +127,6 @@ func (l *linuxStandardInit) Init() error {\n \t\t\treturn &os.SyscallError{Syscall: \"sethostname\", Err: err}\n \t\t}\n \t}\n-\tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n-\t\treturn fmt.Errorf(\"unable to apply apparmor profile: %w\", err)\n-\t}\n-\n \tfor key, value := range l.config.Config.Sysctl {\n \t\tif err := writeSystemProperty(key, value); err != nil {\n \t\t\treturn err\n@@ -150,11 +146,6 @@ func (l *linuxStandardInit) Init() error {\n \tif err != nil {\n \t\treturn fmt.Errorf(\"can't get pdeath signal: %w\", err)\n \t}\n-\tif l.config.NoNewPrivileges {\n-\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n-\t\t\treturn &os.SyscallError{Syscall: \"prctl(SET_NO_NEW_PRIVS)\", Err: err}\n-\t\t}\n-\t}\n \n \t// Tell our parent that we're ready to exec. This must be done before the\n \t// Seccomp rules have been applied, because we need to be able to read and\n@@ -162,6 +153,15 @@ func (l *linuxStandardInit) Init() error {\n \tif err := syncParentReady(l.pipe); err != nil {\n \t\treturn fmt.Errorf(\"sync ready: %w\", err)\n \t}\n+\tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n+\t\treturn fmt.Errorf(\"apply apparmor profile: %w\", err)\n+\t}\n+\tif l.config.NoNewPrivileges {\n+\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n+\t\t\treturn fmt.Errorf(\"set nonewprivileges: %w\", err)\n+\t\t}\n+\t}\n+\n \tif err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {\n \t\treturn fmt.Errorf(\"can't set process label: %w\", err)\n \t}\n-- \n2.43.0\n\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.2.6/0001-apparmor-change-profile-immediately-not-on-exec.patch", "content": "From 5cdb43bdc26e81be36d93fd8b81b7de6ad152c22 Mon Sep 17 00:00:00 2001\nFrom: Alberto Mardegan \nDate: Wed, 16 Jun 2021 15:04:16 +0300\nSubject: [PATCH 1/3] apparmor: change profile immediately, not on exec\n\n---\n libcontainer/apparmor/apparmor_linux.go | 8 ++++----\n 1 file changed, 4 insertions(+), 4 deletions(-)\n\ndiff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go\nindex 17d36ed1..fb159f3c 100644\n--- a/libcontainer/apparmor/apparmor_linux.go\n+++ b/libcontainer/apparmor/apparmor_linux.go\n@@ -53,9 +53,9 @@ func setProcAttr(attr, value string) error {\n \treturn err\n }\n \n-// changeOnExec reimplements aa_change_onexec from libapparmor in Go\n-func changeOnExec(name string) error {\n-\tif err := setProcAttr(\"exec\", \"exec \"+name); err != nil {\n+// changeProfile reimplements aa_change_profile from libapparmor in Go\n+func changeProfile(name string) error {\n+\tif err := setProcAttr(\"current\", \"changeprofile \"+name); err != nil {\n \t\treturn fmt.Errorf(\"apparmor failed to apply profile: %w\", err)\n \t}\n \treturn nil\n@@ -69,5 +69,5 @@ func applyProfile(name string) error {\n \t\treturn nil\n \t}\n \n-\treturn changeOnExec(name)\n+\treturn changeProfile(name)\n }\n-- \n2.49.0\n\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.2.6/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch", "content": "From 259ebdf71e84433f55c1b28efb206ccc3a1b2736 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 1 Feb 2024 11:23:08 +0200\nSubject: [PATCH 2/3] setns_init_linux: set the NNP flag after changing the\n apparmor profile\n\nWith the current version of the AppArmor kernel module, it's not\npossible to switch the AppArmor profile if the NoNewPrivileges flag is\nset. So, we invert the order of the two operations.\n\nAdjusts the previous patch for runc version v1.1.12\n\nCo-Authored-By: Alberto Mardegan \n---\n libcontainer/setns_init_linux.go | 10 +++++-----\n 1 file changed, 5 insertions(+), 5 deletions(-)\n\ndiff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go\nindex 92c6ef77..e9a55e31 100644\n--- a/libcontainer/setns_init_linux.go\n+++ b/libcontainer/setns_init_linux.go\n@@ -62,11 +62,6 @@ func (l *linuxSetnsInit) Init() error {\n \t\t\treturn fmt.Errorf(\"failed to setup pidfd: %w\", err)\n \t\t}\n \t}\n-\tif l.config.NoNewPrivileges {\n-\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n-\t\t\treturn err\n-\t\t}\n-\t}\n \tif l.config.Config.Umask != nil {\n \t\tunix.Umask(int(*l.config.Config.Umask))\n \t}\n@@ -106,6 +101,11 @@ func (l *linuxSetnsInit) Init() error {\n \tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n \t\treturn err\n \t}\n+\tif l.config.NoNewPrivileges {\n+\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n+\t\t\treturn err\n+\t\t}\n+\t}\n \tif l.config.Config.Personality != nil {\n \t\tif err := setupPersonality(l.config.Config); err != nil {\n \t\t\treturn err\n-- \n2.49.0\n\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.2.6/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch", "content": "From 3f82798f6081f060b244d7346524c2aced231287 Mon Sep 17 00:00:00 2001\nFrom: Alberto Mardegan \nDate: Thu, 17 Jun 2021 14:31:35 +0300\nSubject: [PATCH 3/3] standard_init_linux: change AppArmor profile as late as\n possible\n\n---\n libcontainer/standard_init_linux.go | 17 +++++++++--------\n 1 file changed, 9 insertions(+), 8 deletions(-)\n\ndiff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go\nindex 9f7fa45d..c674d927 100644\n--- a/libcontainer/standard_init_linux.go\n+++ b/libcontainer/standard_init_linux.go\n@@ -126,9 +126,6 @@ func (l *linuxStandardInit) Init() error {\n \t\t\treturn &os.SyscallError{Syscall: \"setdomainname\", Err: err}\n \t\t}\n \t}\n-\tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n-\t\treturn fmt.Errorf(\"unable to apply apparmor profile: %w\", err)\n-\t}\n \n \tfor key, value := range l.config.Config.Sysctl {\n \t\tif err := writeSystemProperty(key, value); err != nil {\n@@ -149,11 +146,6 @@ func (l *linuxStandardInit) Init() error {\n \tif err != nil {\n \t\treturn fmt.Errorf(\"can't get pdeath signal: %w\", err)\n \t}\n-\tif l.config.NoNewPrivileges {\n-\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n-\t\t\treturn &os.SyscallError{Syscall: \"prctl(SET_NO_NEW_PRIVS)\", Err: err}\n-\t\t}\n-\t}\n \n \tif l.config.Config.Scheduler != nil {\n \t\tif err := setupScheduler(l.config.Config); err != nil {\n@@ -172,6 +164,15 @@ func (l *linuxStandardInit) Init() error {\n \tif err := syncParentReady(l.pipe); err != nil {\n \t\treturn fmt.Errorf(\"sync ready: %w\", err)\n \t}\n+\tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n+\t\treturn fmt.Errorf(\"apply apparmor profile: %w\", err)\n+\t}\n+\tif l.config.NoNewPrivileges {\n+\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n+\t\t\treturn fmt.Errorf(\"set nonewprivileges: %w\", err)\n+\t\t}\n+\t}\n+\n \tif err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {\n \t\treturn fmt.Errorf(\"can't set process label: %w\", err)\n \t}\n-- \n2.49.0\n\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.3.0/0001-apparmor-change-profile-immediately-not-on-exec.patch", "content": "From 5cdb43bdc26e81be36d93fd8b81b7de6ad152c22 Mon Sep 17 00:00:00 2001\nFrom: Alberto Mardegan \nDate: Wed, 16 Jun 2021 15:04:16 +0300\nSubject: [PATCH 1/3] apparmor: change profile immediately, not on exec\n\n---\n libcontainer/apparmor/apparmor_linux.go | 8 ++++----\n 1 file changed, 4 insertions(+), 4 deletions(-)\n\ndiff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go\nindex 17d36ed1..fb159f3c 100644\n--- a/libcontainer/apparmor/apparmor_linux.go\n+++ b/libcontainer/apparmor/apparmor_linux.go\n@@ -53,9 +53,9 @@ func setProcAttr(attr, value string) error {\n \treturn err\n }\n \n-// changeOnExec reimplements aa_change_onexec from libapparmor in Go\n-func changeOnExec(name string) error {\n-\tif err := setProcAttr(\"exec\", \"exec \"+name); err != nil {\n+// changeProfile reimplements aa_change_profile from libapparmor in Go\n+func changeProfile(name string) error {\n+\tif err := setProcAttr(\"current\", \"changeprofile \"+name); err != nil {\n \t\treturn fmt.Errorf(\"apparmor failed to apply profile: %w\", err)\n \t}\n \treturn nil\n@@ -69,5 +69,5 @@ func applyProfile(name string) error {\n \t\treturn nil\n \t}\n \n-\treturn changeOnExec(name)\n+\treturn changeProfile(name)\n }\n-- \n2.49.0\n\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.3.0/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch", "content": "From 259ebdf71e84433f55c1b28efb206ccc3a1b2736 Mon Sep 17 00:00:00 2001\nFrom: Angelos Kolaitis \nDate: Thu, 1 Feb 2024 11:23:08 +0200\nSubject: [PATCH 2/3] setns_init_linux: set the NNP flag after changing the\n apparmor profile\n\nWith the current version of the AppArmor kernel module, it's not\npossible to switch the AppArmor profile if the NoNewPrivileges flag is\nset. So, we invert the order of the two operations.\n\nAdjusts the previous patch for runc version v1.1.12\n\nCo-Authored-By: Alberto Mardegan \n---\n libcontainer/setns_init_linux.go | 10 +++++-----\n 1 file changed, 5 insertions(+), 5 deletions(-)\n\ndiff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go\nindex 92c6ef77..e9a55e31 100644\n--- a/libcontainer/setns_init_linux.go\n+++ b/libcontainer/setns_init_linux.go\n@@ -62,11 +62,6 @@ func (l *linuxSetnsInit) Init() error {\n \t\t\treturn fmt.Errorf(\"failed to setup pidfd: %w\", err)\n \t\t}\n \t}\n-\tif l.config.NoNewPrivileges {\n-\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n-\t\t\treturn err\n-\t\t}\n-\t}\n \tif l.config.Config.Umask != nil {\n \t\tunix.Umask(int(*l.config.Config.Umask))\n \t}\n@@ -106,6 +101,11 @@ func (l *linuxSetnsInit) Init() error {\n \tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n \t\treturn err\n \t}\n+\tif l.config.NoNewPrivileges {\n+\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n+\t\t\treturn err\n+\t\t}\n+\t}\n \tif l.config.Config.Personality != nil {\n \t\tif err := setupPersonality(l.config.Config); err != nil {\n \t\t\treturn err\n-- \n2.49.0\n\n" }, { "path": "build-scripts/components/runc/strict-patches/v1.3.0/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch", "content": "From 7f91e445a8731856e2d22b2295d8438e07cf2bf7 Mon Sep 17 00:00:00 2001\nFrom: Alberto Mardegan \nDate: Thu, 17 Jun 2021 14:31:35 +0300\nSubject: [PATCH] standard_init_linux: change AppArmor profile as late as\n possible\n\n---\n libcontainer/standard_init_linux.go | 17 +++++++++--------\n 1 file changed, 9 insertions(+), 8 deletions(-)\n\ndiff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go\nindex 384750bf..ccd9297a 100644\n--- a/libcontainer/standard_init_linux.go\n+++ b/libcontainer/standard_init_linux.go\n@@ -126,9 +126,6 @@ func (l *linuxStandardInit) Init() error {\n \t\t\treturn &os.SyscallError{Syscall: \"setdomainname\", Err: err}\n \t\t}\n \t}\n-\tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n-\t\treturn fmt.Errorf(\"unable to apply apparmor profile: %w\", err)\n-\t}\n \n \tfor key, value := range l.config.Config.Sysctl {\n \t\tif err := writeSystemProperty(key, value); err != nil {\n@@ -149,11 +146,6 @@ func (l *linuxStandardInit) Init() error {\n \tif err != nil {\n \t\treturn fmt.Errorf(\"can't get pdeath signal: %w\", err)\n \t}\n-\tif l.config.NoNewPrivileges {\n-\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n-\t\t\treturn &os.SyscallError{Syscall: \"prctl(SET_NO_NEW_PRIVS)\", Err: err}\n-\t\t}\n-\t}\n \n \tif err := setupScheduler(l.config); err != nil {\n \t\treturn err\n@@ -169,6 +161,15 @@ func (l *linuxStandardInit) Init() error {\n \tif err := syncParentReady(l.pipe); err != nil {\n \t\treturn fmt.Errorf(\"sync ready: %w\", err)\n \t}\n+\tif err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {\n+\t\treturn fmt.Errorf(\"apply apparmor profile: %w\", err)\n+\t}\n+\tif l.config.NoNewPrivileges {\n+\t\tif err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {\n+\t\t\treturn fmt.Errorf(\"set nonewprivileges: %w\", err)\n+\t\t}\n+\t}\n+\n \tif err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil {\n \t\treturn fmt.Errorf(\"can't set process label: %w\", err)\n \t}\n-- \n2.43.0\n\n" }, { "path": "build-scripts/components/runc/version.sh", "content": "#!/bin/bash\n\necho \"v1.3.3\"\n" }, { "path": "build-scripts/generate-bom.py", "content": "#!/usr/bin/env python3\n\nimport json\nimport os\nfrom pathlib import Path\nimport subprocess\nimport sys\nimport yaml\n\nDIR = Path(__file__).absolute().parent\n\nSNAPCRAFT_PART_BUILD = Path(os.getenv(\"SNAPCRAFT_PART_BUILD\", \"\"))\nSNAPCRAFT_PART_INSTALL = Path(os.getenv(\"SNAPCRAFT_PART_INSTALL\", \"\"))\n\nBUILD_DIRECTORY = SNAPCRAFT_PART_BUILD.exists() and SNAPCRAFT_PART_BUILD or DIR / \".build\"\nINSTALL_DIRECTORY = SNAPCRAFT_PART_INSTALL.exists() and SNAPCRAFT_PART_INSTALL or DIR / \".install\"\n\n# Location of Python binary\nPYTHON = INSTALL_DIRECTORY / \"..\" / \"..\" / \"python-runtime\" / \"install\" / \"usr\" / \"bin\" / \"python3\"\n\n# Location of MicroK8s addons\nMICROK8S_ADDONS = INSTALL_DIRECTORY / \"..\" / \"..\" / \"microk8s-addons\" / \"install\" / \"addons\"\n\n# List of tools used to build or bundled in the snap\nTOOLS = {\n \"go\": [\"go\", \"version\"],\n \"gcc\": [\"gcc\", \"--version\"],\n \"python\": [PYTHON, \"-B\", \"-VV\"],\n \"python-requirements\": [PYTHON, \"-B\", \"-m\", \"pip\", \"freeze\"],\n}\n\n# Retrieve list of components we care about from the snapcraft.yaml file\nwith open(DIR / \"..\" / \"snap\" / \"snapcraft.yaml\") as fin:\n COMPONENTS = yaml.safe_load(fin)[\"parts\"][\"bom\"][\"after\"]\n\n\ndef _listdir(dir: Path):\n try:\n return sorted(os.listdir(dir))\n except OSError:\n return []\n\n\ndef _parse_output(*args, **kwargs):\n return subprocess.check_output(*args, **kwargs).decode().strip()\n\n\ndef _read_file(path: Path) -> str:\n return path.read_text().strip()\n\n\nif __name__ == \"__main__\":\n BOM = {\n \"microk8s\": {\n \"version\": _parse_output([\"git\", \"rev-parse\", \"--abbrev-ref\", \"HEAD\"]),\n \"revision\": _parse_output([\"git\", \"rev-parse\", \"HEAD\"]),\n },\n \"tools\": {},\n \"components\": {},\n \"addons\": {},\n }\n\n for tool_name, version_cmd in TOOLS.items():\n BOM[\"tools\"][tool_name] = _parse_output(version_cmd).split(\"\\n\")\n\n for component in COMPONENTS:\n component_dir = DIR / \"components\" / component\n\n try:\n version = _parse_output([component_dir / \"version.sh\"])\n patches = _parse_output([PYTHON, DIR / \"print-patches-for.py\", component, version])\n clean_patches = []\n if patches:\n clean_patches = [p[p.find(\"build-scripts/\") :] for p in patches.split(\"\\n\")]\n\n BOM[\"components\"][component] = {\n \"repository\": _read_file(component_dir / \"repository\"),\n \"version\": version,\n \"revision\": _parse_output(\n [\"git\", \"rev-parse\", \"HEAD\"],\n cwd=BUILD_DIRECTORY / \"..\" / \"..\" / component / \"build\" / component,\n ),\n \"patches\": clean_patches,\n }\n except OSError as e:\n print(f\"Could not get info for {component}: {e}\", file=sys.stderr)\n\n for repo in _listdir(MICROK8S_ADDONS):\n repo_dir = MICROK8S_ADDONS / repo\n if not repo_dir.is_dir():\n continue\n\n BOM[\"addons\"][repo] = {\n \"repository\": _parse_output([\"git\", \"remote\", \"get-url\", \"origin\"], cwd=repo_dir),\n \"version\": _parse_output([\"git\", \"rev-parse\", \"--abbrev-ref\", \"HEAD\"], cwd=repo_dir),\n \"revision\": _parse_output([\"git\", \"rev-parse\", \"HEAD\"], cwd=repo_dir),\n }\n\n print(json.dumps(BOM, indent=2))\n" }, { "path": "build-scripts/images.txt", "content": "docker.io/calico/cni:v3.29.3\ndocker.io/calico/kube-controllers:v3.29.3\ndocker.io/calico/node:v3.29.3\ndocker.io/cdkbot/hostpath-provisioner:1.5.0\ndocker.io/coredns/coredns:1.12.3\ndocker.io/library/busybox:1.28.4\ndocker.io/traefik:v3.6.2\nregistry.k8s.io/metrics-server/metrics-server:v0.8.0\nregistry.k8s.io/pause:3.10\n" }, { "path": "build-scripts/print-patches-for.py", "content": "#!/usr/bin/env python3\n\nimport argparse\nimport os\nfrom pathlib import Path\n\nDIR = Path(__file__).absolute().parent\n\n# SNAPCRAFT_PROJECT_DIR is set when building the snap. If unset, resolve based on current file path\n_SNAPCRAFT_PROJECT_DIR = os.getenv(\"SNAPCRAFT_PROJECT_DIR\") or \"\"\nSNAPCRAFT_PROJECT_DIR = _SNAPCRAFT_PROJECT_DIR and Path(_SNAPCRAFT_PROJECT_DIR) or Path(DIR / \"..\")\n\nSTRICT = \"confinement: strict\" in (SNAPCRAFT_PROJECT_DIR / \"snap\" / \"snapcraft.yaml\").read_text()\n\n\nclass Version:\n def __init__(self, version_string: str):\n self.str = version_string\n if version_string[0] == \"v\":\n version_string = version_string[1:]\n if \"-\" in version_string:\n version_string = version_string[: version_string.rfind(\"-\")]\n\n try:\n self.version = [int(x) for x in version_string.split(\".\")]\n self.type = \"semver\"\n except (TypeError, ValueError):\n self.version = self.str\n self.type = \"string\"\n\n def equal_or_older_than(self, v: \"Version\") -> bool:\n \"\"\"Consider the following cases:\n - `v1.1.0` is equal or older than `v1.2.0`.\n - `v1.2.0` is equal or older than `v1.2.0`.\n - `v1.28.0-rc.0` is equal or older than v1.28.0`.\n - `fix/mybug` is not equal or older than `v1.28.0`.\n \"\"\"\n if self.type == v.type == \"semver\" and self.version <= v.version:\n return True\n if {self.type, v.type} & {\"string\"} and v.str.startswith(self.str):\n return True\n\n return False\n\n\ndef find_suitable_patch_version(candidates: list, target_version: Version) -> str:\n \"\"\"pick the version string from a list of candidate versions\"\"\"\n result = None\n has_default = False\n for candidate in candidates:\n if candidate.str == target_version.str:\n return target_version.str\n\n has_default = has_default or candidate.str == \"default\"\n\n if not candidate.equal_or_older_than(target_version):\n continue\n\n if result is None or result.equal_or_older_than(candidate):\n result = candidate\n\n # found a suitable patch directory\n if result:\n return result.str\n\n # no suitable patch directory found, but there is a default\n if has_default:\n return \"default\"\n\n # component does not have any patches\n return None\n\n\ndef get_patches_for(component: str, version_string: str, strict: bool) -> list:\n \"\"\"Return a list of patches that must be applied when building 'component'\n with target 'version'.\n \"\"\"\n component_version = Version(version_string)\n component_dir = SNAPCRAFT_PROJECT_DIR / \"build-scripts\" / \"components\" / component\n patches = []\n\n patch_directories = [\"patches\"]\n if strict:\n patch_directories += [\"strict-patches\"]\n\n for patch_dir_name in patch_directories:\n patches_dir = component_dir / patch_dir_name\n if not patches_dir.is_dir():\n continue\n\n candidates = [\n Version(path.name)\n for path in sorted(\n patches_dir.iterdir(), reverse=True\n ) # sort by reverse to handle exact matches\n if path.is_dir()\n ]\n patch_version = find_suitable_patch_version(candidates, component_version)\n if patch_version is not None:\n patches.extend(\n [p.resolve().as_posix() for p in (patches_dir / patch_version).iterdir()]\n )\n\n return patches\n\n\ndef main():\n parser = argparse.ArgumentParser()\n\n parser.add_argument(\"component\", type=str)\n parser.add_argument(\"version\", type=str)\n\n args = parser.parse_args()\n\n print(\"\\n\".join(get_patches_for(args.component, args.version, STRICT)))\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "build-scripts/update-images.sh", "content": "#!/bin/bash -x\n\n## Description:\n#\n# Install MicroK8s from a specific channel and update the list of images required by MicroK8s\n# and the core addons.\n#\n## Example:\n#\n# $ ./build-scripts/update-images.sh latest/edge build-scripts/images.txt\n\nsudo snap install microk8s --classic --channel $1\nsudo microk8s status --wait-ready\n\nsudo microk8s enable storage ingress metrics-server dns\n\nsudo microk8s kubectl apply -f - < $2\n" }, { "path": "docs/build.md", "content": "# Building and testing MicroK8s\n\n## Building the snap from source\n\nTo build the MicroK8s snap, you need to install Snapcraft.\n\n```shell\nsudo snap install snapcraft --classic\n```\n\n[Building a snap](https://snapcraft.io/docs/snapcraft-overview) is done by running `snapcraft` in the root of the project. Snapcraft spawn a VM managed by [Multipass](https://multipass.run/) and builds MicroK8s inside of it. If you don’t have Multipass installed, snapcraft will first prompt for its automatic installation.\n\nAfter `snapcraft` finishes, you can install the newly compiled snap:\n\n```shell\nsudo snap install microk8s_*_amd64.snap --classic --dangerous\n```\n\n## Building the snap using LXD\n\nAlternatively, you can build the snap in an LXC container. This is useful if you are working in a virtual machine without nested virtualization. Snapcraft and LXD are needed in this case:\n\n```shell\nsudo snap install lxd\nsudo apt-get remove lxd* -y\nsudo apt-get remove lxc* -y\nsudo lxd init\nsudo usermod -a -G lxd ${USER}\n```\n\nBuild the snap with:\n\n```shell\ngit clone http://github.com/canonical/microk8s\ncd ./microk8s/\nsnapcraft --use-lxd\n```\n\nFinally, install it with:\n\n```shell\nsudo snap install microk8s_*_amd64.snap --classic --dangerous\n```\n\n## Building a custom MicroK8s package\n\nTo produce a custom build with specific component versions you cannot use the snapcraft build process on the host OS. You need to\n[prepare an LXC](https://forum.snapcraft.io/t/how-to-create-a-lxd-container-for-snap-development/4658) container with Ubuntu 16:04 and snapcraft:\n\n```shell\nlxc launch ubuntu:16.04 --ephemeral test-build\nlxc exec test-build -- snap install snapcraft --classic\nlxc exec test-build -- apt update\nlxc exec test-build -- git clone https://github.com/canonical/microk8s\n```\n\nYou can then set the following environment variables prior to building:\n\n- KUBE_VERSION: Kubernetes release to package. Defaults to latest stable.\n- ETCD_VERSION: version of etcd.\n- CNI_VERSION: version of CNI tools.\n- KUBE_TRACK: Kubernetes release series (e.g., 1.10) to package. Defaults to latest stable.\n- ISTIO_VERSION: istio release.\n- KNATIVE_SERVING_VERSION: Knative Serving release.\n- KNATIVE_EVENTING_VERSION: Knative Eventing release.\n- RUNC_COMMIT: the commit hash from which to build runc.\n- CONTAINERD_COMMIT: the commit hash from which to build containerd\n- KUBERNETES_REPOSITORY: build the Kubernetes binaries from this repository instead of getting them from upstream\n- KUBERNETES_COMMIT: commit to be used from KUBERNETES_REPOSITORY for building the Kubernetes binaries\n\nFor building you prepend the variables you need as well as `SNAPCRAFT_BUILD_ENVIRONMENT=host` so the current LXC container is used. For example to build the MicroK8s snap for Kubernetes v1.9.6, run:\n\n```shell\nlxc exec test-build -- sh -c \"cd microk8s && SNAPCRAFT_BUILD_ENVIRONMENT=host KUBE_VERSION=v1.9.6 snapcraft\"\n```\n\nThe produced snap is inside the ephemeral LXC container, you need to copy it to the host:\n\n```shell\nlxc file pull test-build/root/microk8s/microk8s_v1.9.6_amd64.snap .\n```\n\nAfter copying it, you can install it with:\n\n```shell\nsnap install microk8s_*_amd64.snap --classic --dangerous\n```\n\n## Assembling the Calico CNI manifest\n\nThe calico CNI manifest can be found under `upgrade-scripts/000-switch-to-calico/resources/calico.yaml`.\nBuilding the manifest is subject to the upstream calico project k8s installation process.\nThe `calico.yaml` manifest is a slightly modified version of:\n`https://projectcalico.docs.tigera.io/archive/v3.25/manifests/calico.yaml`:\n\n- Set the calico backend from \"bird\" to \"vxlan\": `calico_backend: \"vxlan\"`\n- `CALICO_IPV4POOL_CIDR` was set to \"10.1.0.0/16\"\n- `CNI_NET_DIR` had to be added and set to \"/var/snap/microk8s/current/args/cni-network\"\n- We set the following mount paths:\n 1. `var-run-calico` to `/var/snap/microk8s/current/var/run/calico`\n 1. `var-lib-calico` to `/var/snap/microk8s/current/var/lib/calico`\n 1. `cni-bin-dir` to `/var/snap/microk8s/current/opt/cni/bin`\n 1. `cni-net-dir` to `/var/snap/microk8s/current/args/cni-network`\n 1. `cni-log-dir` to `/var/snap/microk8s/common/var/log/calico/cni`\n 1. `host-local-net-dir` to `/var/snap/microk8s/current/var/lib/cni/networks`\n 1. `policysync` to `/var/snap/microk8s/current/var/run/nodeagent`\n- We enabled vxlan following the instructions in [the official docs.](https://docs.projectcalico.org/getting-started/kubernetes/installation/config-options#switching-from-ip-in-ip-to-vxlan)\n- The `liveness` and `readiness` probes of `bird-live` was commented out\n- We set the IP autodetection method to\n\n ```dtd\n - name: IP_AUTODETECTION_METHOD\n value: \"first-found\"\n ```\n- In the `cni_network_config` in the calico manifest we also set:\n ```dtd\n \"log_file_path\": \"/var/snap/microk8s/common/var/log/calico/cni/cni.log\",\n \"nodename_file\": \"/var/snap/microk8s/current/var/lib/calico/nodename\",\n ```\n- The `mount-bpffs` pod is commented out. This disables eBPF support but allows the CNI to deploy inside an LXC container.\n- The bpffs mount is commented out:\n\n ```dtd\n - name: bpffs\n mountPath: /sys/fs/bpf\n ```\n end\n\n ```dtd\n - name: bpffs\n hostPath:\n path: /sys/fs/bpf\n type: Directory\n ```\n\n## Running the tests locally\n\nTo successfully run the tests you need to install:\n\n1. From your distribution's repository:\n - python3\n - pytest\n - pip3\n - docker.io\n - tox\n1. From pip3:\n - requests\n - pyyaml\n - sh\n\nOn Ubuntu 20.04, for example, run the following commands to get these dependencies.\n\n```shell\nsudo apt install python3-pip docker.io tox -y\npip3 install -U pytest requests pyyaml sh\n```\n\nFirst, run the static analysis using Tox. Run the following command in the root of the project.\n\n```shell\ntox -e lint\n```\n\nThen, use the \"Building the snap from source\" instructions to build the snap and install it locally. The tests check this locally installed MicroK8s instance.\n\nFinally, run the tests themselves. The `test-simple.py` and `test-upgrade.py` files under the `tests` directory are the two main files of our test suite. Running the tests is done with pytest:\n\n```shell\npytest -s tests/test-simple.py\npytest -s tests/test-upgrade.py\n```\n\nNote: the `ingress` and `dashboard-ingress` tests make use of nip.io for wildcard ingress domains on localhost. [DNS rebinding protection](https://en.wikipedia.org/wiki/DNS_rebinding) may prevent the resolution of the domains used in the tests.\n\nA workaround is adding these entries to `/etc/hosts`:\n```\n127.0.0.1 kubernetes-dashboard.127.0.0.1.nip.io\n127.0.0.1 microbot.127.0.0.1.nip.io\n```\n" }, { "path": "docs/community.md", "content": "# MicroK8s in the Wild\n\nPeople are doing amazing things with MicroK8s. Feel free to submit a pull\nrequest with updates to this page.\n\n| Event/Mention | Author(s) | Source | Date |\n|---------------------------------------------------------------------|--------------|----------|---------------|\n| [Kubernetes Automation Toolkit](https://github.com/BigBitBusInc/kubernetes-automation-toolkit) | Sachin Agarwal (BigBitBus) | github.com | 6-Aug-2021 |\n| [IoT/edge: Akri (MSFT) on MicroK8s](https://github.com/didier-durand/microk8s-akri) | Didier Durand | github.com | 27-Nov-2020 |\n| [Kata Containers on MicroK8s](https://github.com/didier-durand/microk8s-kata-containers) | Didier Durand | github.com | 13-Nov-2020 |\n| [MicroK8s analyzed with kube-bench](https://github.com/didier-durand/microk8s-kube-bench) | Didier Durand | github.com | 23-Oct-2020 |\n| [MicroK8s on Raspberry Pi](https://github.com/trulede/raspberry.microk8s) | Tim Rule | github.com | 27-April-2020 |\n| [Getting started with Kubernetes MicroK8s & Linode](https://www.youtube.com/watch?v=RiV2TNcjAFw) | Egee | Youtube.com | 6-Oct-2019 |\n| [How do I set up Tilt to use MicroK8s?](https://docs.tilt.dev/faq.html#q-how-do-i-set-up-tilt-to-use-microk8s) | The tilt.dev team | docs.tilt.dev | 8-Aug-2019 |\n| [Microk8s + Flask (Part 2): Exposing Flask app in a Microk8s service](https://medium.com/@giorgos.apost.1994/microk8s-flask-part-2-280ac2d9eff) | George Apostolopoulos | medium.com | 19-Aug-2019 |\n| [Microk8s + Flask (Part 1): Dockerize a Flask Application](https://medium.com/@giorgos.apost.1994/microk8s-flask-part-1-efa3cc4e0e00) | George Apostolopoulos | medium.com | 19-Jul-2019 |\n| [Getting Started with Cilium on Microk8s](https://docs.cilium.io/en/stable/gettingstarted/microk8s/)| Joe Stringer | docs.cilium.io | 23-Apr-2019 |\n| [Single-User Production on Microk8s](https://transcrob.es/post/single-user-prod-microk8s/)| Anton Melser | transcrob.es | 05-Feb-2019 |\n| [MicroK8s, Part 3: How To Deploy a Pod in Kubernetes](https://virtualizationreview.com/articles/2019/02/01/microk8s-part-3-how-to-deploy-a-pod-in-kubernetes.aspx)| Tom Fenton | virtualizationreview.com | 02-Feb-2019 |\n| [MicroK8s, Part 2: How To Monitor and Manage Kubernetes](https://virtualizationreview.com/articles/2019/01/30/microk8s-part-2-how-to-monitor-and-manage-kubernetes.aspx)| Tom Fenton | virtualizationreview.com | 30-Jan-2019 |\n| [MicroK8s: How To Install and Use Kubernetes](https://virtualizationreview.com/articles/2019/01/28/microk8s-how-to-install-and-use-kubernetes.aspx)| Tom Fenton | virtualizationreview.com | 28-Jan-2019 |\n| [SUSE Cloud Application Platform (CAP) on Microk8s](https://dimitris.karakasilis.me/2019/01/27/scf-on-microk8s.html)| Dimitris Karakasilis | dimitris.karakasilis.me | 27-Jan-2019 |\n| [Files upload from Kubeless on MicroK8s to Minio](https://itnext.io/files-upload-from-kubeless-on-microk8s-to-minio-607e06598a4b) | Rafał bluszcz Zawadzki | itnext.io | 07-Jan-2019 |\n| [Get into Kubernetes with MicroK8s](https://medium.com/devopslinks/get-into-kubernetes-with-microk8s-85f0e0231c4a) | Milindu S. Kumarage | medium.com | 06-Jan-2019 |\n| [Serverless MicroK8s Kubernetes](https://medium.com/@bluszcz/serverless-microk8s-kubernetes-fcd6b875cd33) | Rafał bluszcz Zawadzki | medium.com | 06-Jan-2019 |\n| [Deploy OpenFaaS with MicroK8s](https://johnmccabe.net/technology/projects/openfaas-on-microk8s/) | John McCabe | johnmccabe.net | 01-Jan-2019 |\n| [Docker desktop, microk8s and the battle for the k8s laptop](https://blog.linnovate.net/docker-desktop-microk8s-and-the-battle-for-the-k8s-laptop-8d3fb50dd543) | liorkesos | blog.linnovate.net | 18-Dec-2018 |\n| [Canonical Launches MicroK8s](http://www.linux-magazine.com/Online/News/Canonical-Launches-MicroK8s) | Swapnil Bhartiya | linux-magazine.com | 12-Dec-2018 |\n| [Kubernetes single node cluster using microk8s](https://www.youtube.com/watch?v=FEELm-o__gU) | Just me and Opensource | youtube.com | 02-Dec-2018 |\n| [Getting started with Kubeflow (machine learning toolkit for Kubernetes) via Microk8s and Multipass](https://github.com/leigh-johnson/kubeflow-microk8s-demo) | Leigh Johnson | github.com | 20-Nov-2018 |\n| [Kubernetes CICD Hacks with MicroK8s and Kubeflow](https://www.youtube.com/watch?v=1SSvS2w5OMQ), KubeCon China 2018 | Land Lu, Zhang Lei Mao from Canonical | KubeCon 2018 | 14-Nov-2018 |\n| [Kubernetes On A Laptop](https://kubedex.com/local-kubernetes/) | Steven Acreman | kubedex.com | 03-Nov-2018 |\n| [Local Kubernetes for Linux – MiniKube vs MicroK8s](https://codefresh.io/kubernetes-tutorial/local-kubernetes-linux-minikube-vs-microk8s/) | Thomas Pliakas | codefresh.io | 01-Nov-2018 |\n| [A local Kubernetes with microk8s](https://itnext.io/a-local-kubernetes-with-microk8s-33ee31d1eed9) | Marco Ceppi | itnext.io | 22-May-2018 |\n" }, { "path": "docs/k8s-patches.md", "content": "# Patches for Kubernetes\n \nThe patch to add kubelite is applied on top of the Kubernetes source code during\nthe MicroK8s build process.\n\nThe Kubernetes patches are under `build-scripts/patches/`.\n\nThe patches are maintained in the [kubernetes-dqlite repository](https://github.com/canonical/kubernetes-dqlite).\nTo produce them checkout the kubernetes-dqlite repository and head to the Kubernetes\nbranch you would like to build. For example:\n```\ngit clone https://github.com/canonical/kubernetes-dqlite.git\ncd kubernetes-dqlite/\ngit checkout release-1.18\n``` \nFrom `git log` note down the commits related to k8s patches. These commits are at the top.\nForm the patch with for example:\n```\ngit format-patch c3c94660a58998dde628de7c716a63b695327016^..f0cc11e7a398d9454e8c4d3a526d6372cf1a1889 --stdout > k8s.patch\n```\n\nRemember to copy the produced patch file back to the `microk8s` source tree.\n" }, { "path": "installer/.gitignore", "content": ".venv/\nbin/\nlib/\nlib64\npyvenv.cfg\nshare/\n" }, { "path": "installer/__init__.py", "content": "" }, { "path": "installer/build-linux.sh", "content": "#!/bin/bash\n\n\nvirtualenv -p python3 .venv\nsource ./.venv/bin/activate\npip install -r requirements.txt\npyinstaller ./microk8s.spec\ndeactivate\nrm -rf .venv\n" }, { "path": "installer/cli/__init__.py", "content": "" }, { "path": "installer/cli/echo.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2017 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\"\"\"Facilities to output to the terminal.\n\nThese methods, which are named after common logging levels, wrap around\nclick.echo adding the corresponding color codes for each level.\n\"\"\"\nimport click\nimport sys\n\nfrom typing import Any\n\nfrom common.definitions import MAX_CHARACTERS_WRAP\n\n\nclass Echo:\n @staticmethod\n def is_tty_connected() -> bool:\n \"\"\"Check to see if running under TTY.\"\"\"\n return sys.stdin.isatty()\n\n @staticmethod\n def wrapped(msg: str) -> None:\n \"\"\"Output msg wrapped to the terminal width to stdout.\n\n The maximum wrapping is determined by\n MAX_CHARACTERS_WRAP\n \"\"\"\n click.echo(\n click.formatting.wrap_text(msg, width=MAX_CHARACTERS_WRAP, preserve_paragraphs=True)\n )\n\n @staticmethod\n def info(msg: str) -> None:\n \"\"\"Output msg as informative to stdout.\n If the terminal supports colors the output will be green.\n \"\"\"\n click.echo(\"\\033[0;32m{}\\033[0m\".format(msg))\n\n @staticmethod\n def warning(msg: str) -> None:\n \"\"\"Output msg as a warning to stdout.\n If the terminal supports color the output will be yellow.\n \"\"\"\n click.echo(\"\\033[1;33m{}\\033[0m\".format(msg))\n\n @staticmethod\n def error(msg: str) -> None:\n \"\"\"Output msg as an error to stdout.\n If the terminal supports color the output will be red.\n \"\"\"\n click.echo(\"\\033[0;31m{}\\033[0m\".format(msg))\n\n def confirm(\n self,\n msg: str,\n default: bool = False,\n abort: bool = False,\n prompt_suffix: str = \": \",\n show_default: bool = True,\n err: bool = False,\n ) -> bool:\n \"\"\"Output message as a confirmation prompt.\n If not running on a tty, assume the default value.\n \"\"\"\n return (\n click.confirm(\n msg,\n default=default,\n abort=abort,\n prompt_suffix=prompt_suffix,\n show_default=show_default,\n err=err,\n )\n if self.is_tty_connected()\n else default\n )\n\n def prompt(\n self,\n msg: str,\n default: Any = None,\n hide_input: bool = False,\n confirmation_prompt: bool = False,\n type=None,\n value_proc=None,\n prompt_suffix: str = \": \",\n show_default: bool = True,\n err: bool = False,\n ) -> Any:\n \"\"\"Output message as a generic prompt.\n If not running on a tty, assume the default value.\n \"\"\"\n return (\n click.prompt(\n msg,\n default=default,\n hide_input=hide_input,\n confirmation_prompt=confirmation_prompt,\n type=type,\n value_proc=value_proc,\n prompt_suffix=prompt_suffix,\n show_default=show_default,\n err=err,\n )\n if self.is_tty_connected()\n else default\n )\n" }, { "path": "installer/cli/microk8s.py", "content": "import argparse\nimport logging\nimport traceback\nfrom typing import List\nfrom sys import exit, platform\nfrom os import getcwd\n\nimport click\n\nfrom cli.echo import Echo\nfrom common import definitions\nfrom common.auxiliary import Windows, MacOS, Linux\nfrom common.errors import BaseError\nfrom common.file_utils import get_kubeconfig_path, clear_kubeconfig\nfrom vm_providers.factory import get_provider_for\nfrom vm_providers.errors import ProviderNotFound, ProviderInstanceNotFoundError\n\nlogger = logging.getLogger(__name__)\n\n\n@click.command(\n name=\"microk8s\",\n context_settings=dict(\n ignore_unknown_options=True,\n allow_extra_args=True,\n ),\n)\n@click.option(\"-h\", \"--help\", is_flag=True)\n@click.pass_context\ndef cli(ctx, help):\n try:\n if help and len(ctx.args) == 0:\n show_help()\n exit(0)\n elif help:\n ctx.args.append(\"--help\")\n\n if len(ctx.args) == 0:\n show_error()\n exit(1)\n if ctx.args[0] == \"install\":\n install(ctx.args[1:])\n exit(0)\n elif ctx.args[0] == \"uninstall\":\n uninstall()\n exit(0)\n elif ctx.args[0] == \"start\":\n start()\n run(ctx.args)\n exit(0)\n elif ctx.args[0] == \"stop\":\n run(ctx.args)\n stop()\n exit(0)\n elif ctx.args[0] == \"kubectl\":\n exit(kubectl(ctx.args[1:]))\n elif ctx.args[0] == \"dashboard-proxy\":\n dashboard_proxy()\n exit(0)\n elif ctx.args[0] == \"inspect\":\n inspect()\n exit(0)\n else:\n run(ctx.args)\n exit(0)\n\n except BaseError as e:\n Echo.error(str(e))\n exit(e.get_exit_code())\n except Exception as e:\n Echo.error(\"An unexpected error occurred.\")\n Echo.info(str(e))\n Echo.info(traceback.print_exc())\n exit(254)\n\n\ndef show_error():\n msg = \"\"\"Usage: microk8s [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n --help Shows the available COMMANDS.\"\"\"\n click.echo(msg)\n\n\ndef show_help():\n msg = \"\"\"Usage: microk8s [OPTIONS] COMMAND [ARGS]...\n\nOptions:\n --help Show this message and exit.\n\nCommands:\n install Installs MicroK8s. Use --cpu, --mem, --disk, --channel, and --image to configure your setup.\n uninstall Removes MicroK8s\"\"\"\n click.echo(msg)\n commands = _get_microk8s_commands()\n for command in commands:\n if command in definitions.command_descriptions:\n click.echo(\" {:<15} {}\".format(command, definitions.command_descriptions[command]))\n else:\n click.echo(\" {:<15}\".format(command))\n if len(commands) == 2:\n click.echo(\"\")\n click.echo(\"Install and start MicroK8s to see the full list of commands.\")\n\n\ndef _show_install_help():\n msg = f\"\"\"Usage: microk8s install OPTIONS\n\n Options:\n --help Show this message and exit.\n --cpu Cores used by MicroK8s (default={definitions.DEFAULT_CORES}, min={definitions.MIN_CORES})\n --mem RAM in GB used by MicroK8s (default={definitions.DEFAULT_MEMORY_GB}, min={definitions.MIN_MEMORY_GB})\n --disk Max volume in GB of the dynamically expandable hard disk to be used (default={definitions.DEFAULT_DISK_GB}, min={definitions.MIN_DISK_GB})\n --channel Kubernetes version to install (default={definitions.DEFAULT_CHANNEL})\n --image Ubuntu version to install (default={definitions.DEFAULT_IMAGE})\n -y, --assume-yes Automatic yes to prompts\"\"\" # noqa\n Echo.info(msg)\n\n\ndef memory(mem_gb: str) -> int:\n \"\"\"\n Validates the value in --mem parameter of the install command.\n \"\"\"\n mem_gb = int(mem_gb)\n if mem_gb < definitions.MIN_MEMORY_GB:\n raise ValueError(\"Out of valid memory range\")\n return mem_gb\n\n\ndef cpu(cpus: str) -> int:\n \"\"\"\n Validates the value in --cpu parameter of the install command.\n \"\"\"\n cpus = int(cpus)\n if cpus < definitions.MIN_CORES:\n raise ValueError(\"Invalid number of cpus\")\n return cpus\n\n\ndef disk(disk_gb: str) -> int:\n \"\"\"\n Validates the value in --disk parameter of the install command.\n \"\"\"\n disk_gb = int(disk_gb)\n if disk_gb < definitions.MIN_DISK_GB:\n raise ValueError(\"Out of valid disk range\")\n return disk_gb\n\n\ndef install(args) -> None:\n if \"--help\" in args or \"-h\" in args:\n _show_install_help()\n return\n\n parser = argparse.ArgumentParser(\"microk8s install\")\n parser.add_argument(\"--cpu\", default=definitions.DEFAULT_CORES, type=cpu)\n parser.add_argument(\"--mem\", default=definitions.DEFAULT_MEMORY_GB, type=memory)\n parser.add_argument(\"--disk\", default=definitions.DEFAULT_DISK_GB, type=disk)\n parser.add_argument(\"--channel\", default=definitions.DEFAULT_CHANNEL, type=str)\n parser.add_argument(\"--image\", default=definitions.DEFAULT_IMAGE, type=str)\n parser.add_argument(\n \"-y\", \"--assume-yes\", action=\"store_true\", default=definitions.DEFAULT_ASSUME\n )\n args = parser.parse_args(args)\n\n echo = Echo()\n\n if platform == \"win32\":\n host = Windows(args)\n elif platform == \"darwin\":\n host = MacOS(args)\n else:\n host = Linux(args)\n\n if not host.has_enough_cpus():\n echo.error(\"VM cpus requested exceed number of available cores on host.\")\n exit(1)\n if not host.has_enough_memory():\n echo.warning(\"VM memory requested exceeds the total memory on host.\")\n exit(1)\n if not host.has_enough_disk_space():\n echo.warning(\"VM disk size requested exceeds free space on host.\")\n\n vm_provider_name: str = \"multipass\"\n vm_provider_class = get_provider_for(vm_provider_name)\n try:\n vm_provider_class.ensure_provider()\n except ProviderNotFound as provider_error:\n if provider_error.prompt_installable:\n if args.assume_yes or (\n echo.is_tty_connected()\n and echo.confirm(\n \"Support for {!r} needs to be set up. \"\n \"Would you like to do that now?\".format(provider_error.provider)\n )\n ):\n vm_provider_class.setup_provider(echoer=echo)\n else:\n raise provider_error\n else:\n raise provider_error\n\n instance = vm_provider_class(echoer=echo)\n spec = vars(args)\n spec.update({\"kubeconfig\": get_kubeconfig_path()})\n instance.launch_instance(spec)\n echo.info(\"MicroK8s is up and running. See the available commands with `microk8s --help`.\")\n\n\ndef uninstall() -> None:\n vm_provider_name = \"multipass\"\n vm_provider_class = get_provider_for(vm_provider_name)\n echo = Echo()\n try:\n vm_provider_class.ensure_provider()\n except ProviderNotFound as provider_error:\n if provider_error.prompt_installable:\n if echo.is_tty_connected():\n echo.warning(\n (\n \"MicroK8s is not running. VM provider {!r} has been removed.\".format(\n provider_error.provider\n )\n )\n )\n return 1\n else:\n raise provider_error\n\n instance = vm_provider_class(echoer=echo)\n instance.destroy()\n clear_kubeconfig()\n echo.info(\"Thank you for using MicroK8s!\")\n\n\ndef kubectl(args) -> int:\n if platform == \"win32\":\n return Windows(args).kubectl()\n if platform == \"darwin\":\n return MacOS(args).kubectl()\n else:\n return Linux(args).kubectl()\n\n\ndef inspect() -> None:\n vm_provider_name = \"multipass\"\n vm_provider_class = get_provider_for(vm_provider_name)\n echo = Echo()\n try:\n vm_provider_class.ensure_provider()\n instance = vm_provider_class(echoer=echo)\n instance.get_instance_info()\n\n command = [\"microk8s.inspect\"]\n output = instance.run(command, hide_output=True)\n tarball_location = None\n host_destination = getcwd()\n if b\"Report tarball is at\" not in output:\n echo.error(\"Report tarball not generated\")\n else:\n for line_out in output.split(b\"\\n\"):\n line_out = line_out.decode()\n line = line_out.strip()\n if line.startswith(\"Report tarball is at \"):\n tarball_location = line.split(\"Report tarball is at \")[1]\n break\n echo.wrapped(line_out)\n if not tarball_location:\n echo.error(\"Cannot find tarball file location\")\n else:\n instance.pull_file(name=tarball_location, destination=host_destination)\n echo.wrapped(\n \"The report tarball {} is stored on the current directory\".format(\n tarball_location.split(\"/\")[-1]\n )\n )\n\n except ProviderInstanceNotFoundError:\n _not_installed(echo)\n return 1\n\n\ndef dashboard_proxy() -> None:\n vm_provider_name = \"multipass\"\n vm_provider_class = get_provider_for(vm_provider_name)\n echo = Echo()\n try:\n vm_provider_class.ensure_provider()\n instance = vm_provider_class(echoer=echo)\n instance.get_instance_info()\n\n echo.info(\"Checking if Dashboard is running.\")\n command = [\"microk8s.enable\", \"dashboard\"]\n output = instance.run(command, hide_output=True)\n if b\"Addon dashboard is already enabled.\" not in output:\n echo.info(\"Waiting for Dashboard to come up.\")\n command = [\n \"microk8s.kubectl\",\n \"-n\",\n \"kube-system\",\n \"wait\",\n \"--timeout=240s\",\n \"deployment\",\n \"kubernetes-dashboard\",\n \"--for\",\n \"condition=available\",\n ]\n instance.run(command, hide_output=True)\n\n command = [\"microk8s.kubectl\", \"-n\", \"kube-system\", \"create\", \"token\", \"default\"]\n token = instance.run(command, hide_output=True).decode().strip()\n\n if not token:\n echo.error(\"Could not generate secret token to access dashboard.\")\n exit(1)\n\n ip = instance.get_instance_info().ipv4[0]\n\n echo.info(\"Dashboard will be available at https://{}:10443\".format(ip))\n echo.info(\"Use the following token to login:\")\n echo.info(token)\n\n command = [\n \"microk8s.kubectl\",\n \"port-forward\",\n \"-n\",\n \"kube-system\",\n \"service/kubernetes-dashboard\",\n \"10443:443\",\n \"--address\",\n \"0.0.0.0\",\n ]\n\n try:\n instance.run(command)\n except KeyboardInterrupt:\n return\n except ProviderInstanceNotFoundError:\n _not_installed(echo)\n return 1\n\n\ndef start() -> None:\n vm_provider_name = \"multipass\"\n vm_provider_class = get_provider_for(vm_provider_name)\n vm_provider_class.ensure_provider()\n instance = vm_provider_class(echoer=Echo())\n instance_info = instance.get_instance_info()\n if not instance_info.is_running():\n instance.start()\n instance.run([\"microk8s.start\"])\n\n\ndef stop() -> None:\n vm_provider_name = \"multipass\"\n vm_provider_class = get_provider_for(vm_provider_name)\n vm_provider_class.ensure_provider()\n instance = vm_provider_class(echoer=Echo())\n instance_info = instance.get_instance_info()\n if instance_info.is_running():\n instance.stop()\n\n\ndef run(cmd) -> None:\n vm_provider_name = \"multipass\"\n vm_provider_class = get_provider_for(vm_provider_name)\n echo = Echo()\n try:\n vm_provider_class.ensure_provider()\n instance = vm_provider_class(echoer=echo)\n instance_info = instance.get_instance_info()\n if not instance_info.is_running():\n echo.warning(\"MicroK8s is not running. Please run `microk8s start`.\")\n return 1\n command = cmd[0]\n cmd[0] = \"microk8s.{}\".format(command)\n instance.run(cmd)\n except ProviderInstanceNotFoundError:\n _not_installed(echo)\n return 1\n\n\ndef _not_installed(echo) -> None:\n if echo.is_tty_connected():\n echo.warning(\"MicroK8s is not installed. Please run `microk8s install`.\")\n\n\ndef _get_microk8s_commands() -> List:\n vm_provider_name = \"multipass\"\n vm_provider_class = get_provider_for(vm_provider_name)\n echo = Echo()\n try:\n vm_provider_class.ensure_provider()\n instance = vm_provider_class(echoer=echo)\n instance_info = instance.get_instance_info()\n if instance_info.is_running():\n commands = instance.run(\"ls -1 /snap/bin/\".split(), hide_output=True)\n mk8s = [\n c.decode().replace(\"microk8s.\", \"\")\n for c in commands.split()\n if c.decode().startswith(\"microk8s.\")\n ]\n complete = mk8s\n if \"dashboard-proxy\" not in mk8s:\n complete += [\"dashboard-proxy\"]\n complete.sort()\n return complete\n else:\n return [\"start\", \"stop\"]\n except ProviderNotFound:\n return [\"start\", \"stop\"]\n\n\nif __name__ == \"__main__\":\n cli()\n" }, { "path": "installer/common/__init__.py", "content": "" }, { "path": "installer/common/auxiliary.py", "content": "import ctypes\nimport logging\nimport os\nimport psutil\nimport subprocess\n\nfrom abc import ABC\nfrom os.path import realpath\nfrom shutil import disk_usage\n\nfrom .file_utils import get_kubeconfig_path, get_kubectl_directory\nfrom . import definitions\n\nlogger = logging.getLogger(__name__)\n\n\nclass Auxiliary(ABC):\n \"\"\"\n Base OS auxiliary class.\n \"\"\"\n\n def __init__(self, args) -> None:\n \"\"\"\n :param args: ArgumentParser\n :return: None\n \"\"\"\n self._args = args\n try:\n self.requested_disk = self._args.disk * 1024 * 1024 * 1024\n self.requested_memory = self._args.mem * 1024 * 1024 * 1024\n self.requested_cores = self._args.cpu\n except AttributeError:\n self.requested_disk = definitions.DEFAULT_DISK_GB\n self.requested_memory = definitions.DEFAULT_MEMORY_GB\n self.requested_cores = definitions.DEFAULT_CORES\n\n @staticmethod\n def _free_disk_space() -> int:\n \"\"\"\n Get space free of disk that this script is installed to.\n\n :return: Integer free space\n \"\"\"\n return disk_usage(realpath(\"/\")).free\n\n @staticmethod\n def _total_memory() -> int:\n \"\"\"\n Get available memory in machine this script is installed to.\n\n :return: Available memory in bytes\n \"\"\"\n return psutil.virtual_memory().total\n\n @staticmethod\n def _cpu_count() -> int:\n \"\"\"\n Get the number of cpus on the machine this script is installed to.\n\n :return: Number of cpus\n \"\"\"\n return psutil.cpu_count(logical=False)\n\n def has_enough_disk_space(self) -> bool:\n \"\"\"\n Compare free space with minimum.\n\n :return: Boolean\n \"\"\"\n return self._free_disk_space() > self.requested_disk\n\n def has_enough_memory(self) -> bool:\n \"\"\"\n Compare requested memory against available\n\n :return: Boolean\n \"\"\"\n return self._total_memory() > self.requested_memory\n\n def has_enough_cpus(self) -> bool:\n \"\"\"\n Compare requested cpus against available cores.\n\n :return: Boolean\n \"\"\"\n return self._cpu_count() >= self.requested_cores\n\n def get_kubectl_directory(self) -> str:\n \"\"\"\n Get the correct directory to install kubectl into,\n we can then call this when running `microk8s kubectl`\n without interfering with any systemwide install.\n\n :return: String\n \"\"\"\n return get_kubectl_directory()\n\n def get_kubeconfig_path(self) -> str:\n \"\"\"\n Get the correct path to write the kubeconfig\n file to. This is then read by the installed\n kubectl and won't interfere with one in the user's\n home.\n\n :return: String\n \"\"\"\n return get_kubeconfig_path()\n\n def kubectl(self) -> int:\n \"\"\"\n Run kubectl on the host, with the generated kubeconf.\n\n :return: None\n \"\"\"\n kctl_dir = self.get_kubectl_directory()\n try:\n exit_code = subprocess.check_call(\n [\n os.path.join(kctl_dir, \"kubectl\"),\n \"--kubeconfig={}\".format(self.get_kubeconfig_path()),\n ]\n + self._args,\n )\n except subprocess.CalledProcessError as e:\n return e.returncode\n else:\n return exit_code\n\n\nclass Windows(Auxiliary):\n \"\"\"\n Windows auxiliary methods.\n \"\"\"\n\n def __init__(self, args) -> None:\n \"\"\"\n :param args: ArgumentParser\n :return: None\n \"\"\"\n super(Windows, self).__init__(args)\n\n @staticmethod\n def check_admin() -> bool:\n \"\"\"\n Check if running as admin.\n\n :return: Boolean\n \"\"\"\n return ctypes.windll.shell32.IsUserAnAdmin() == 1\n\n @staticmethod\n def check_hyperv() -> bool:\n \"\"\"\n Check if Hyper V is already enabled.\n\n :return: Boolean\n \"\"\"\n try:\n out = subprocess.check_output(\n [\"DISM\", \"/Online\", \"/Get-FeatureInfo\", \"/FeatureName:Microsoft-Hyper-V\"]\n )\n except subprocess.CalledProcessError:\n return False\n\n if \"State : Disabled\" in out.decode():\n return False\n\n return True\n\n @staticmethod\n def enable_hyperv() -> None:\n \"\"\"\n Enable Hyper V feature.\n\n :return: None\n \"\"\"\n try:\n subprocess.check_call(\n [\n \"DISM\",\n \"/Online\",\n \"/Enable-Feature\",\n \"/All\",\n \"/NoRestart\",\n \"/FeatureName:Microsoft-Hyper-V\",\n ]\n )\n except subprocess.CalledProcessError as e:\n if e.returncode == 3010:\n pass # This is fine, because Windows.\n else:\n raise\n\n\nclass Linux(Auxiliary):\n \"\"\"\n MacOS auxiliary methods.\n \"\"\"\n\n def __init__(self, args) -> None:\n \"\"\"\n :param args: ArgumentParser\n :return: None\n \"\"\"\n super(Linux, self).__init__(args)\n\n\nclass MacOS(Linux):\n \"\"\"\n MacOS auxiliary methods.\n \"\"\"\n\n def __init__(self, args) -> None:\n \"\"\"\n :param args: ArgumentParser\n :return: None\n \"\"\"\n super(MacOS, self).__init__(args)\n" }, { "path": "installer/common/definitions.py", "content": "MAX_CHARACTERS_WRAP: int = 120\ncommand_descriptions = {\n \"add-node\": \"Adds a node to a cluster\",\n \"ambassador\": \"Ambassador API Gateway and Ingress\",\n \"cilium\": \"The cilium client\",\n \"config\": \"Print the kubeconfig\",\n \"ctr\": \"The containerd client\",\n \"dashboard-proxy\": \"Enable the Kubernetes dashboard and proxy to host\",\n \"dbctl\": \"Backup and restore the Kubernetes datastore\",\n \"disable\": \"Disables running add-ons\",\n \"enable\": \"Enables useful add-ons\",\n \"helm\": \"The helm client\",\n \"helm3\": \"The helm3 client\",\n \"inspect\": \"Checks the cluster and gathers logs\",\n \"istioctl\": \"The istio client\",\n \"join\": \"Joins this instance as a node to a cluster\",\n \"kubectl\": \"The kubernetes client\",\n \"leave\": \"Disconnects this node from any cluster it has joined\",\n \"linkerd\": \"The linkerd client\",\n \"refresh-certs\": \"Refresh the CA certificates in this deployment\",\n \"remove-node\": \"Removes a node from the cluster\",\n \"reset\": \"Cleans the cluster from all workloads\",\n \"start\": \"Starts the kubernetes cluster\",\n \"status\": \"Displays the status of the cluster\",\n \"stop\": \"Stops the kubernetes cluster\",\n}\nDEFAULT_CORES: int = 2\nDEFAULT_MEMORY_GB: int = 4\nDEFAULT_DISK_GB: int = 50\nDEFAULT_ASSUME: bool = False\nDEFAULT_CHANNEL: str = \"1.28/stable\"\nDEFAULT_IMAGE: str = \"22.04\"\n\nMIN_CORES: int = 2\nMIN_MEMORY_GB: int = 2\nMIN_DISK_GB: int = 10\n" }, { "path": "installer/common/errors.py", "content": "class BaseError(Exception):\n \"\"\"Base class for all exceptions.\n\n :cvar fmt: A format string that daughter classes override\n\n \"\"\"\n\n fmt = \"Daughter classes should redefine this\"\n\n def __init__(self, **kwargs) -> None:\n for key, value in kwargs.items():\n setattr(self, key, value)\n\n def __str__(self):\n return self.fmt.format([], **self.__dict__)\n\n def get_exit_code(self):\n \"\"\"Exit code to use if this exception causes the program to exit.\"\"\"\n return 2\n" }, { "path": "installer/common/file_utils.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2016-2019 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nimport hashlib\nimport logging\nimport os\nimport shutil\nimport sys\n\nif sys.version_info < (3, 6):\n pass\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef _file_reader_iter(path: str, block_size=2 ** 20):\n with open(path, \"rb\") as f:\n block = f.read(block_size)\n while len(block) > 0:\n yield block\n block = f.read(block_size)\n\n\ndef calculate_sha3_384(path: str) -> str:\n \"\"\"Calculate sha3 384 hash, reading the file in 1MB chunks.\"\"\"\n return calculate_hash(path, algorithm=\"sha3_384\")\n\n\ndef calculate_hash(path: str, *, algorithm: str) -> str:\n \"\"\"Calculate the hash for path with algorithm.\"\"\"\n # This will raise an AttributeError if algorithm is unsupported\n hasher = getattr(hashlib, algorithm)()\n\n for block in _file_reader_iter(path):\n hasher.update(block)\n return hasher.hexdigest()\n\n\ndef is_dumb_terminal():\n \"\"\"Return True if on a dumb terminal.\"\"\"\n is_stdout_tty = os.isatty(1)\n is_term_dumb = os.environ.get(\"TERM\", \"\") == \"dumb\"\n return not is_stdout_tty or is_term_dumb\n\n\ndef get_kubectl_directory() -> str:\n \"\"\"\n Get the correct directory to install kubectl into,\n we can then call this when running `microk8s kubectl`\n without interfering with any systemwide install.\n\n :return: String\n \"\"\"\n if sys.platform == \"win32\":\n if getattr(sys, \"frozen\", None):\n d = os.path.dirname(sys.executable)\n else:\n d = os.path.dirname(os.path.abspath(__file__))\n\n return os.path.join(d, \"kubectl\")\n else:\n full_path = shutil.which(\"kubectl\")\n return os.path.dirname(full_path)\n\n\ndef get_kubeconfig_path():\n \"\"\"Return a MicroK8s specific kubeconfig path.\"\"\"\n if sys.platform == \"win32\":\n return os.path.join(os.environ.get(\"LocalAppData\"), \"MicroK8s\", \"config\")\n else:\n return os.path.join(os.path.expanduser(\"~\"), \".microk8s\", \"config\")\n\n\ndef clear_kubeconfig():\n \"\"\"Clean kubeconfig file.\"\"\"\n if os.path.isdir(get_kubeconfig_path()):\n shutil.rmtree(os.path.dirname(get_kubeconfig_path()))\n" }, { "path": "installer/microk8s.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2020 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nfrom cli import microk8s\n\nif __name__ == \"__main__\":\n microk8s.cli()\n" }, { "path": "installer/microk8s.spec", "content": "# -*- mode: python ; coding: utf-8 -*-\n\nimport PyInstaller.config\nPyInstaller.config.CONF['distpath'] = \"./\"\n\nblock_cipher = None\n\na = Analysis(\n ['microk8s.py'],\n pathex=[''],\n binaries=[],\n datas=[],\n hiddenimports=[\n \"click\",\n \"responses\",\n \"pkg_resources.py2_warn\"\n ],\n hookspath=[],\n runtime_hooks=[],\n excludes=[],\n win_no_prefer_redirects=False,\n win_private_assemblies=False,\n cipher=block_cipher,\n noarchive=False\n)\n\npyz = PYZ(\n a.pure,\n a.zipped_data,\n cipher=block_cipher\n)\n\nexe = EXE(\n pyz,\n a.scripts,\n a.binaries,\n a.zipfiles,\n a.datas,\n [],\n name='microk8s',\n debug=False,\n bootloader_ignore_signals=False,\n strip=False,\n upx=True,\n upx_exclude=[],\n runtime_tmpdir=None,\n icon=\"windows/microk8s.ico\",\n console=True\n)\n\n" }, { "path": "installer/requirements.txt", "content": "urllib3==1.26.19\nclick==7.1.2\nprogressbar33==2.4\npsutil==5.9.0\nrequests==2.32.2\nrequests_unixsocket==0.1.5\npysha3==1.0.2; python_version < '3.6'\nsimplejson==3.8.2\ntoml==0.10.0\ncertifi==2024.7.4\nchardet==3.0.4\nidna==3.7\npyinstaller\nhttps://pyyaml.org/download/pyyaml/PyYAML-5.3.1-cp38-cp38-win_amd64.whl; sys_platform == 'win32'\n" }, { "path": "installer/setup.cfg", "content": "[flake8]\nmax-line-length = 120\n" }, { "path": "installer/setup.py", "content": "from setuptools import setup\n\nsetup(\n name=\"microk8s\",\n version=\"1.0.2\",\n url=\"https://github.com/canonical/microk8s\",\n license=\"Apache-2.0\",\n author=\"Joe Borg\",\n author_email=\"joseph.borg@canonical.com\",\n description=\"MicroK8s is a small, fast, single-package Kubernetes for developers, IoT and edge\",\n packages=[\"cli\", \"common\", \"vm_providers\", \"vm_providers._multipass\", \"vm_providers.repo\"],\n install_requires=[\n \"click~=7.0\",\n \"progressbar33==2.4\",\n \"requests==2.32.2\",\n \"requests_unixsocket==0.1.5\",\n \"simplejson==3.8.2\",\n \"toml==0.10.0\",\n \"urllib3==1.26.19\",\n ],\n platforms=\"any\",\n entry_points={\n \"console_scripts\": [\n \"microk8s=cli.microk8s:cli\",\n ]\n },\n)\n" }, { "path": "installer/tests/__init__.py", "content": "" }, { "path": "installer/tests/integration/test_cli.py", "content": "import os\nimport subprocess\nimport platform\nfrom unittest import mock\n\nimport pytest\nfrom click.testing import CliRunner\n\nfrom cli.microk8s import cli\n\n\nclass TestClass:\n @pytest.mark.skipif(\n platform.system() != \"Linux\", reason=\"Add/remove multipass is available on Linux\"\n )\n @pytest.mark.skipif(os.getuid() != 0, reason=\"Add/remove multipass is possible to root\")\n @mock.patch(\"sys.stdin.isatty\", return_value=True)\n def test_install_remove_multipass(self, tty_mock):\n \"\"\"\n We are root on a Linux box.\n Test the install and remove of multipass\n \"\"\"\n runner = CliRunner()\n # making sure we start on a clean machine with multipass\n result = runner.invoke(cli, \"uninstall\")\n subprocess.check_call(\"sudo snap install multipass --classic\".split())\n assert os.path.isfile(\"/snap/bin/multipass\")\n assert result.exit_code == 0\n\n # making sure we start on a clean machine\n result = runner.invoke(cli, \"install\")\n assert result.exit_code == 0\n assert os.path.isfile(\"/snap/bin/multipass\")\n\n result = runner.invoke(cli, \"status --wait-ready --timeout=60\")\n assert result.exit_code == 0\n\n result = runner.invoke(cli, \"install\")\n assert os.path.isfile(\"/snap/bin/multipass\")\n assert result.exit_code == 0\n\n def test_all_cli(self):\n runner = CliRunner()\n\n # Test no args. We should get an error.\n result = runner.invoke(cli)\n assert result.exit_code == 1\n\n # Test no args. We should get an error.\n result = runner.invoke(cli, \"--help\")\n assert result.exit_code == 0\n\n def test_install_argument_are_validated(self):\n runner = CliRunner()\n result = runner.invoke(cli, [\"install\", \"--mem\", \"1\"])\n assert result.exit_code == 2\n assert \"invalid memory value\" in result.output\n\n result = runner.invoke(cli, [\"install\", \"--cpu\", \"1\"])\n assert result.exit_code == 2\n assert \"invalid cpu value\" in result.output\n\n result = runner.invoke(cli, [\"install\", \"--disk\", \"1\"])\n assert result.exit_code == 2\n assert \"invalid disk value\" in result.output\n" }, { "path": "installer/tests/unit/test_auxiliary.py", "content": "from common.auxiliary import Auxiliary\nfrom unittest.mock import Mock, patch\n\n\ndef get_mocked_args(disk=1, mem=1, cpu=1):\n args = Mock(mem=mem, disk=disk, cpu=cpu)\n return args\n\n\n@patch(\"common.auxiliary.psutil.virtual_memory\")\ndef test_has_enough_memory(virtual_memory_mock):\n args = get_mocked_args(mem=50)\n host = Auxiliary(args)\n\n # 3 Bytes of memory\n virtual_memory_mock.return_value.total = 3\n assert host.has_enough_memory() is False\n\n # 300 GB of memory\n virtual_memory_mock.return_value.total = 300 * 1024 * 1024 * 1024\n assert host.has_enough_memory() is True\n\n\n@patch(\"common.auxiliary.psutil.cpu_count\")\ndef test_has_enough_cpus(cpu_count_mock):\n args = get_mocked_args(cpu=4)\n host = Auxiliary(args)\n\n cpu_count_mock.return_value = 3\n assert host.has_enough_cpus() is False\n\n cpu_count_mock.return_value = 5\n assert host.has_enough_cpus() is True\n\n\n@patch(\"common.auxiliary.disk_usage\")\ndef test_has_enough_disk_space(disk_usage_mock):\n args = get_mocked_args(disk=10)\n host = Auxiliary(args)\n\n disk_usage_mock.return_value.free = 3\n assert host.has_enough_disk_space() is False\n\n disk_usage_mock.return_value.free = 50 * 1024 * 1024 * 1024\n assert host.has_enough_disk_space() is True\n" }, { "path": "installer/tests/unit/test_cli.py", "content": "from cli.microk8s import install\nfrom unittest.mock import patch, MagicMock\nimport pytest\n\n\n@patch(\"common.auxiliary.Auxiliary.has_enough_cpus\", return_value=False)\ndef test_install_exits_on_cpus_requested_exceed_available_on_host(has_enough_cpus_mock):\n with pytest.raises(SystemExit) as exc:\n install(MagicMock())\n assert exc.value.code == 1\n has_enough_cpus_mock.assert_called_once()\n\n\n@patch(\"common.auxiliary.Auxiliary.has_enough_memory\", return_value=False)\n@patch(\"common.auxiliary.Auxiliary.has_enough_cpus\", return_value=True)\ndef test_install_exits_on_memory_requested_exceed_available_on_host(\n has_enough_cpus_mock,\n has_enough_memory_mock,\n):\n with pytest.raises(SystemExit) as exc:\n install(MagicMock())\n assert exc.value.code == 1\n has_enough_memory_mock.assert_called_once()\n" }, { "path": "installer/vm_providers/__init__.py", "content": "" }, { "path": "installer/vm_providers/_base_provider.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2018-2019 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nimport abc\nimport logging\nimport os\nimport pathlib\nimport requests\nimport shlex\nimport sys\nfrom typing import Dict\nfrom typing import Optional, Sequence\n\nfrom . import errors\nfrom ._multipass._instance_info import InstanceInfo\n\nlogger = logging.getLogger(__name__)\n\n\nclass Provider(abc.ABC):\n def __init__(\n self,\n *,\n echoer,\n is_ephemeral: bool = False,\n build_provider_flags: Dict[str, str] = None,\n ) -> None:\n self.echoer = echoer\n self._is_ephemeral = is_ephemeral\n\n self.instance_name = \"microk8s-vm\"\n\n if build_provider_flags is None:\n build_provider_flags = dict()\n self.build_provider_flags = build_provider_flags.copy()\n\n self._cached_home_directory: Optional[pathlib.Path] = None\n\n @classmethod\n def ensure_provider(cls) -> None:\n \"\"\"Necessary steps to ensure the provider is correctly setup.\"\"\"\n\n @classmethod\n def setup_provider(cls, *, echoer) -> None:\n \"\"\"Necessary steps to install the provider on the host.\"\"\"\n\n @classmethod\n def _get_provider_name(cls) -> str:\n \"\"\"Return the provider name.\"\"\"\n\n @classmethod\n def _get_is_snap_injection_capable(cls) -> bool:\n \"\"\"Return whether the provider can install snaps from the host.\"\"\"\n\n @abc.abstractmethod\n def create(self) -> None:\n \"\"\"Provider steps needed to create a fully functioning environment.\"\"\"\n\n @abc.abstractmethod\n def destroy(self) -> None:\n \"\"\"Provider steps needed to ensure the instance is destroyed.\n\n This method should be safe to call multiple times and do nothing\n if the instance to destroy is already destroyed.\n \"\"\"\n\n @abc.abstractmethod\n def get_instance_info(self) -> InstanceInfo:\n \"\"\"Return the instance info.\"\"\"\n\n @abc.abstractmethod\n def run(self, command: Sequence[str], hide_output: bool = False) -> Optional[bytes]:\n \"\"\"Run a command on the instance.\"\"\"\n\n @abc.abstractmethod\n def _launch(self, specs: Dict):\n \"\"\"Launch the instance.\"\"\"\n\n @abc.abstractmethod\n def _start(self):\n \"\"\"Start an existing the instance.\"\"\"\n\n @abc.abstractmethod\n def _push_file(self, *, source: str, destination: str) -> None:\n \"\"\"Push a file into the instance.\"\"\"\n\n @abc.abstractmethod\n def pull_file(self, name: str, destination: str, delete: bool = False) -> None:\n \"\"\"\n Provider steps needed to retrieve a file from the instance, optionally\n deleting the source file after a successful retrieval.\n\n :param name: the remote filename.\n :type name: str\n :param destination: the local filename.\n :type destination: str\n :param delete: whether the file should be deleted.\n :type delete: bool\n \"\"\"\n\n @abc.abstractmethod\n def shell(self) -> None:\n \"\"\"Provider steps to provide a shell into the instance.\"\"\"\n\n def launch_instance(self, specs: Dict) -> None:\n try:\n # An ProviderStartError exception here means we need to create.\n self._start()\n except errors.ProviderInstanceNotFoundError:\n self._launch(specs)\n self._check_connectivity()\n # We need to setup MicroK8s and scan for cli commands.\n self._setup_microk8s(specs)\n self._copy_kubeconfig_to_kubectl(specs)\n\n def _check_connectivity(self) -> None:\n \"\"\"Check that the VM can access the internet.\"\"\"\n try:\n requests.get(\"https://snapcraft.io\")\n except requests.exceptions.RequestException:\n self.destroy()\n url = None\n if sys.platform == \"win32\":\n url = \"https://multipass.run/docs/troubleshooting-networking-on-windows\"\n elif sys.platform == \"darwin\":\n url = \"https://multipass.run/docs/troubleshooting-networking-on-macos\"\n\n if url:\n raise errors.ConnectivityError(\n \"The VM cannot connect to snapcraft.io, please see {}\".format(url)\n )\n else:\n raise\n\n def _copy_kubeconfig_to_kubectl(self, specs: Dict):\n kubeconfig_path = specs.get(\"kubeconfig\")\n kubeconfig = self.run(command=[\"microk8s\", \"config\"], hide_output=True)\n\n if not os.path.isdir(os.path.dirname(kubeconfig_path)):\n os.mkdir(os.path.dirname(kubeconfig_path))\n\n with open(kubeconfig_path, \"wb\") as f:\n f.write(kubeconfig)\n\n def _setup_microk8s(self, specs: Dict) -> None:\n self.run(\"snap install microk8s --classic --channel {}\".format(specs[\"channel\"]).split())\n if sys.platform == \"win32\":\n self.run(\"snap install microk8s-integrator-windows\".split())\n elif sys.platform == \"darwin\":\n self.run(\"snap install microk8s-integrator-macos\".split())\n\n def _get_env_command(self) -> Sequence[str]:\n \"\"\"Get command sequence for `env` with configured flags.\"\"\"\n\n env_list = [\"env\"]\n\n # Pass through configurable environment variables.\n for key in [\"http_proxy\", \"https_proxy\"]:\n value = self.build_provider_flags.get(key)\n if not value:\n continue\n\n # Ensure item is treated as string and append it.\n value = str(value)\n env_list.append(f\"{key}={value}\")\n\n return env_list\n\n def _get_home_directory(self) -> pathlib.Path:\n \"\"\"Get user's home directory path.\"\"\"\n if self._cached_home_directory is not None:\n return self._cached_home_directory\n\n command = [\"printenv\", \"HOME\"]\n run_output = self.run(command=command, hide_output=True)\n\n # Shouldn't happen, but due to _run()'s return type as being Optional,\n # we need to check for it anyways for mypy.\n if not run_output:\n provider_name = self._get_provider_name()\n raise errors.ProviderExecError(\n provider_name=provider_name, command=command, exit_code=2\n )\n\n cached_home_directory = pathlib.Path(run_output.decode().strip())\n\n self._cached_home_directory = cached_home_directory\n return cached_home_directory\n\n def _base_has_changed(self, base: str, provider_base: str) -> bool:\n # Make it backwards compatible with instances without project info\n if base == \"core18\" and provider_base is None:\n return False\n elif base != provider_base:\n return True\n\n return False\n\n def _log_run(self, command: Sequence[str]) -> None:\n cmd_string = \" \".join([shlex.quote(c) for c in command])\n logger.debug(f\"Running: {cmd_string}\")\n\n @abc.abstractmethod\n def stop(self) -> None:\n pass\n" }, { "path": "installer/vm_providers/_multipass/__init__.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2018 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nfrom ._multipass import Multipass # noqa: F401\nfrom ._multipass_command import MultipassCommand # noqa: F401\n" }, { "path": "installer/vm_providers/_multipass/_instance_info.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2018 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nimport json\nfrom typing import Any, Dict, Type\n\nfrom vm_providers import errors\n\n\nclass InstanceInfo:\n @classmethod\n def from_json(\n cls: Type[\"InstanceInfo\"], *, instance_name: str, json_info: str\n ) -> \"InstanceInfo\":\n \"\"\"Create an InstanceInfo from json_info retrieved from multipass.\n\n :param str instance_name: the name of the instance.\n :param str json_info: a json formatted string with the structure\n that would follow the output of a json formatted\n multipass info command.\n :returns: an InstanceInfo.\n :rtype: InstanceInfo\n :raises vm_providers.ProviderInfoDataKeyError:\n if the instance name cannot be found in the given json or if a\n required key is missing from that data structure for the instance.\n \"\"\"\n try:\n json_data = json.loads(json_info)\n except json.decoder.JSONDecodeError as decode_error:\n raise errors.ProviderBadDataError(\n provider_name=\"multipass\", data=json_info\n ) from decode_error\n try:\n instance_info = json_data[\"info\"][instance_name]\n except KeyError as missing_key:\n raise errors.ProviderInfoDataKeyError(\n provider_name=\"multipass\", missing_key=str(missing_key), data=json_data\n ) from missing_key\n try:\n return cls(\n name=instance_name,\n state=instance_info[\"state\"],\n image_release=instance_info[\"image_release\"],\n mounts=instance_info[\"mounts\"],\n ipv4=instance_info[\"ipv4\"],\n )\n except KeyError as missing_key:\n raise errors.ProviderInfoDataKeyError(\n provider_name=\"multipass\",\n missing_key=str(missing_key),\n data=instance_info,\n ) from missing_key\n\n def __init__(\n self, *, name: str, state: str, image_release: str, mounts: Dict[str, Any], ipv4: str\n ) -> None:\n \"\"\"Initialize an InstanceInfo.\n\n :param str name: the instance name.\n :param str state: the state of the instance which can be any one of\n RUNNING, STOPPED, DELETED.\n :param str image_release: the Operating System release string for the\n image.\n \"\"\"\n # We do not check for validity of state given that new states could\n # be introduced.\n self.name = name\n self.state = state\n self.image_release = image_release\n self.mounts = mounts\n self.ipv4 = ipv4\n\n def is_mounted(self, mountpoint: str) -> bool:\n return mountpoint in self.mounts\n\n def is_stopped(self) -> bool:\n return self.state.upper() == \"STOPPED\"\n\n def is_running(self) -> bool:\n return self.state.upper() == \"RUNNING\"\n" }, { "path": "installer/vm_providers/_multipass/_multipass.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2018-2019 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nimport logging\nimport sys\nfrom typing import Dict, Optional, Sequence\n\nfrom time import sleep\nfrom vm_providers import errors\nfrom .._base_provider import Provider\nfrom ._instance_info import InstanceInfo\nfrom ._multipass_command import MultipassCommand\n\nlogger = logging.getLogger(__name__)\n\n\nclass Multipass(Provider):\n \"\"\"A multipass provider for MicroK8s to execute its lifecycle.\"\"\"\n\n @classmethod\n def ensure_provider(cls):\n MultipassCommand.ensure_multipass(platform=sys.platform)\n\n @classmethod\n def setup_provider(cls, *, echoer) -> None:\n MultipassCommand.setup_multipass(echoer=echoer, platform=sys.platform)\n\n @classmethod\n def _get_is_snap_injection_capable(cls) -> bool:\n return True\n\n @classmethod\n def _get_provider_name(cls):\n return \"multipass\"\n\n def run(self, command: Sequence[str], hide_output: bool = False) -> Optional[bytes]:\n cmd = [\"sudo\"]\n cmd.extend(command)\n self._log_run(cmd)\n\n return self._multipass_cmd.execute(\n instance_name=self.instance_name, command=cmd, hide_output=hide_output\n )\n\n def _launch(self, specs: Dict) -> None:\n\n # prepare core launch setting\n image = specs[\"image\"]\n cpus = \"{}\".format(specs[\"cpu\"])\n mem = \"{}G\".format(specs[\"mem\"])\n disk = \"{}G\".format(specs[\"disk\"])\n\n try_for = 10\n\n while True:\n try:\n self._multipass_cmd.launch(\n instance_name=self.instance_name, cpus=cpus, mem=mem, disk=disk, image=image\n )\n except Exception:\n if try_for > 0:\n try_for -= 1\n sleep(1)\n continue\n else:\n raise\n else:\n break\n\n def get_instance_info(self) -> InstanceInfo:\n try:\n instance_info = self._get_instance_info()\n return instance_info\n except errors.ProviderInfoError as instance_error:\n # Until we have proper multipass error codes to know if this\n # was a communication error we should keep this error tracking\n # and generation here.\n raise errors.ProviderInstanceNotFoundError(\n instance_name=self.instance_name\n ) from instance_error\n\n def _start(self):\n try:\n instance_info = self._get_instance_info()\n if not instance_info.is_running():\n self._multipass_cmd.start(instance_name=self.instance_name)\n\n except errors.ProviderInfoError as instance_error:\n # Until we have proper multipass error codes to know if this\n # was a communication error we should keep this error tracking\n # and generation here.\n raise errors.ProviderInstanceNotFoundError(\n instance_name=self.instance_name\n ) from instance_error\n\n def _umount(self, *, mountpoint: str) -> None:\n mount = \"{}:{}\".format(self.instance_name, mountpoint)\n self._multipass_cmd.umount(mount=mount)\n\n def _push_file(self, *, source: str, destination: str) -> None:\n destination = \"{}:{}\".format(self.instance_name, destination)\n self._multipass_cmd.copy_files(source=source, destination=destination)\n\n def __init__(\n self,\n *,\n echoer,\n is_ephemeral: bool = False,\n build_provider_flags: Dict[str, str] = None,\n ) -> None:\n super().__init__(\n echoer=echoer,\n is_ephemeral=is_ephemeral,\n build_provider_flags=build_provider_flags,\n )\n self._multipass_cmd = MultipassCommand(platform=sys.platform)\n self._instance_info: Optional[InstanceInfo] = None\n\n def create(self, specs: Dict) -> None:\n \"\"\"Create the multipass instance and setup the build environment.\"\"\"\n self.echoer.info(\"Launching a VM.\")\n self.launch_instance(specs)\n self._instance_info = self._get_instance_info()\n\n def destroy(self) -> None:\n \"\"\"Destroy the instance, trying to stop it first.\"\"\"\n try:\n instance_info = self._instance_info = self._get_instance_info()\n except errors.ProviderInfoError:\n return\n\n if instance_info.is_stopped():\n return\n\n self._multipass_cmd.stop(instance_name=self.instance_name)\n self._multipass_cmd.delete(instance_name=self.instance_name, purge=True)\n\n def pull_file(self, name: str, destination: str, delete: bool = False) -> None:\n # TODO add instance check.\n\n # check if file exists in instance\n self.run(command=[\"test\", \"-f\", name])\n\n # copy file from instance\n source = \"{}:{}\".format(self.instance_name, name)\n self._multipass_cmd.copy_files(source=source, destination=destination)\n if delete:\n self.run(command=[\"rm\", name])\n\n def shell(self) -> None:\n self.run(command=[\"/bin/bash\"])\n\n def _get_instance_info(self) -> InstanceInfo:\n instance_info_raw = self._multipass_cmd.info(\n instance_name=self.instance_name, output_format=\"json\"\n )\n return InstanceInfo.from_json(\n instance_name=self.instance_name, json_info=instance_info_raw.decode()\n )\n\n def start(self) -> None:\n instance_info = self._instance_info = self._get_instance_info()\n if not instance_info.is_stopped():\n return\n\n self._multipass_cmd.start(instance_name=self.instance_name)\n\n def stop(self) -> None:\n instance_info = self._instance_info = self._get_instance_info()\n if instance_info.is_stopped():\n return\n\n self._multipass_cmd.stop(instance_name=self.instance_name)\n" }, { "path": "installer/vm_providers/_multipass/_multipass_command.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2018 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nimport logging\nimport shutil\nimport subprocess\n\nfrom time import sleep\nfrom typing import Dict, Optional, Sequence # noqa: F401\n\nfrom ._windows import windows_reload_multipass_path_env, windows_install_multipass\n\nfrom vm_providers import errors\nfrom vm_providers.repo.snaps import install_snaps\n\nlogger = logging.getLogger(__name__)\n\n\ndef _run(command: Sequence[str], stdin=subprocess.DEVNULL) -> None:\n logger.debug(\"Running {}\".format(\" \".join(command)))\n subprocess.check_call(command, stdin=stdin)\n\n\ndef _run_output(command: Sequence[str], **kwargs) -> bytes:\n logger.debug(\"Running {}\".format(\" \".join(command)))\n return subprocess.check_output(command, **kwargs)\n\n\nclass MultipassCommand:\n \"\"\"An object representation of common multipass cli commands.\"\"\"\n\n provider_name = \"multipass\"\n provider_cmd = \"multipass\"\n\n @classmethod\n def ensure_multipass(cls, platform: str) -> None:\n if platform == \"win32\":\n # Reload path env just in case multipass was installed without\n # launching a new command prompt / shell.\n windows_reload_multipass_path_env()\n\n if shutil.which(cls.provider_cmd):\n return\n\n if platform == \"darwin\":\n prompt_installable = True\n elif platform == \"linux\" and shutil.which(\"snap\"):\n prompt_installable = True\n elif platform == \"win32\":\n prompt_installable = True\n else:\n prompt_installable = False\n\n raise errors.ProviderNotFound(\n provider=cls.provider_name,\n prompt_installable=prompt_installable,\n error_message=\"https://multipass.run\",\n )\n\n @classmethod\n def _wait_for_multipass_ready(cls, *, echoer):\n echoer.wrapped(\"Waiting for multipass...\")\n retry_count = 60\n while retry_count:\n try:\n output = subprocess.check_output([cls.provider_cmd, \"version\"]).decode()\n except subprocess.CalledProcessError:\n output = \"\"\n except FileNotFoundError:\n raise errors.ProviderStartError(\n provider_name=cls.provider_name,\n error_message=\"multipass not found - please check that it\"\n \" can be found in the configured PATH\",\n )\n\n # if multipassd is in the version information, it means the service is up\n # and we can carry on\n if \"multipassd\" in output:\n break\n\n retry_count -= 1\n sleep(1)\n\n # No need to worry about getting to this point by exhausting our retry count,\n # the rest of the stack will handle the error appropriately.\n\n @classmethod\n def setup_multipass(cls, *, echoer, platform: str) -> None:\n if platform == \"linux\":\n install_snaps([\"multipass/latest/stable\"])\n elif platform == \"darwin\":\n try:\n subprocess.check_call([\"brew\", \"install\", \"multipass\", \"--cask\"])\n except subprocess.CalledProcessError:\n raise errors.ProviderStartError(\n provider_name=cls.provider_name,\n error_message=\"Failed to install multipass using homebrew.\\n\"\n \"Verify your homebrew installation and try again.\\n\"\n \"Alternatively, manually install multipass by running\"\n \" 'brew install multipass --cask'.\",\n )\n elif platform == \"win32\":\n windows_install_multipass(echoer)\n else:\n raise EnvironmentError(\n \"Setting up multipass for {!r} is not supported.\".format(platform)\n )\n\n # wait for multipassd to be available\n cls._wait_for_multipass_ready(echoer=echoer)\n\n def __init__(self, *, platform: str) -> None:\n \"\"\"Initialize a MultipassCommand instance.\n\n :raises errors.ProviderCommandNotFound:\n if the multipass command is not found.\n \"\"\"\n self.ensure_multipass(platform=platform)\n\n def launch(\n self,\n *,\n instance_name: str,\n image: str,\n cpus: str = None,\n mem: str = None,\n disk: str = None,\n remote: str = None,\n cloud_init: str = None\n ) -> None:\n \"\"\"Passthrough for running multipass launch.\n\n :param str instance_name: the name the launched instance will have.\n :param str image: the image to create the instance with.\n :param str cpus: amount of virtual CPUs to assign to the launched instance.\n :param str mem: amount of RAM to assign to the launched instance.\n :param str disk: amount of disk space the instance will see.\n :param str remote: the remote server to retrieve the image from.\n :param str cloud_init: path to a user-data cloud-init configuration.\n \"\"\"\n if remote is not None:\n image = \"{}:{}\".format(remote, image)\n cmd = [self.provider_cmd, \"launch\", image, \"--name\", instance_name]\n if cloud_init is not None:\n cmd.extend([\"--cloud-init\", cloud_init])\n if cpus is not None:\n cmd.extend([\"--cpus\", cpus])\n if mem is not None:\n cmd.extend([\"--memory\", mem])\n if disk is not None:\n cmd.extend([\"--disk\", disk])\n try:\n _run(cmd)\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderLaunchError(\n provider_name=self.provider_name, exit_code=process_error.returncode\n ) from process_error\n\n def start(self, *, instance_name: str) -> None:\n \"\"\"Passthrough for running multipass start.\n\n :param str instance_name: the name of the instance to start.\n \"\"\"\n cmd = [self.provider_cmd, \"start\", instance_name]\n try:\n _run(cmd)\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderStartError(\n provider_name=self.provider_name, exit_code=process_error.returncode\n ) from process_error\n\n def stop(self, *, instance_name: str, time: int = None) -> None:\n \"\"\"Passthrough for running multipass stop.\n\n :param str instance_name: the name of the instance to stop.\n :param str time: time from now, in minutes, to delay shutdown of the\n instance.\n \"\"\"\n cmd = [self.provider_cmd, \"stop\"]\n if time:\n cmd.extend([\"--time\", str(time)])\n cmd.append(instance_name)\n try:\n _run(cmd)\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderStopError(\n provider_name=self.provider_name, exit_code=process_error.returncode\n ) from process_error\n\n def delete(self, *, instance_name: str, purge=True) -> None:\n \"\"\"Passthrough for running multipass delete.\n\n :param str instance_name: the name of the instance to delete.\n :param bool purge: if true, purge the instance's image after deleting.\n \"\"\"\n cmd = [self.provider_cmd, \"delete\", instance_name]\n if purge:\n cmd.append(\"--purge\")\n try:\n _run(cmd)\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderDeleteError(\n provider_name=self.provider_name, exit_code=process_error.returncode\n ) from process_error\n\n def execute(\n self, *, command: Sequence[str], instance_name: str, hide_output: bool = False\n ) -> Optional[bytes]:\n \"\"\"Passthrough for running multipass exec.\n\n :param list command: the command to execute on the instance.\n :param str instance_name: the name of the instance to execute command.\n :param bool hide_output: hide the output from stdout.\n \"\"\"\n cmd = [self.provider_cmd, \"exec\", instance_name, \"--\"] + list(command)\n output = None\n try:\n if hide_output:\n output = _run_output(cmd)\n else:\n _run(cmd, stdin=None)\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderExecError(\n provider_name=self.provider_name,\n command=command,\n exit_code=process_error.returncode,\n ) from process_error\n\n return output\n\n def shell(self, *, instance_name: str) -> None:\n \"\"\"Passthrough for running multipass shell.\n\n :param str instance_name: the name of the instance to execute command.\n \"\"\"\n try:\n _run([self.provider_cmd, \"shell\", instance_name])\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderShellError(\n provider_name=self.provider_name, exit_code=process_error.returncode\n ) from process_error\n\n def mount(\n self,\n *,\n source: str,\n target: str,\n uid_map: Dict[str, str] = None,\n gid_map: Dict[str, str] = None\n ) -> None:\n \"\"\"Passthrough for running multipass mount.\n\n :param str source: path to the local directory to mount.\n :param str target: mountpoint inside the instance in the form of\n :path.\n :param dict uid_map: A mapping of user IDs for use in the mount of the form\n -> .\n File and folder ownership will be mapped from\n to inside the instance.\n :param dict gid_map: A mapping of group IDs for use in the mount of the form\n -> .\n File and folder ownership will be mapped from\n to inside the instance.\n :raises errors.ProviderMountError: when the mount operation fails.\n \"\"\"\n cmd = [self.provider_cmd, \"mount\", source, target]\n if uid_map is None:\n uid_map = dict()\n for host_map, instance_map in uid_map.items():\n cmd.extend([\"--uid-map\", \"{}:{}\".format(host_map, instance_map)])\n if gid_map is None:\n gid_map = dict()\n for host_map, instance_map in gid_map.items():\n cmd.extend([\"--gid-map\", \"{}:{}\".format(host_map, instance_map)])\n try:\n _run(cmd)\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderMountError(\n provider_name=self.provider_name, exit_code=process_error.returncode\n ) from process_error\n\n def umount(self, *, mount: str) -> None:\n \"\"\"Passthrough for running multipass mount.\n\n :param str mount: mountpoint inside the instance in the form of\n :path to unmount.\n :raises errors.ProviderUMountError: when the unmount operation fails.\n \"\"\"\n cmd = [self.provider_cmd, \"umount\", mount]\n try:\n _run(cmd)\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderUnMountError(\n provider_name=self.provider_name, exit_code=process_error.returncode\n ) from process_error\n\n def copy_files(self, *, source: str, destination: str) -> None:\n \"\"\"Passthrough for running multipass copy-files.\n\n :param str source: the source file to copy, using syntax expected\n by multipass.\n :param str destination: the destination of the copied file, using\n syntax expected by multipass.\n \"\"\"\n cmd = [self.provider_cmd, \"copy-files\", source, destination]\n try:\n _run(cmd)\n except subprocess.CalledProcessError as process_error:\n raise errors.ProviderFileCopyError(\n provider_name=self.provider_name, exit_code=process_error.returncode\n ) from process_error\n\n def info(self, *, instance_name: str, output_format: str = None) -> bytes:\n \"\"\"Passthrough for running multipass info.\"\"\"\n cmd = [self.provider_cmd, \"info\", instance_name]\n if output_format is not None:\n cmd.extend([\"--format\", output_format])\n process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n stdout, stderr = process.communicate()\n if process.returncode != 0:\n raise errors.ProviderInfoError(\n provider_name=self.provider_name,\n exit_code=process.returncode,\n stderr=stderr,\n )\n return stdout\n" }, { "path": "installer/vm_providers/_multipass/_windows.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2018 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nimport logging\nimport os.path\nimport requests\nimport shutil\nimport simplejson\nimport subprocess\nimport sys\nimport tempfile\n\nfrom progressbar import AnimatedMarker, Bar, Percentage, ProgressBar, UnknownLength\n\nfrom common.file_utils import calculate_sha3_384, is_dumb_terminal\nfrom vm_providers.errors import (\n ProviderMultipassDownloadFailed,\n ProviderMultipassInstallationFailed,\n)\n\nif sys.platform == \"win32\":\n import winreg\n\n\nlogger = logging.getLogger(__name__)\n\n\n_MULTIPASS_RELEASES_API_URL = \"https://api.github.com/repos/canonical/multipass/releases\"\n_MULTIPASS_DL_VERSION = \"1.12.2\"\n_MULTIPASS_DL_NAME = \"multipass-{version}+win-win64.exe\".format(version=_MULTIPASS_DL_VERSION)\n\n# Download multipass installer and calculate hash:\n# python3 -c \"from installer.common.file_utils import calculate_sha3_384; print(calculate_sha3_384('$HOME/Downloads/multipass-1.11.1+win-win64.exe'))\" # noqa: E501\n_MULTIPASS_DL_SHA3_384 = \"9031c8fc98b941df1094a832c356e12f281c70d0eb10bee15b5576c61af4c8a17ef32b833f0043c8df0e04897e69c8bc\" # noqa: E501\n\n\ndef windows_reload_multipass_path_env():\n \"\"\"Update PATH to include installed Multipass, if not already set.\"\"\"\n\n assert sys.platform == \"win32\"\n\n key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, \"Environment\")\n\n paths = os.environ[\"PATH\"].split(\";\")\n\n # Drop empty placeholder for trailing comma, if present.\n if paths[-1] == \"\":\n del paths[-1]\n\n reg_user_path, _ = winreg.QueryValueEx(key, \"Path\")\n for path in reg_user_path.split(\";\"):\n if path not in paths and \"Multipass\" in path:\n paths.append(path)\n\n # Restore path with trailing comma.\n os.environ[\"PATH\"] = \";\".join(paths) + \";\"\n\n\ndef _run_installer(installer_path: str, echoer):\n \"\"\"Execute multipass installer.\"\"\"\n\n echoer.info(\"Installing Multipass...\")\n\n # Multipass requires administrative privileges to install, which requires\n # the use of `runas` functionality. Some of the options included:\n # (1) https://stackoverflow.com/a/34216774\n # (2) ShellExecuteW and wait on installer by attempting to delete it.\n # Windows would prevent us from deleting installer with a PermissionError:\n # PermissionError: [WinError 32] The process cannot access the file because\n # it is being used by another process: \n # (3) Use PowerShell's \"Start-Process\" with RunAs verb as shown below.\n # None of the options are quite ideal, but #3 will do.\n cmd = \"\"\"\n & {{\n try {{\n $Output = Start-Process -FilePath {path!r} -Args /S -Verb RunAs -Wait -PassThru\n }} catch {{\n [Environment]::Exit(1)\n }}\n }}\n \"\"\".format(\n path=installer_path\n )\n\n try:\n subprocess.check_call([\"powershell.exe\", \"-Command\", cmd])\n except subprocess.CalledProcessError:\n raise ProviderMultipassInstallationFailed(\"error launching installer\")\n\n # Reload path environment to see if we can find multipass now.\n windows_reload_multipass_path_env()\n\n if not shutil.which(\"multipass.exe\"):\n # Installation failed.\n raise ProviderMultipassInstallationFailed(\"installation did not complete successfully\")\n\n echoer.info(\"Multipass installation completed successfully.\")\n\n\ndef _requests_exception_hint(e: requests.RequestException) -> str:\n # Use the __doc__ description to give the user a hint. It seems to be a\n # a decent option over trying to enumerate all of possible types.\n if e.__doc__:\n split_lines = e.__doc__.splitlines()\n if split_lines:\n return e.__doc__.splitlines()[0].decode().strip()\n\n # Should never get here.\n return \"unknown download error\"\n\n\ndef _fetch_installer_url() -> str:\n \"\"\"Verify version set is a valid\n ref in GitHub and return the full\n URL.\n \"\"\"\n\n try:\n resp = requests.get(_MULTIPASS_RELEASES_API_URL)\n except requests.RequestException as e:\n raise ProviderMultipassDownloadFailed(_requests_exception_hint(e))\n\n try:\n data = resp.json()\n except simplejson.JSONDecodeError:\n raise ProviderMultipassDownloadFailed(\n \"failed to fetch valid release data from {}\".format(_MULTIPASS_RELEASES_API_URL)\n )\n\n for assets in data:\n for asset in assets.get(\"assets\", list()):\n # Find matching name.\n if asset.get(\"name\") != _MULTIPASS_DL_NAME:\n continue\n\n return asset.get(\"browser_download_url\")\n\n # Something changed we don't know about - we will simply categorize\n # all possible events as an updated version we do not yet know about.\n raise ProviderMultipassDownloadFailed(\"ref specified is not a valid ref in GitHub\")\n\n\ndef _download_multipass(dl_dir: str, echoer) -> str:\n \"\"\"Creates temporary Downloads installer to temp directory.\"\"\"\n\n dl_url = _fetch_installer_url()\n dl_basename = os.path.basename(dl_url)\n dl_path = os.path.join(dl_dir, dl_basename)\n\n echoer.info(\"Downloading Multipass installer...\\n{} -> {}\".format(dl_url, dl_path))\n\n try:\n request = requests.get(dl_url, stream=True, allow_redirects=True)\n request.raise_for_status()\n download_requests_stream(request, dl_path)\n except requests.RequestException as e:\n raise ProviderMultipassDownloadFailed(_requests_exception_hint(e))\n\n digest = calculate_sha3_384(dl_path)\n if digest != _MULTIPASS_DL_SHA3_384:\n raise ProviderMultipassDownloadFailed(\n \"download failed verification (expected={} but found={})\".format(\n _MULTIPASS_DL_SHA3_384, digest\n )\n )\n\n echoer.info(\"Verified installer successfully...\")\n return dl_path\n\n\ndef windows_install_multipass(echoer) -> None:\n \"\"\"Download and install multipass.\"\"\"\n\n assert sys.platform == \"win32\"\n\n dl_dir = tempfile.mkdtemp()\n dl_path = _download_multipass(dl_dir, echoer)\n _run_installer(dl_path, echoer)\n\n # Cleanup.\n shutil.rmtree(dl_dir)\n\n\ndef _init_progress_bar(total_length, destination, message=None):\n if not message:\n message = \"Downloading {!r}\".format(os.path.basename(destination))\n\n valid_length = total_length and total_length > 0\n\n if valid_length and is_dumb_terminal():\n widgets = [message, \" \", Percentage()]\n maxval = total_length\n elif valid_length and not is_dumb_terminal():\n widgets = [message, Bar(marker=\"=\", left=\"[\", right=\"]\"), \" \", Percentage()]\n maxval = total_length\n elif not valid_length and is_dumb_terminal():\n widgets = [message]\n maxval = UnknownLength\n else:\n widgets = [message, AnimatedMarker()]\n maxval = UnknownLength\n\n return ProgressBar(widgets=widgets, maxval=maxval)\n\n\ndef download_requests_stream(request_stream, destination, message=None, total_read=0):\n \"\"\"This is a facility to download a request with nice progress bars.\"\"\"\n\n # Doing len(request_stream.content) may defeat the purpose of a\n # progress bar\n total_length = 0\n if not request_stream.headers.get(\"Content-Encoding\", \"\"):\n total_length = int(request_stream.headers.get(\"Content-Length\", \"0\"))\n # Content-Length in the case of resuming will be\n # Content-Length - total_read so we add back up to have the feel of\n # resuming\n if os.path.exists(destination):\n total_length += total_read\n\n progress_bar = _init_progress_bar(total_length, destination, message)\n progress_bar.start()\n\n if os.path.exists(destination):\n mode = \"ab\"\n else:\n mode = \"wb\"\n with open(destination, mode) as destination_file:\n for buf in request_stream.iter_content(1024):\n destination_file.write(buf)\n if not is_dumb_terminal():\n total_read += len(buf)\n progress_bar.update(total_read)\n progress_bar.finish()\n" }, { "path": "installer/vm_providers/errors.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2018-2019 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nimport shlex\nfrom typing import Any, Dict, Optional\nfrom typing import Sequence # noqa: F401\n\nfrom common.errors import BaseError\n\n\nclass ConnectivityError(BaseError):\n pass\n\n\nclass ProviderBaseError(BaseError):\n pass\n\n\nclass ProviderNotSupportedError(ProviderBaseError):\n\n fmt = (\n \"The {provider!r} provider is not supported, please choose a \"\n \"different one and try again.\"\n )\n\n def __init__(self, *, provider: str) -> None:\n super().__init__(provider=provider)\n\n\nclass ProviderNotFound(ProviderBaseError):\n\n fmt = \"You need {provider!r} set-up to build snaps: {error_message}.\"\n\n def __init__(self, *, provider: str, prompt_installable: bool, error_message: str) -> None:\n super().__init__(\n provider=provider,\n prompt_installable=prompt_installable,\n error_message=error_message,\n )\n self.prompt_installable = prompt_installable\n self.provider = provider\n\n\nclass _GenericProviderError(ProviderBaseError):\n\n _FMT_ERROR_MESSAGE_AND_EXIT_CODE = (\n \"An error occurred with the instance when trying to {action} with \"\n \"{provider_name!r}: returned exit code {exit_code!r}: {error_message}.\\n\"\n \"Ensure that {provider_name!r} is setup correctly and try again.\"\n )\n\n _FMT_ERROR_MESSAGE = (\n \"An error occurred with the instance when trying to {action} with \"\n \"{provider_name!r}: {error_message}.\\n\"\n \"Ensure that {provider_name!r} is setup correctly and try again.\"\n )\n\n _FMT_EXIT_CODE = (\n \"An error occurred with the instance when trying to {action} with \"\n \"{provider_name!r}: returned exit code {exit_code!r}.\\n\"\n \"Ensure that {provider_name!r} is setup correctly and try again.\"\n )\n\n def __init__(\n self,\n *,\n provider_name: str,\n action: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n if exit_code is not None and error_message is not None:\n fmt = self._FMT_ERROR_MESSAGE_AND_EXIT_CODE\n elif error_message:\n fmt = self._FMT_ERROR_MESSAGE\n elif exit_code:\n fmt = self._FMT_EXIT_CODE\n else:\n raise RuntimeError(\"error_message nor exit_code are set\")\n\n self.fmt = fmt\n\n super().__init__(\n provider_name=provider_name,\n action=action,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderCommunicationError(ProviderBaseError):\n\n fmt = (\n \"An error occurred when trying to communicate with the \"\n \"{provider_name!r} provider: {message}.\"\n )\n\n def __init__(self, *, provider_name: str, message: str) -> None:\n super().__init__(provider_name=provider_name, message=message)\n\n\nclass ProviderLaunchError(_GenericProviderError):\n def __init__(\n self,\n *,\n provider_name: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n super().__init__(\n action=\"launch\",\n provider_name=provider_name,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderStartError(_GenericProviderError):\n def __init__(\n self,\n *,\n provider_name: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n super().__init__(\n action=\"start\",\n provider_name=provider_name,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderStopError(_GenericProviderError):\n def __init__(\n self,\n *,\n provider_name: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n super().__init__(\n action=\"stop\",\n provider_name=provider_name,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderDeleteError(_GenericProviderError):\n def __init__(\n self,\n *,\n provider_name: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n super().__init__(\n action=\"delete\",\n provider_name=provider_name,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderExecError(ProviderBaseError):\n\n fmt = (\n \"An error occurred when trying to execute {command_string!r} with \"\n \"{provider_name!r}: returned exit code {exit_code!r}.\"\n )\n\n def __init__(self, *, provider_name: str, command: Sequence[str], exit_code: int) -> None:\n command_string = \" \".join(shlex.quote(i) for i in command)\n super().__init__(\n provider_name=provider_name,\n command=command,\n command_string=command_string,\n exit_code=exit_code,\n )\n\n\nclass ProviderShellError(_GenericProviderError):\n def __init__(\n self,\n *,\n provider_name: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n super().__init__(\n action=\"shell\",\n provider_name=provider_name,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderMountError(_GenericProviderError):\n def __init__(\n self,\n *,\n provider_name: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n super().__init__(\n action=\"mount\",\n provider_name=provider_name,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderUnMountError(_GenericProviderError):\n def __init__(\n self,\n *,\n provider_name: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n super().__init__(\n action=\"unmount\",\n provider_name=provider_name,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderFileCopyError(_GenericProviderError):\n def __init__(\n self,\n *,\n provider_name: str,\n error_message: Optional[str] = None,\n exit_code: Optional[int] = None\n ) -> None:\n super().__init__(\n action=\"copy files\",\n provider_name=provider_name,\n error_message=error_message,\n exit_code=exit_code,\n )\n\n\nclass ProviderInfoError(ProviderBaseError):\n\n fmt = (\n \"An error occurred when using {provider_name!r} to \"\n \"query the status of the instance: returned exit code {exit_code!r}: {stderr!s}.\"\n )\n\n def __init__(self, *, provider_name: str, exit_code: int, stderr: bytes) -> None:\n super().__init__(provider_name=provider_name, exit_code=exit_code, stderr=stderr.decode())\n\n\nclass ProviderInstanceNotFoundError(ProviderBaseError):\n\n fmt = \"Cannot find an instance named {instance_name!r}.\"\n\n def __init__(self, *, instance_name: str) -> None:\n super().__init__(instance_name=instance_name)\n\n\nclass ProviderInfoDataKeyError(ProviderBaseError):\n\n fmt = (\n \"The data returned by {provider_name!r} was not expected. \"\n \"It is missing a required key {missing_key!r} in {data!r}.\"\n )\n\n def __init__(self, *, provider_name: str, missing_key: str, data: Dict[str, Any]) -> None:\n super().__init__(provider_name=provider_name, missing_key=missing_key, data=data)\n\n\nclass ProviderBadDataError(ProviderBaseError):\n\n fmt = (\n \"The data returned by {provider_name!r} was not expected \"\n \"or in the wrong format: {data!r}.\"\n )\n\n def __init__(self, *, provider_name: str, data: str) -> None:\n super().__init__(provider_name=provider_name, data=data)\n\n\nclass ProviderMultipassDownloadFailed(ProviderBaseError):\n fmt = (\n \"Failed to download Multipass: {message!r}\\n\"\n \"Please install manually. You can find the latest release at:\\n\"\n \"https://multipass.run\"\n )\n\n def __init__(self, message):\n super().__init__(message=message)\n\n\nclass ProviderMultipassInstallationFailed(ProviderBaseError):\n fmt = (\n \"Failed to install Multipass: {message!r}\\n\"\n \"Please install manually. You can find the latest release at:\\n\"\n \"https://multipass.run\"\n )\n\n def __init__(self, message):\n super().__init__(message=message)\n" }, { "path": "installer/vm_providers/factory.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2018 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nfrom typing import TYPE_CHECKING\n\nfrom . import errors\nfrom ._multipass import Multipass\n\nif TYPE_CHECKING:\n from typing import Type # noqa: F401\n from ._base_provider import Provider # noqa: F401\n\n\ndef get_provider_for(provider_name: str) -> \"Type[Provider]\":\n \"\"\"Returns a Type that can build with provider_name.\"\"\"\n if provider_name == \"multipass\":\n return Multipass\n else:\n raise errors.ProviderNotSupportedError(provider=provider_name)\n" }, { "path": "installer/vm_providers/repo/__init__.py", "content": "" }, { "path": "installer/vm_providers/repo/errors.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2015-2019 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nfrom typing import Sequence\nfrom typing import List\n\nfrom common.errors import BaseError\n\n\nclass RepoError(BaseError):\n pass\n\n\nclass NoNativeBackendError(RepoError):\n\n fmt = \"Native builds aren't supported on {distro}.\"\n\n\nclass CacheUpdateFailedError(RepoError):\n\n fmt = (\n \"Failed to update the package cache: \"\n \"Some files could not be downloaded:{errors}\"\n \"Check that the sources on your host are configured correctly.\"\n )\n\n def __init__(self, errors: str) -> None:\n if errors:\n errors = \"\\n\\n{}\\n\\n\".format(errors.replace(\", \", \"\\n\"))\n else:\n errors = \" \"\n super().__init__(errors=errors)\n\n\nclass FileProviderNotFound(RepoError):\n\n fmt = \"{file_path} is not provided by any package.\"\n\n def __init__(self, *, file_path: str) -> None:\n super().__init__(file_path=file_path)\n\n\nclass BuildPackageNotFoundError(RepoError):\n\n fmt = \"Could not find a required package in 'build-packages': {package}\"\n\n def __init__(self, package):\n super().__init__(package=package)\n\n\nclass BuildPackagesNotInstalledError(RepoError):\n\n fmt = \"Could not install all requested build packages: {packages}\"\n\n def __init__(self, *, packages: List[str]) -> None:\n super().__init__(packages=\" \".join(packages))\n\n\nclass PackageFetchError(RepoError):\n\n fmt = \"Package fetch error: {message}\"\n\n def __init__(self, message: str) -> None:\n super().__init__(message=message)\n\n\nclass PackageBrokenError(RepoError):\n\n fmt = \"The package {package} has unmet dependencies: {deps}\"\n\n def __init__(self, package: str, deps: List[str]) -> None:\n super().__init__(package=package, deps=\" \".join(deps))\n\n\nclass PackageNotFoundError(RepoError):\n @property\n def message(self):\n message = \"The package {!r} was not found.\".format(self.package_name)\n # If the package was multiarch, try to help.\n return message\n\n def __init__(self, package_name):\n self.package_name = package_name\n\n def __str__(self):\n return self.message\n\n\nclass UnpackError(RepoError):\n\n fmt = \"Error while provisioning {package!r}\"\n\n def __init__(self, package):\n super().__init__(package=package)\n\n\nclass SnapUnavailableError(RepoError):\n\n fmt = (\n \"Failed to install or refresh a snap: {snap_name!r} does not exist \"\n \"or is not available on the desired channel {snap_channel!r}. \"\n \"Use `snap info {snap_name}` to get a list of channels the \"\n \"snap is available on.\"\n )\n\n def __init__(self, *, snap_name: str, snap_channel: str) -> None:\n super().__init__(snap_name=snap_name, snap_channel=snap_channel)\n\n\nclass SnapFindError(RepoError):\n\n fmt = (\n \"Could not find the snap {snap_name!r} installed on this host.\\n\"\n \"Install the snap and try again.\"\n )\n\n def __init__(self, *, snap_name):\n super().__init__(snap_name=snap_name)\n\n\nclass SnapInstallError(RepoError):\n\n fmt = \"Error while installing snap {snap_name!r} from channel {snap_channel!r}\"\n\n def __init__(self, *, snap_name, snap_channel):\n super().__init__(snap_name=snap_name, snap_channel=snap_channel)\n\n\nclass SnapDownloadError(RepoError):\n\n fmt = \"Error while downloading snap {snap_name!r} from channel {snap_channel!r}\"\n\n def __init__(self, *, snap_name, snap_channel):\n super().__init__(snap_name=snap_name, snap_channel=snap_channel)\n\n\nclass SnapGetAssertionError(RepoError):\n\n fmt = (\n \"Error while retrieving assertion with parameters \"\n \"{assertion_params!r}\\n\"\n \"Verify the assertion exists and try again.\"\n )\n\n def __init__(self, *, assertion_params: Sequence[str]) -> None:\n super().__init__(assertion_params=assertion_params)\n\n\nclass SnapRefreshError(RepoError):\n\n fmt = \"Error while refreshing snap {snap_name!r} to channel {snap_channel!r}\"\n\n def __init__(self, *, snap_name, snap_channel):\n super().__init__(snap_name=snap_name, snap_channel=snap_channel)\n\n\nclass SnapdConnectionError(RepoError):\n\n fmt = \"Failed to get information for snap {snap_name!r}: \" \"could not connect to {url!r}.\"\n\n def __init__(self, snap_name: str, url: str) -> None:\n super().__init__(snap_name=snap_name, url=url)\n" }, { "path": "installer/vm_providers/repo/snaps.py", "content": "# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-\n#\n# Copyright (C) 2017-2019 Canonical Ltd\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License version 3 as\n# published by the Free Software Foundation.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n\nimport contextlib\nimport logging\nimport os\nimport sys\nfrom subprocess import check_call, check_output, CalledProcessError\nfrom typing import List, Sequence, Set, Union\nfrom urllib import parse\n\nimport requests_unixsocket\nfrom requests import exceptions\n\nfrom . import errors\n\n\n_STORE_ASSERTION = [\n \"account-key\",\n \"public-key-sha3-384=BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0hqUel3m8ul\",\n]\n\n_CHANNEL_RISKS = [\"stable\", \"candidate\", \"beta\", \"edge\"]\nlogger = logging.getLogger(__name__)\n\n\nclass SnapPackage:\n \"\"\"SnapPackage acts as a mediator to install or refresh a snap.\n\n It uses information provided by snapd implicitly referring to the local\n and remote stores to obtain information about the snap, such as its\n confinement value and channel availability.\n\n This information can also be used to determine if a snap should be\n installed or refreshed.\n\n There are risks of the data falling out of date between the query and the\n requested action given that it is not possible to hold a global lock on\n snapd and the store data can change in between validation and execution.\n \"\"\"\n\n @classmethod\n def is_valid_snap(cls, snap):\n return cls(snap).is_valid()\n\n @classmethod\n def is_snap_installed(cls, snap):\n return cls(snap).installed\n\n def __init__(self, snap):\n \"\"\"Lifecycle handler for a snap of the format /.\"\"\"\n self.name, self.channel = _get_parsed_snap(snap)\n self._original_channel = self.channel\n if not self.channel or self.channel == \"stable\":\n self.channel = \"latest/stable\"\n\n # This store information from a local request\n self._local_snap_info = None\n # And this stores information from a remote request\n self._store_snap_info = None\n\n self._is_installed = None\n self._is_in_store = None\n\n @property\n def installed(self):\n if self._is_installed is None:\n self._is_installed = self.get_local_snap_info() is not None\n return self._is_installed\n\n @property\n def in_store(self):\n if self._is_in_store is None:\n try:\n self._is_in_store = self.get_store_snap_info() is not None\n except errors.SnapUnavailableError:\n self._is_in_store = False\n return self._is_in_store\n\n def get_local_snap_info(self):\n \"\"\"Returns a local payload for the snap.\n\n Validity of the results are determined by checking self.installed.\"\"\"\n if self._is_installed is None:\n with contextlib.suppress(exceptions.HTTPError):\n self._local_snap_info = _get_local_snap_info(self.name)\n return self._local_snap_info\n\n def get_store_snap_info(self):\n \"\"\"Returns a store payload for the snap.\"\"\"\n if self._is_in_store is None:\n # Some environments timeout often, like the armv7 testing\n # infrastructure. Given that constraint, we add some retry\n # logic.\n retry_count = 5\n while retry_count > 0:\n try:\n self._store_snap_info = _get_store_snap_info(self.name)\n break\n except exceptions.HTTPError as http_error:\n logger.debug(\n \"The http error when checking the store for \"\n \"{!r} is {!r} (retries left {})\".format(\n self.name, http_error.response.status_code, retry_count\n )\n )\n if http_error.response.status_code == 404:\n raise errors.SnapUnavailableError(\n snap_name=self.name, snap_channel=self.channel\n )\n retry_count -= 1\n\n return self._store_snap_info\n\n def _get_store_channels(self):\n snap_store_info = self.get_store_snap_info()\n if not self.in_store:\n return dict()\n\n return snap_store_info[\"channels\"]\n\n def get_current_channel(self):\n current_channel = \"\"\n if self.installed:\n local_snap_info = self.get_local_snap_info()\n current_channel = local_snap_info[\"channel\"]\n if any([current_channel.startswith(risk) for risk in _CHANNEL_RISKS]):\n current_channel = \"latest/{}\".format(current_channel)\n return current_channel\n\n def has_assertions(self) -> bool:\n # A revision starting with x has been installed with\n # --dangerous.\n return not self.get_local_snap_info()[\"revision\"].startswith(\"x\")\n\n def is_classic(self) -> bool:\n store_channels = self._get_store_channels()\n try:\n return store_channels[self.channel][\"confinement\"] == \"classic\"\n except KeyError:\n # We have seen some KeyError issues when running tests that are\n # hard to debug as they only occur there, logging in debug mode\n # will help uncover the root cause if it happens again.\n logger.debug(\n \"Current store channels are {!r} and the store\"\n \"payload is {!r}\".format(store_channels, self._store_snap_info)\n )\n raise\n\n def is_valid(self) -> bool:\n \"\"\"Check if the snap is valid.\"\"\"\n if not self.in_store:\n return False\n store_channels = self._get_store_channels()\n return self.channel in store_channels.keys()\n\n def local_download(self, *, snap_path: str, assertion_path: str) -> None:\n assertions = list() # type: List[List[str]]\n # We write an empty assertions file for dangerous installs to\n # have a consistent interface.\n if self.has_assertions():\n assertions.append([\"snap-declaration\", \"snap-name={}\".format(self.name)])\n assertions.append(\n [\n \"snap-revision\",\n \"snap-revision={}\".format(self._local_snap_info[\"revision\"]),\n \"snap-id={}\".format(self._local_snap_info[\"id\"]),\n ]\n )\n\n if assertions:\n assertions.insert(0, _STORE_ASSERTION)\n\n with open(assertion_path, \"wb\") as assertion_file:\n for assertion in assertions:\n assertion_file.write(get_assertion(assertion))\n assertion_file.write(b\"\\n\")\n\n snap_file_iter = _get_local_snap_file_iter(self.name, chunk_size=1024)\n with open(snap_path, \"wb\") as snap_file:\n for buf in snap_file_iter:\n snap_file.write(buf)\n\n def download(self, *, directory: str = None):\n \"\"\"Downloads a given snap.\"\"\"\n # We use the `snap download` command here on recommendation\n # of the snapd team.\n snap_download_cmd = [\"snap\", \"download\", self.name]\n if self._original_channel:\n snap_download_cmd.extend([\"--channel\", self._original_channel])\n try:\n check_output(snap_download_cmd, cwd=directory)\n except CalledProcessError:\n raise errors.SnapDownloadError(snap_name=self.name, snap_channel=self.channel)\n\n def install(self):\n \"\"\"Installs the snap onto the system.\"\"\"\n if not self.is_valid():\n raise errors.SnapUnavailableError(snap_name=self.name, snap_channel=self.channel)\n\n snap_install_cmd = []\n if _snap_command_requires_sudo():\n snap_install_cmd = [\"sudo\"]\n snap_install_cmd.extend([\"snap\", \"install\", self.name])\n if self._original_channel:\n snap_install_cmd.extend([\"--channel\", self._original_channel])\n if self.is_classic():\n # TODO make this a user explicit choice\n snap_install_cmd.append(\"--classic\")\n try:\n check_call(snap_install_cmd)\n except CalledProcessError:\n raise errors.SnapInstallError(snap_name=self.name, snap_channel=self.channel)\n\n # Now that the snap is installed, invalidate the data we had on it.\n self._is_installed = None\n\n def refresh(self):\n \"\"\"Refreshes a snap onto a channel on the system.\"\"\"\n if not self.is_valid():\n raise errors.SnapUnavailableError(snap_name=self.name, snap_channel=self.channel)\n\n snap_refresh_cmd = []\n if _snap_command_requires_sudo():\n snap_refresh_cmd = [\"sudo\"]\n snap_refresh_cmd.extend([\"snap\", \"refresh\", self.name, \"--channel\", self.channel])\n if self.is_classic():\n # TODO make this a user explicit choice\n snap_refresh_cmd.append(\"--classic\")\n try:\n check_call(snap_refresh_cmd)\n except CalledProcessError:\n raise errors.SnapRefreshError(snap_name=self.name, snap_channel=self.channel)\n\n # Now that the snap is refreshed, invalidate the data we had on it.\n self._is_installed = None\n\n\ndef download_snaps(*, snaps_list: Sequence[str], directory: str) -> None:\n \"\"\"\n Download snaps of the format / into directory.\n\n The target directory is created if it does not exist.\n \"\"\"\n # TODO manifest.yaml with snap revision from future machine output\n # for `snap download`.\n os.makedirs(directory, exist_ok=True)\n for snap in snaps_list:\n snap_pkg = SnapPackage(snap)\n if not snap_pkg.is_valid():\n raise errors.SnapUnavailableError(\n snap_name=snap_pkg.name, snap_channel=snap_pkg.channel\n )\n\n # TODO: use dependency injected echoer\n logger.info(\"Downloading snap {!r}\".format(snap_pkg.name))\n snap_pkg.download(directory=directory)\n\n\ndef install_snaps(snaps_list: Union[Sequence[str], Set[str]]) -> List[str]:\n \"\"\"Install snaps of the format /.\n\n :return: a list of \"name=revision\" for the snaps installed.\n \"\"\"\n snaps_installed = []\n for snap in snaps_list:\n snap_pkg = SnapPackage(snap)\n\n # Allow bases to be installed from non stable channels.\n snap_pkg_channel = snap_pkg.get_store_snap_info()[\"channel\"]\n snap_pkg_type = snap_pkg.get_store_snap_info()[\"type\"]\n if snap_pkg_channel != \"stable\" and snap_pkg_type == \"base\":\n snap_pkg = SnapPackage(\n \"{snap_name}/latest/{channel}\".format(\n snap_name=snap_pkg.name, channel=snap_pkg_channel\n )\n )\n\n if not snap_pkg.installed:\n snap_pkg.install()\n elif snap_pkg.get_current_channel() != snap_pkg.channel:\n snap_pkg.refresh()\n\n snaps_installed.append(\n \"{}={}\".format(snap_pkg.name, snap_pkg.get_local_snap_info()[\"revision\"])\n )\n return snaps_installed\n\n\ndef _snap_command_requires_sudo():\n # snap whoami returns - if the user is not logged in.\n output = check_output([\"snap\", \"whoami\"])\n whoami = output.decode(sys.getfilesystemencoding())\n try:\n requires_root = whoami.split(\":\")[1].strip() == \"-\"\n # A safeguard if the output changes\n except IndexError:\n requires_root = True\n if requires_root:\n logger.warning(\"snapd is not logged in, snap install commands will use sudo\")\n return requires_root\n\n\ndef get_assertion(assertion_params: Sequence[str]) -> bytes:\n \"\"\"Get assertion information.\n\n :param assertion_params: a sequence of strings to pass to 'snap known'.\n :returns: a stream of bytes from the assertion.\n :rtype: bytes\n \"\"\"\n try:\n return check_output([\"snap\", \"known\", *assertion_params])\n except CalledProcessError as call_error:\n raise errors.SnapGetAssertionError(assertion_params=assertion_params) from call_error\n\n\ndef _get_parsed_snap(snap):\n if \"/\" in snap:\n sep_index = snap.find(\"/\")\n snap_name = snap[:sep_index]\n snap_channel = snap[sep_index + 1 :]\n else:\n snap_name = snap\n snap_channel = \"\"\n return snap_name, snap_channel\n\n\ndef get_snapd_socket_path_template():\n return \"http+unix://%2Frun%2Fsnapd.socket/v2/{}\"\n\n\ndef _get_local_snap_file_iter(snap_name, *, chunk_size: int):\n slug = \"snaps/{}/file\".format(parse.quote(snap_name, safe=\"\"))\n url = get_snapd_socket_path_template().format(slug)\n try:\n snap_file = requests_unixsocket.get(url)\n except exceptions.ConnectionError as e:\n raise errors.SnapdConnectionError(snap_name, url) from e\n snap_file.raise_for_status()\n return snap_file.iter_content(chunk_size)\n\n\ndef _get_local_snap_info(snap_name):\n slug = \"snaps/{}\".format(parse.quote(snap_name, safe=\"\"))\n url = get_snapd_socket_path_template().format(slug)\n try:\n snap_info = requests_unixsocket.get(url)\n except exceptions.ConnectionError as e:\n raise errors.SnapdConnectionError(snap_name, url) from e\n snap_info.raise_for_status()\n return snap_info.json()[\"result\"]\n\n\ndef _get_store_snap_info(snap_name):\n # This logic uses /v2/find returns an array of results, given that\n # we do a strict search either 1 result or a 404 will be returned.\n slug = \"find?{}\".format(parse.urlencode(dict(name=snap_name)))\n url = get_snapd_socket_path_template().format(slug)\n snap_info = requests_unixsocket.get(url)\n snap_info.raise_for_status()\n return snap_info.json()[\"result\"][0]\n\n\ndef get_installed_snaps():\n \"\"\"Return all the snaps installed in the system.\n\n :return: a list of \"name=revision\" for the snaps installed.\n \"\"\"\n slug = \"snaps\"\n url = get_snapd_socket_path_template().format(slug)\n try:\n snap_info = requests_unixsocket.get(url)\n snap_info.raise_for_status()\n local_snaps = snap_info.json()[\"result\"]\n except exceptions.ConnectionError:\n local_snaps = []\n return [\"{}={}\".format(snap[\"name\"], snap[\"revision\"]) for snap in local_snaps]\n" }, { "path": "installer/windows/README.md", "content": "# MicroK8s Installer\n\n## Windows\nUses the NSIS package to create the installer. \n\n### Prerequisites\n\n* Requires [NSIS 3](https://nsis.sourceforge.io/Download).\n* Requires the [EnVar plug in](https://nsis.sourceforge.io/EnVar_plug-in) to compile.\n\n### Building\n\n2 files must be placed in the working directory before compiling the installer. These are:\n\n* `microk8s.exe`: This is the file generated by PyInstaller.\n* `multipass.exe`: This is the Multipass installer exe, which is run at install time.\n\nUse the following command to compile the installer.\n\n```\nmakensis microk8s.nsi\n```\n" }, { "path": "installer/windows/microk8s.nsi", "content": "!include \"MUI2.nsh\"\n!include \"nsDialogs.nsh\"\n!include \"LogicLib.nsh\"\n!include \"Sections.nsh\"\n\n!define PRODUCT_NAME \"MicroK8s\"\n!define PRODUCT_VERSION \"2.3.3\"\n!define PRODUCT_PUBLISHER \"Canonical\"\n!define MUI_ICON \".\\microk8s.ico\"\n!define MUI_HEADERIMAGE\n!define MUI_HEADERIMAGE_BITMAP \".\\microk8s.bmp\"\n!define MUI_HEADERIMAGE_RIGHT\n\nUnicode True\nName \"${PRODUCT_NAME} for Windows ${PRODUCT_VERSION}\"\nBrandingText \"Canonical Ltd.\"\nIcon \".\\microk8s.ico\"\nOutFile \"microk8s-installer.exe\"\nInstallDir \"$PROGRAMFILES64\\${PRODUCT_NAME}\"\nShowInstDetails hide\nRequestExecutionLevel admin\nSetCompress auto\n\n!insertmacro MUI_PAGE_WELCOME\n!insertmacro MUI_PAGE_LICENSE \"..\\..\\LICENSE\"\n!define MUI_COMPONENTSPAGE_TEXT_COMPLIST \" \"\n!define MUI_COMPONENTSPAGE_TEXT_INSTTYPE \" \"\n!insertmacro MUI_PAGE_COMPONENTS\n!insertmacro MUI_PAGE_INSTFILES\nPage custom ConfigureVm LaunchVm\n!insertmacro MUI_PAGE_FINISH\n\n!insertmacro MUI_UNPAGE_CONFIRM\n!insertmacro MUI_UNPAGE_INSTFILES\n\n!insertmacro MUI_LANGUAGE \"English\"\n\nVar MultipassExitCode\nVar VmConfigureDialog\nVar VmConfigureDialogCpu\nVar VmConfigureDialogCpuLabel\nVar VmConfigureDialogMem\nVar VmConfigureDialogMemLabel\nVar VmConfigureDialogDisk\nVar VmConfigureDialogDiskLabel\nVar VmConfigureDialogTrack\nVar VmConfigureDialogTrackLabel\nVar VmConfigureDialogFootnote\n\nSection \"Multipass (Required)\" multipass_id\n;Exit 0 : Multipass is installed and backing hypervisor is ready to roll (e.g. HyperV is already enabled).\n;Exit 3010: Multipass is installed but backing hypervisor requires a reboot (e.g. HyperV just enabled).\n;Exit 3011: Multipass is installed but no backing hypervisor has been (e.g. virtualbox needs installing).\n SectionIn RO\n IfFileExists $PROGRAMFILES64\\Multipass\\bin\\multipass.exe endMultipass beginMultipass\n Goto endMultipass\n beginMultipass:\n SetOutPath $INSTDIR\n File \"multipass.exe\"\n ${If} ${Silent}\n ExecWait \"multipass.exe /NoRestart /S\" $0\n ${Else}\n ExecWait \"multipass.exe /NoRestart\" $0\n ${EndIf}\n StrCpy $MultipassExitCode $0\n IfErrors failedMultipass\n Delete \"$INSTDIR\\multipass.exe\"\n Goto endMultipass\n failedMultipass:\n Abort\n endMultipass:\nSectionEnd\n\nSection \"Kubectl (Required)\" kubectl_id\n SectionIn RO\n beginKubectl:\n SetOutPath $INSTDIR\n File \"kubectl.exe\"\n CopyFiles \"$INSTDIR\\kubectl.exe\" \"$INSTDIR\\kubectl\\kubectl.exe\"\n Delete \"$INSTDIR\\kubectl.exe\"\n Goto endKubectl\n endKubectl:\nSectionEnd\n\nSection -Install\n SectionIn RO\n SetOutPath $INSTDIR\n File \"microk8s.exe\"\nSectionEnd\n\nSection -WriteUninstaller\n SectionIn RO\n WriteUninstaller $INSTDIR\\uninstall.exe\n WriteRegStr HKLM \\\n \"Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\${PRODUCT_NAME}\" \\\n \"DisplayName\" \"${PRODUCT_NAME} ${PRODUCT_VERSION}\"\n WriteRegStr HKLM \\\n \"Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\${PRODUCT_NAME}\" \\\n \"UninstallString\" \"$INSTDIR\\uninstall.exe\"\nSectionEnd\n\nSection \"Add 'microk8s' to PATH\" add_to_path_id\n EnVar::AddValue \"path\" \"$INSTDIR\"\nSectionEnd\n\nSection /o \"Add 'kubectl' to PATH\" add_kubectl_to_path_id\n EnVar::AddValue \"path\" \"$INSTDIR\\kubectl\"\nSectionEnd\n\n!insertmacro MUI_FUNCTION_DESCRIPTION_BEGIN\n !insertmacro MUI_DESCRIPTION_TEXT ${multipass_id} \"REQUIRED: If already installed, will be unticked and skipped.$\\n$\\nSee https://multipass.run for more.\"\n !insertmacro MUI_DESCRIPTION_TEXT ${add_to_path_id} \"Add the 'microk8s' executable to PATH.$\\n$\\nThis will allow you to run the command 'microk8s' in cmd and PowerShell in any directory.\"\n !insertmacro MUI_DESCRIPTION_TEXT ${add_kubectl_to_path_id} \"Add the 'kubectl' executable to PATH.$\\n$\\nThis will set the bundled 'kubectl' as system default.\"\n!insertmacro MUI_FUNCTION_DESCRIPTION_END\n\nFunction .onInit\n DeleteRegKey HKLM \"Software\\canonical\\multipass\"\n IfFileExists $PROGRAMFILES64\\Multipass\\bin\\multipass.exe untickMultipass tickMultipass\n tickMultipass:\n IntOp $0 ${SF_SELECTED} | ${SF_RO}\n SectionSetFlags ${multipass_id} $0\n Return\n untickMultipass:\n StrCpy $MultipassExitCode \"0\" ; Multipass is installed.\n SectionSetFlags ${multipass_id} ${SF_RO}\n Return\nFunctionEnd\n\nFunction \"ConfigureVm\"\n ${IfNot} $MultipassExitCode == \"0\"\n ${If} $MultipassExitCode == \"3010\"\n MessageBox MB_ICONEXCLAMATION \"Cannot configure the ${PRODUCT_NAME} VM as a reboot is required. Please re-run this wizard after reboot to configure the VM.\"\n SetRebootFlag true\n Abort\n ${ElseIf} $MultipassExitCode == \"3011\"\n MessageBox MB_ICONEXCLAMATION \"Cannot configure the ${PRODUCT_NAME} VM as VirtualBox needs to be installed. Please re-run this wizard after installing VirtualBox to configure the VM.\"\n Abort\n ${EndIf}\n ${EndIf}\n MessageBox MB_YESNO \"Do you want to configure and launch the ${PRODUCT_NAME} VM now?\" /SD IDYES IDNO endLaunch\n nsDialogs::Create 1018\n Pop $VmConfigureDialog\n\n ${NSD_CreateLabel} 19% 0 35u 10u \"CPUs\"\n Pop $VmConfigureDialogCpuLabel\n\n ${NSD_CreateLabel} 44% 0 35u 10u \"Mem (GB)\"\n Pop $VmConfigureDialogMemLabel\n\n ${NSD_CreateLabel} 69% 0 35u 10u \"Disk (GB)\"\n Pop $VmConfigureDialogDiskLabel\n\n ${NSD_CreateNumber} 19% 17.5 35u 10u \"2\"\n Pop $VmConfigureDialogCpu\n\n ${NSD_CreateNumber} 44% 17.5 35u 10u \"4\"\n Pop $VmConfigureDialogMem\n\n ${NSD_CreateNumber} 69% 17.5 35u 10u \"50\"\n Pop $VmConfigureDialogDisk\n\n ${NSD_CreateLabel} 42% 50 50u 10u \"Snap Track\"\n Pop $VmConfigureDialogTrackLabel\n\n ${NSD_CreateText} 42% 67.5 50u 10u \"1.28/stable\"\n Pop $VmConfigureDialogTrack\n\n ${NSD_CreateLabel} 8% 102.5 100% 10u \"These are the minimum recommended parameters for the VM running ${PRODUCT_NAME}\"\n Pop $VmConfigureDialogFootnote\n\n nsDialogs::Show\n endLaunch:\n Abort\nFunctionEnd\n\nFunction \"LaunchVM\"\n ${NSD_GetText} $VmConfigureDialogCpu $0\n ${NSD_GetText} $VmConfigureDialogMem $1\n ${NSD_GetText} $VmConfigureDialogDisk $2\n ${NSD_GetText} $VmConfigureDialogTrack $3\n\n ExecWait \"$INSTDIR\\microk8s.exe install --cpu $0 --mem $1 --disk $2 --channel $3 --assume-yes\"\nFunctionEnd\n\nFunction un.onInit\n DeleteRegKey HKLM \"Software\\canonical\\multipass\"\nFunctionEnd\n\nSection \"Uninstall\"\n ExecWait \"$INSTDIR\\microk8s.exe uninstall\"\n Delete $INSTDIR\\uninstall.exe\n Delete $INSTDIR\\microk8s.exe\n Delete $INSTDIR\\kubectl\\kubectl.exe\n RMDir $INSTDIR\\kubectl\n RMDir $INSTDIR\n\n DeleteRegKey HKLM \\\n \"Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\${PRODUCT_NAME}\"\n\n EnVar::DeleteValue \"path\" \"$INSTDIR\"\n EnVar::DeleteValue \"path\" \"$INSTDIR\\kubectl\"\nSectionEnd\n" }, { "path": "microk8s-resources/actions/common/utils.sh", "content": "#!/usr/bin/env bash\n\nget_microk8s_group() {\n if is_strict\n then\n echo \"snap_microk8s\"\n else\n echo \"microk8s\"\n fi\n}\n\nget_microk8s_or_cis_group() {\n if [ -e $SNAP_DATA/var/lock/cis-hardening ]\n then\n echo \"root\"\n else\n get_microk8s_group\n fi\n}\n\n\nexit_if_no_permissions() {\n # test if we can access the default kubeconfig\n if [ ! -r $SNAP_DATA/credentials/client.config ]; then\n local group=$(get_microk8s_group)\n\n echo \"Insufficient permissions to access MicroK8s.\" >&2\n echo \"You can either try again with sudo or add the user $USER to the '${group}' group:\" >&2\n echo \"\" >&2\n echo \" sudo usermod -a -G ${group} $USER\" >&2\n echo \" sudo chown -R $USER ~/.kube\" >&2\n echo \"\" >&2\n echo \"After this, reload the user groups either via a reboot or by running 'newgrp ${group}'.\" >&2\n exit 1\n fi\n}\n\nexit_if_stopped() {\n # test if the snap is marked as stopped\n if [ -e ${SNAP_DATA}/var/lock/stopped.lock ]\n then\n echo \"microk8s is not running, try microk8s start\" >&2\n exit 1\n fi\n}\n\nexit_if_service_not_expected_to_start() {\n # exit if a lock is available for the service\n local service=\"$1\"\n if [ -f ${SNAP_DATA}/var/lock/no-${service} ]\n then\n exit 0\n fi\n}\n\nis_service_expected_to_start() {\n # return 1 if service is expected to start\n local service=\"$1\"\n if [ -f ${SNAP_DATA}/var/lock/no-${service} ]\n then\n echo \"0\"\n else\n echo \"1\"\n fi\n}\n\nset_service_not_expected_to_start() {\n # mark service as not starting\n local service=\"$1\"\n run_with_sudo $SNAP/bin/touch ${SNAP_DATA}/var/lock/no-${service}\n}\n\nset_service_expected_to_start() {\n # mark service as not starting\n local service=\"$1\"\n rm -rf ${SNAP_DATA}/var/lock/no-${service}\n}\n\nremove_vxlan_interfaces() {\n links=\"$(${SNAP}/sbin/ip link show type vxlan | $SNAP/bin/grep -E 'flannel|cilium_vxlan' | $SNAP/usr/bin/gawk '{print $2}' | $SNAP/usr/bin/tr -d :)\"\n for link in $links\n do\n if ! [ -z \"$link\" ] && $SNAP/sbin/ip link show ${link} &> /dev/null\n then\n echo \"Deleting old ${link} link\" >&2\n run_with_sudo $SNAP/sbin/ip link delete ${link}\n fi\n done\n}\n\nlist_env_vars() {\n env | awk -F= '{print $1}' | paste -sd,\n}\n\nrun_with_sudo() {\n if [ \"$1\" == \"preserve_env\" ]; then\n shift\n fi\n if (is_strict); then\n \"$@\"\n else\n local SAVE_LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}\"\n LD_LIBRARY_PATH=\"\" sudo --preserve-env=\"$(list_env_vars)\" PATH=\"${PATH}\" LD_LIBRARY_PATH=\"${SAVE_LD_LIBRARY_PATH}\" PYTHONPATH=\"${PYTHONPATH:-}\" \"$@\"\n fi\n}\n\nget_opt_in_config() {\n # return the value of an option in a configuration file or \"\"\n local opt=\"$1\"\n local config_file=\"$SNAP_DATA/args/$2\"\n val=\"\"\n if $($SNAP/bin/grep -qE \"^$opt=\" $config_file); then\n val=\"$($SNAP/bin/grep -E \"^$opt\" \"$config_file\" | $SNAP/usr/bin/cut -d'=' -f2)\"\n elif $($SNAP/bin/grep -qE \"^$opt \" $config_file); then\n val=\"$($SNAP/bin/grep -E \"^$opt\" \"$config_file\" | $SNAP/usr/bin/cut -d' ' -f2)\"\n fi\n echo \"$val\"\n}\n\nrefresh_opt_in_local_config() {\n # add or replace an option inside the local config file.\n # Create the file if doesn't exist\n local opt=\"--$1\"\n local value=\"$2\"\n local config_file=\"$SNAP_DATA/args/$3\"\n local replace_line=\"$opt=$value\"\n if $($SNAP/bin/grep -qE \"^$opt=\" $config_file); then\n run_with_sudo \"$SNAP/bin/sed\" -i \"s@^$opt=.*@$replace_line@\" $config_file\n else\n run_with_sudo \"$SNAP/bin/sed\" -i \"1i$replace_line\" \"$config_file\"\n fi\n}\n\nrefresh_opt_in_config() {\n # add or replace an option inside the config file and propagate change.\n # Create the file if doesn't exist\n refresh_opt_in_local_config \"$1\" \"$2\" \"$3\"\n\n local opt=\"--$1\"\n local value=\"$2\"\n local config_file=\"$SNAP_DATA/args/$3\"\n local replace_line=\"$opt=$value\"\n\n if [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ]\n then\n run_with_sudo preserve_env \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/wrappers/distributed_op.py\" update_argument \"$3\" \"$opt\" \"$value\"\n fi\n\n if [ -e \"${SNAP_DATA}/credentials/callback-tokens.txt\" ]\n then\n tokens=$(run_with_sudo \"$SNAP/bin/cat\" \"${SNAP_DATA}/credentials/callback-tokens.txt\" | \"$SNAP/usr/bin/wc\" -l)\n if [[ \"$tokens\" -ge \"0\" ]]\n then\n run_with_sudo preserve_env \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/wrappers/distributed_op.py\" update_argument \"$3\" \"$opt\" \"$value\"\n fi\n fi\n}\n\n\nnodes_addon() {\n # Enable or disable a, addon across all nodes\n # state should be either 'enable' or 'disable'\n local addon=\"$1\"\n local state=\"$2\"\n\n if [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ]\n then\n run_with_sudo preserve_env \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/wrappers/distributed_op.py\" set_addon \"$addon\" \"$state\"\n fi\n\n if [ -e \"${SNAP_DATA}/credentials/callback-tokens.txt\" ]\n then\n tokens=$(run_with_sudo \"$SNAP/bin/cat\" \"${SNAP_DATA}/credentials/callback-tokens.txt\" | \"$SNAP/usr/bin/wc\" -l)\n if [[ \"$tokens\" -ge \"0\" ]]\n then\n run_with_sudo preserve_env \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/wrappers/distributed_op.py\" set_addon \"$addon\" \"$state\"\n fi\n fi\n}\n\n\nskip_opt_in_local_config() {\n # remove an option inside the config file.\n # argument $1 is the option to be removed\n # argument $2 is the configuration file under $SNAP_DATA/args\n local opt=\"--$1\"\n local config_file=\"$SNAP_DATA/args/$2\"\n\n # regex is \"$opt[= ]\", otherwise we remove all arguments with the same prefix\n run_with_sudo \"${SNAP}/bin/sed\" -i '/'\"$opt[= ]\"'/d' \"${config_file}\"\n}\n\n\nskip_opt_in_config() {\n # remove an option inside the config file.\n # argument $1 is the option to be removed\n # argument $2 is the configuration file under $SNAP_DATA/args\n skip_opt_in_local_config \"$1\" \"$2\"\n\n local opt=\"--$1\"\n\n if [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ]\n then\n run_with_sudo preserve_env \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/wrappers/distributed_op.py\" remove_argument \"$2\" \"$opt\"\n fi\n\n if [ -e \"${SNAP_DATA}/credentials/callback-tokens.txt\" ]\n then\n tokens=$(run_with_sudo \"$SNAP/bin/cat\" \"${SNAP_DATA}/credentials/callback-tokens.txt\" | \"$SNAP/usr/bin/wc\" -l)\n if [[ \"$tokens\" -ge \"0\" ]]\n then\n run_with_sudo preserve_env \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/wrappers/distributed_op.py\" remove_argument \"$2\" \"$opt\"\n fi\n fi\n}\n\n\nremove_args() {\n # Removes arguments from respective service\n # argument $1: the service\n # rest of arguments: the arguments to be removed\n local service_name=\"$1\"\n shift\n local args=(\"$@\")\n for arg in \"${args[@]}\"; do\n if $SNAP/bin/grep -q \"$arg\" \"$SNAP_DATA/args/$service_name\"; then\n echo \"Removing argument: $arg from $service_name\"\n skip_opt_in_local_config \"$arg\" \"$service_name\"\n fi\n done\n}\n\n\nsanitise_args_kubeapi_server() {\n # Function to sanitise arguments for API server\n local args=(\n # Removed klog flags from 1.26+\n # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md\n \"log-dir\"\n \"log-file\"\n \"log-flush-frequency\"\n \"logtostderr\"\n \"alsologtostderr\"\n \"one-output\"\n \"stderrthreshold\"\n \"log-file-max-size\"\n \"skip-log-headers\"\n \"add-dir-header\"\n \"skip-headers\"\n \"log-backtrace-at\"\n # Remove insecure-port from 1.24+\n \"insecure-port\"\n \"insecure-bind-address\"\n \"port\"\n \"address\"\n # Remove service-account-api-audiences from 1.25+\n # https://github.com/kubernetes/kubernetes/commit/92707cafbb67a5664324eb891ef70ab3d1dd4a97\n \"service-account-api-audiences\"\n # extra\n \"feature-gates=RemoveSelfLink\"\n \"experimental-encryption-provider-config\"\n \"target-ram-mb\"\n )\n\n remove_args \"kube-apiserver\" \"${args[@]}\"\n}\n\n\nsanitise_args_kubelet() {\n # Function to sanitise arguments for kubelet\n local args=(\n # Removed klog flags from 1.26+\n # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md\n \"log-dir\"\n \"log-file\"\n \"log-flush-frequency\"\n \"logtostderr\"\n \"alsologtostderr\"\n \"one-output\"\n \"stderrthreshold\"\n \"log-file-max-size\"\n \"skip-log-headers\"\n \"add-dir-header\"\n \"skip-headers\"\n \"log-backtrace-at\"\n # Removed dockershim flags from 1.24+\n # https://github.com/kubernetes/enhancements/issues/2221\n \"docker-endpoint\"\n \"image-pull-progress-deadline\"\n \"network-plugin\"\n \"cni-conf-dir\"\n \"cni-bin-dir\"\n \"cni-cache-dir\"\n \"network-plugin-mtu\"\n # extra\n \"experimental-kernel-memcg-notification\"\n \"pod-infra-container-image\"\n \"experimental-dockershim-root-directory\"\n \"non-masquerade-cidr\"\n # Remove container-runtime flag from 1.27+\n \"container-runtime\"\n )\n\n remove_args \"kubelet\" \"${args[@]}\"\n\n # Remove 'DevicePlugins=true' from feature-gates from 1.28+\n $SNAP/bin/sed -i 's,DevicePlugins=true,,' \"$SNAP_DATA/args/kubelet\"\n}\n\n\nsanitise_args_kube_proxy() {\n # Function to sanitise arguments for kube-proxy\n\n # userspace proxy-mode is not allowed on the 1.26+ k8s\n # https://kubernetes.io/blog/2022/11/18/upcoming-changes-in-kubernetes-1-26/#removal-of-kube-proxy-userspace-modes\n if $SNAP/bin/grep -- \"--proxy-mode=userspace\" $SNAP_DATA/args/kube-proxy\n then\n echo \"Removing --proxy-mode=userspace flag from kube-proxy, since it breaks Calico.\"\n skip_opt_in_local_config \"proxy-mode\" \"kube-proxy\"\n fi\n\n local args=(\n # Removed klog flags from 1.26+\n # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md\n \"log-dir\"\n \"log-file\"\n \"log-flush-frequency\"\n \"logtostderr\"\n \"alsologtostderr\"\n \"one-output\"\n \"stderrthreshold\"\n \"log-file-max-size\"\n \"skip-log-headers\"\n \"add-dir-header\"\n \"skip-headers\"\n \"log-backtrace-at\"\n )\n\n remove_args \"kube-proxy\" \"${args[@]}\"\n}\n\n\nsanitise_args_kube_controller_manager() {\n # Function to sanitise arguments for kube-controller-manager\n local args=(\n # Removed klog flags from 1.26+\n # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md\n \"log-dir\"\n \"log-file\"\n \"log-flush-frequency\"\n \"logtostderr\"\n \"alsologtostderr\"\n \"one-output\"\n \"stderrthreshold\"\n \"log-file-max-size\"\n \"skip-log-headers\"\n \"add-dir-header\"\n \"skip-headers\"\n \"log-backtrace-at\"\n # Remove insecure ports from 1.24+\n # https://github.com/kubernetes/kubernetes/pull/96216/files\n \"address\"\n \"port\"\n # extra\n \"experimental-cluster-signing-duration\"\n )\n\n remove_args \"kube-controller-manager\" \"${args[@]}\"\n}\n\n\nsanitise_args_kube_scheduler() {\n # Function to sanitise arguments for kube-scheduler\n local args=(\n # Removed klog flags from 1.26+\n # https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/2845-deprecate-klog-specific-flags-in-k8s-components/README.md\n \"log-dir\"\n \"log-file\"\n \"log-flush-frequency\"\n \"logtostderr\"\n \"alsologtostderr\"\n \"one-output\"\n \"stderrthreshold\"\n \"log-file-max-size\"\n \"skip-log-headers\"\n \"add-dir-header\"\n \"skip-headers\"\n \"log-backtrace-at\"\n # Remove insecure ports from 1.24+\n # https://github.com/kubernetes/kubernetes/pull/96345/files\n \"address\"\n \"port\"\n )\n\n remove_args \"kube-scheduler\" \"${args[@]}\"\n}\n\n\nrestart_service() {\n # restart a systemd service\n # argument $1 is the service name\n\n if [ \"$1\" == \"apiserver\" ] || [ \"$1\" == \"proxy\" ] || [ \"$1\" == \"kubelet\" ] || [ \"$1\" == \"scheduler\" ] || [ \"$1\" == \"controller-manager\" ]\n then\n run_with_sudo preserve_env snapctl restart \"microk8s.daemon-kubelite\"\n else\n run_with_sudo preserve_env snapctl restart \"microk8s.daemon-$1\"\n fi\n\n if [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ]\n then\n run_with_sudo preserve_env \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/wrappers/distributed_op.py\" restart \"$1\"\n fi\n\n if [ -e \"${SNAP_DATA}/credentials/callback-tokens.txt\" ]\n then\n tokens=$(run_with_sudo \"$SNAP/bin/cat\" \"${SNAP_DATA}/credentials/callback-tokens.txt\" | \"$SNAP/usr/bin/wc\" -l)\n if [[ \"$tokens\" -ge \"0\" ]]\n then\n run_with_sudo preserve_env \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/wrappers/distributed_op.py\" restart \"$1\"\n fi\n fi\n}\n\n\narch() {\n case \"${SNAP_ARCH}\" in\n ppc64el) echo ppc64le ;;\n *) echo \"${SNAP_ARCH}\" ;;\n esac\n}\n\n\nsnapshotter() {\n # Determine the underlying filesystem that containerd will be running on\n FSTYPE=$($SNAP/usr/bin/stat -f -c %T \"${SNAP_COMMON}\")\n # ZFS is supported through the native snapshotter\n if [ \"$FSTYPE\" = \"zfs\" ]; then\n echo \"native\"\n else\n echo \"overlayfs\"\n fi\n}\n\nuse_manifest() {\n # Perform an action (apply or delete) on a manifest.\n # Optionally replace strings in the manifest\n #\n # Parameters:\n # $1 the name of the manifest. Should be ${SNAP}/actions/ and should not\n # include the trailing .yaml eg ingress, dns\n # $2 the action to be performed on the manifest, eg apply, delete\n # $3 (optional) an associative array with keys the string to be replaced and value what to\n # replace with. The string $ARCH is always injected to this array.\n #\n local manifest=\"$1.yaml\"; shift\n local action=\"$1\"; shift\n if ! [ \"$#\" = \"0\" ]\n then\n eval \"declare -A items=\"${1#*=}\n else\n declare -A items\n fi\n local tmp_manifest=\"${SNAP_USER_DATA}/tmp/temp.yaml\"\n items[\\$ARCH]=$(arch)\n\n $SNAP/bin/mkdir -p ${SNAP_USER_DATA}/tmp\n $SNAP/bin/cp \"${SNAP}/addons/core/addons/${manifest}\" \"${tmp_manifest}\"\n for i in \"${!items[@]}\"\n do\n \"$SNAP/bin/sed\" -i 's@'$i'@'\"${items[$i]}\"'@g' \"${tmp_manifest}\"\n done\n \"$SNAP/kubectl\" \"--kubeconfig=$SNAP_DATA/credentials/client.config\" \"$action\" -f \"${tmp_manifest}\"\n use_manifest_result=\"$?\"\n $SNAP/bin/rm \"${tmp_manifest}\"\n}\n\naddon_name() {\n # Extracts the addon from the argument.\n # addons can have arguments in the form of :=;=\n # Example: enable linkerd:proxy-auto-inject=on;other-args=xyz\n # Parameter:\n # $1 the full addon command\n # Returns:\n # \n\n local IFS=':'\n read -ra ADD_ON <<< \"$1\"\n echo \"${ADD_ON[0]}\"\n}\n\naddon_arguments() {\n # Extracts the addon arguments.\n # Example: enable linkerd:proxy-auto-inject=on;other-args=xyz\n # Parameter:\n # $1 the addon arguments in array\n # Returns:\n # add-on arguments array\n local IFS=':'\n read -ra ADD_ON <<< \"$1\"\n local IFS=';'\n read -ra ARGUMENTS <<< \"${ADD_ON[1]}\"\n echo \"${ARGUMENTS[@]}\"\n}\n\nwait_for_service() {\n # Wait for a service to start\n # Return fail if the service did not start in 30 seconds\n local service_name=\"$1\"\n if [ \"$1\" == \"apiserver\" ] || [ \"$1\" == \"proxy\" ] || [ \"$1\" == \"kubelet\" ] || [ \"$1\" == \"scheduler\" ] || [ \"$1\" == \"controller-manager\" ]\n then\n if [ -e \"${SNAP_DATA}/var/lock/lite.lock\" ]\n then\n service_name=\"kubelite\"\n fi\n fi\n\n local TRY_ATTEMPT=0\n while ! (run_with_sudo preserve_env snapctl services ${SNAP_NAME}.daemon-${service_name} | grep active) &&\n ! [ ${TRY_ATTEMPT} -eq 30 ]\n do\n TRY_ATTEMPT=$((TRY_ATTEMPT+1))\n sleep 1\n done\n if [ ${TRY_ATTEMPT} -eq 30 ]\n then\n echo \"fail\"\n fi\n}\n\nwait_for_service_shutdown() {\n # Wait for a service to stop\n # Return fail if the service did not stop in 30 seconds\n\n local namespace=\"$1\"\n local labels=\"$2\"\n local shutdown_timeout=30\n local start_timer=\"$($SNAP/bin/date +%s)\"\n KUBECTL=\"$SNAP/kubectl --kubeconfig=$SNAP/client.config\"\n\n while ($KUBECTL get po -n \"$namespace\" -l \"$labels\" | $SNAP/bin/grep -z \" Terminating\") &> /dev/null\n do\n now=\"$($SNAP/bin/date +%s)\"\n if [[ \"$now\" > \"$(($start_timer + $shutdown_timeout))\" ]] ; then\n echo \"fail\"\n break\n fi\n sleep 5\n done\n}\n\nget_default_ip() {\n # Get the IP of the default interface\n local DEFAULT_INTERFACE=\"$($SNAP/sbin/ip route show default | $SNAP/usr/bin/gawk '{for(i=1; i/dev/null | $SNAP/bin/grep -v 'inet 169.254' | $SNAP/usr/bin/gawk '{print $4}' | $SNAP/usr/bin/cut -d/ -f1 | head -1)\"\n CNI_IPS=\"/$CNI_IP/$CNI_IPS\"\n done\n\n if [[ -z \"$IP_ADDR\" ]]; then\n echo \"none\"\n else\n local ips=\"\";\n for ip in $IP_ADDR; do\n # Append IP address only iff not in cni IP addresses\n (echo \"$CNI_IPS\" | $SNAP/bin/grep -q \"/$ip/\") || ips+=\"${ips:+ }$ip\";\n done\n echo \"$ips\"\n fi\n}\n\ngen_server_cert() (\n \"${SNAP}/openssl.wrapper\" req -new -sha256 -key ${SNAP_DATA}/certs/server.key -out ${SNAP_DATA}/certs/server.csr -config ${SNAP_DATA}/certs/csr.conf\n \"${SNAP}/openssl.wrapper\" x509 -req -sha256 -in ${SNAP_DATA}/certs/server.csr -CA ${SNAP_DATA}/certs/ca.crt -CAkey ${SNAP_DATA}/certs/ca.key -CAcreateserial -out ${SNAP_DATA}/certs/server.crt -days 365 -extensions v3_ext -extfile ${SNAP_DATA}/certs/csr.conf\n)\n\ngen_proxy_client_cert() (\n \"${SNAP}/openssl.wrapper\" req -new -sha256 -key ${SNAP_DATA}/certs/front-proxy-client.key -out ${SNAP_DATA}/certs/front-proxy-client.csr -config <(sed '/^prompt = no/d' ${SNAP_DATA}/certs/csr.conf) -subj \"/CN=front-proxy-client\"\n \"${SNAP}/openssl.wrapper\" x509 -req -sha256 -in ${SNAP_DATA}/certs/front-proxy-client.csr -CA ${SNAP_DATA}/certs/front-proxy-ca.crt -CAkey ${SNAP_DATA}/certs/front-proxy-ca.key -CAcreateserial -out ${SNAP_DATA}/certs/front-proxy-client.crt -days 365 -extensions v3_ext -extfile ${SNAP_DATA}/certs/csr.conf\n)\n\ncreate_user_certs_and_configs() {\n create_user_certificates\n create_user_kubeconfigs\n}\n\ncreate_user_certificates() {\n hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]')\n generate_csr_with_sans \"/CN=system:node:$hostname/O=system:nodes\" \"${SNAP_DATA}/certs/kubelet.key\" | sign_certificate > \"${SNAP_DATA}/certs/kubelet.crt\"\n generate_csr /CN=admin/O=system:masters \"${SNAP_DATA}/certs/client.key\" | sign_certificate > \"${SNAP_DATA}/certs/client.crt\"\n generate_csr /CN=system:kube-proxy \"${SNAP_DATA}/certs/proxy.key\" | sign_certificate > \"${SNAP_DATA}/certs/proxy.crt\"\n generate_csr /CN=system:kube-scheduler \"${SNAP_DATA}/certs/scheduler.key\" | sign_certificate > \"${SNAP_DATA}/certs/scheduler.crt\"\n generate_csr /CN=system:kube-controller-manager \"${SNAP_DATA}/certs/controller.key\" | sign_certificate > \"${SNAP_DATA}/certs/controller.crt\"\n generate_csr /CN=kube-apiserver-kubelet-client/O=system:masters \"${SNAP_DATA}/certs/apiserver-kubelet-client.key\" | sign_certificate > \"${SNAP_DATA}/certs/apiserver-kubelet-client.crt\"\n}\n\ncreate_user_kubeconfigs() {\n hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]')\n create_kubeconfig_x509 \"client.config\" \"admin\" ${SNAP_DATA}/certs/client.crt ${SNAP_DATA}/certs/client.key ${SNAP_DATA}/certs/ca.crt\n create_kubeconfig_x509 \"controller.config\" \"system:kube-controller-manager\" ${SNAP_DATA}/certs/controller.crt ${SNAP_DATA}/certs/controller.key ${SNAP_DATA}/certs/ca.crt\n create_kubeconfig_x509 \"scheduler.config\" \"system:kube-scheduler\" ${SNAP_DATA}/certs/scheduler.crt ${SNAP_DATA}/certs/scheduler.key ${SNAP_DATA}/certs/ca.crt\n create_kubeconfig_x509 \"proxy.config\" \"system:kube-proxy\" ${SNAP_DATA}/certs/proxy.crt ${SNAP_DATA}/certs/proxy.key ${SNAP_DATA}/certs/ca.crt\n create_kubeconfig_x509 \"kubelet.config\" \"system:node:${hostname}\" ${SNAP_DATA}/certs/kubelet.crt ${SNAP_DATA}/certs/kubelet.key ${SNAP_DATA}/certs/ca.crt\n}\n\ncreate_worker_kubeconfigs() {\n # $1: (Optional) API server IP, defaults to IP address from traefik-template.yaml (or empty)\n # $2: (Optional) API server port, defaults to port from traefik-template.yaml\n\n # NOTE(neoaggelos): Examples for how \"${var%:*}\" and ${var##*:} below behave\n # apiserver_proxy_listen=\":16443\" => apiserver_default=\"\" port_default=\"16443\"\n # apiserver_proxy_listen=\"1.1.1.1:6443\" => apiserver_default=\"1.1.1.1\" port_default=\"6443\"\n # apiserver_proxy_listen=\"[::1]:16443\" => apiserver_default=\"[::1]\" port_default=\"16443\"\n # apiserver_proxy_listen=\"\" => apiserver_default=\"\" port_default=\"\"\n\n local apiserver_proxy_listen=\"$($SNAP/bin/cat $SNAP_DATA/args/traefik/traefik-template.yaml | $SNAP/bin/grep \"address:\" | $SNAP/usr/bin/cut -d'\"' -f2)\"\n local apiserver_default=\"${apiserver_proxy_listen%:*}\" # drop last ':' and everything after\n local port_default=\"${apiserver_proxy_listen##*:}\" # drop last ':' and everything before\n\n local apiserver=\"${1:-$apiserver_default}\"\n local port=\"${2:-$port_default}\"\n\n hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]')\n create_kubeconfig_x509 \"proxy.config\" \"system:kube-proxy\" ${SNAP_DATA}/certs/proxy.crt ${SNAP_DATA}/certs/proxy.key ${SNAP_DATA}/certs/ca.remote.crt \"${apiserver}\" \"${port}\"\n create_kubeconfig_x509 \"kubelet.config\" \"system:node:${hostname}\" ${SNAP_DATA}/certs/kubelet.crt ${SNAP_DATA}/certs/kubelet.key ${SNAP_DATA}/certs/ca.remote.crt \"${apiserver}\" \"${port}\"\n}\n\ncreate_kubeconfig_x509() {\n # Create a kubeconfig file with x509 auth\n # $1: the name of the config file\n # $2: the user to use al login\n # $3: path to certificate file\n # $4: path to certificate key file\n # $5: path to ca file\n # $6: (optional) API server IP, optional defaults to 127.0.0.1\n # $7: (optional) API server port, optional defaults to the port of the current node\n\n kubeconfig=$1\n user=$2\n cert=$3\n key=$4\n ca=$5\n\n # optional arguments\n apiserver=\"${6:-\"127.0.0.1\"}\"\n apiserver_port=\"$($SNAP/bin/cat $SNAP_DATA/args/kube-apiserver | $SNAP/bin/grep -- \"--secure-port\" | $SNAP/usr/bin/tr \"=\" \" \" | $SNAP/usr/bin/gawk '{print $2}')\"\n port=\"${7:-${apiserver_port}}\"\n\n ca_data=$($SNAP/bin/cat ${ca} | ${SNAP}/usr/bin/base64 -w 0)\n cert_data=$($SNAP/bin/cat ${cert} | ${SNAP}/usr/bin/base64 -w 0)\n key_data=$($SNAP/bin/cat ${key} | ${SNAP}/usr/bin/base64 -w 0)\n config_file=\"${SNAP_DATA}/credentials/${kubeconfig}\"\n\n $SNAP/bin/cp \"${SNAP}/client-x509.config.template\" \"${config_file}\"\n $SNAP/bin/sed -i 's/CADATA/'\"${ca_data}\"'/g' \"${config_file}\"\n $SNAP/bin/sed -i 's/NAME/'\"${user}\"'/g' \"${config_file}\"\n $SNAP/bin/sed -i 's/PATHTOCERT/'\"${cert_data}\"'/g' \"${config_file}\"\n $SNAP/bin/sed -i 's/PATHTOKEYCERT/'\"${key_data}\"'/g' \"${config_file}\"\n $SNAP/bin/sed -i 's/client-certificate/client-certificate-data/g' \"${config_file}\"\n $SNAP/bin/sed -i 's/client-key/client-key-data/g' \"${config_file}\"\n $SNAP/bin/sed -i 's/127.0.0.1/'\"${apiserver}\"'/g' \"${config_file}\"\n $SNAP/bin/sed -i 's/16443/'\"${port}\"'/g' \"${config_file}\"\n}\n\nproduce_certs() {\n # Generate RSA keys if not yet\n for key in serviceaccount.key ca.key server.key front-proxy-ca.key front-proxy-client.key; do\n if ! [ -f ${SNAP_DATA}/certs/$key ]; then\n \"${SNAP}/openssl.wrapper\" genrsa -out ${SNAP_DATA}/certs/$key 2048\n fi\n done\n\n # Generate apiserver CA\n if ! [ -f ${SNAP_DATA}/certs/ca.crt ]; then\n \"${SNAP}/openssl.wrapper\" req -x509 -new -sha256 -nodes -days 3650 -key ${SNAP_DATA}/certs/ca.key -subj \"/CN=10.152.183.1\" -out ${SNAP_DATA}/certs/ca.crt\n fi\n\n # Generate front proxy CA\n if ! [ -f ${SNAP_DATA}/certs/front-proxy-ca.crt ]; then\n \"${SNAP}/openssl.wrapper\" req -x509 -new -sha256 -nodes -days 3650 -key ${SNAP_DATA}/certs/front-proxy-ca.key -subj \"/CN=front-proxy-ca\" -out ${SNAP_DATA}/certs/front-proxy-ca.crt\n fi\n\n # Produce certificates based on the rendered csr.conf.rendered.\n # The file csr.conf.rendered is compared with csr.conf to determine if a regeneration of the certs must be done.\n #\n # Returns\n # 0 if no change\n # 1 otherwise.\n\n render_csr_conf\n if ! [ -f \"${SNAP_DATA}/certs/csr.conf\" ]; then\n echo \"changeme\" > \"${SNAP_DATA}/certs/csr.conf\"\n fi\n\n local force\n if ! \"${SNAP}/usr/bin/cmp\" -s \"${SNAP_DATA}/certs/csr.conf.rendered\" \"${SNAP_DATA}/certs/csr.conf\"; then\n force=true\n $SNAP/bin/cp ${SNAP_DATA}/certs/csr.conf.rendered ${SNAP_DATA}/certs/csr.conf\n else\n force=false\n fi\n\n if $force; then\n gen_server_cert\n gen_proxy_client_cert\n echo \"1\"\n elif [ ! -f \"${SNAP_DATA}/certs/front-proxy-client.crt\" ] ||\n [ \"$(\"${SNAP}/openssl.wrapper\" < ${SNAP_DATA}/certs/front-proxy-client.crt x509 -noout -issuer)\" == \"issuer=CN = 127.0.0.1\" ]; then\n gen_proxy_client_cert\n echo \"1\"\n else\n echo \"0\"\n fi\n}\n\nensure_server_ca() {\n # ensure the server.crt is issued by ca.crt\n # in a ca chain it is only verified that the server.crt is issued by the intermediate ca\n # if current csr.conf is invalid, regenerate front-proxy-client certificates as well\n\n if ! \"${SNAP}/openssl.wrapper\" verify -no-CAfile -no-CApath -partial_chain -trusted ${SNAP_DATA}/certs/ca.crt ${SNAP_DATA}/certs/server.crt &>/dev/null\n then\n csr_modified=\"$(ensure_csr_conf_conservative)\"\n gen_server_cert\n\n if [[ \"$csr_modified\" -eq \"1\" ]]\n then\n gen_proxy_client_cert\n fi\n\n echo \"1\"\n else\n echo \"0\"\n fi\n}\n\ncheck_csr_conf() {\n # if no argument is given, default csr.conf will be checked\n csr_conf=\"${1:-${SNAP_DATA}/certs/csr.conf}\"\n \"${SNAP}/openssl.wrapper\" req -new -config $csr_conf -noout -nodes -keyout /dev/null &>/dev/null\n}\n\nrefresh_csr_conf() {\n render_csr_conf\n $SNAP/bin/cp ${SNAP_DATA}/certs/csr.conf.rendered ${SNAP_DATA}/certs/csr.conf\n}\n\nensure_csr_conf_conservative() {\n # ensure csr.conf is a valid csr config file; if not:\n # copy csr.conf.rendered if valid, or render new if not\n\n if ! check_csr_conf\n then\n if ! check_csr_conf ${SNAP_DATA}/certs/csr.conf.rendered\n then\n render_csr_conf\n fi\n\n $SNAP/bin/cp ${SNAP_DATA}/certs/csr.conf.rendered ${SNAP_DATA}/certs/csr.conf\n echo \"1\"\n else\n echo \"0\"\n fi\n}\n\nrender_csr_conf() {\n # Render csr.conf.template to csr.conf.rendered\n\n local IP_ADDRESSES=\"$(get_ips)\"\n\n $SNAP/bin/cp ${SNAP_DATA}/certs/csr.conf.template ${SNAP_DATA}/certs/csr.conf.rendered\n if ! [ \"$IP_ADDRESSES\" == \"127.0.0.1\" ] && ! [ \"$IP_ADDRESSES\" == \"none\" ]\n then\n local ips='' sep=''\n local -i i=3\n for IP_ADDR in $(echo \"$IP_ADDRESSES\"); do\n local ip_id=\"IP.$((i++))\"\n\t while $SNAP/bin/grep '^'\"${ip_id}\" ${SNAP_DATA}/certs/csr.conf.template > /dev/null ; do\n ip_id=\"IP.$((i++))\"\n done\n ips+=\"${sep}${ip_id} = ${IP_ADDR}\"\n sep='\\n'\n done\n \"$SNAP/bin/sed\" -i \"s/#MOREIPS/${ips}/g\" ${SNAP_DATA}/certs/csr.conf.rendered\n else\n \"$SNAP/bin/sed\" -i 's/#MOREIPS//g' ${SNAP_DATA}/certs/csr.conf.rendered\n fi\n}\n\nget_node() {\n # Returns the node name or no_node_found in case no node is present\n\n KUBECTL=\"$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config\"\n\n timeout=60\n start_timer=\"$($SNAP/bin/date +%s)\"\n node_found=\"yes\"\n while ! ($KUBECTL get no | $SNAP/bin/grep -z \" Ready\") &> /dev/null\n do\n now=\"$($SNAP/bin/date +%s)\"\n if ! [ -z $timeout ] && [[ \"$now\" > \"$(($start_timer + $timeout))\" ]] ; then\n node_found=\"no\"\n echo \"no_node_found\"\n break\n fi\n sleep 2\n done\n\n if [ \"${node_found}\" == \"yes\" ]\n then\n node=\"$($KUBECTL get no | $SNAP/bin/grep ' Ready' | $SNAP/usr/bin/gawk '{print $1}')\"\n echo $node\n fi\n}\n\nwait_for_node() {\n get_node &> /dev/null\n}\n\ndrain_node() {\n # Drain node\n\n node=\"$(get_node)\"\n KUBECTL=\"$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config\"\n if ! [ \"${node}\" == \"no_node_found\" ]\n then\n $KUBECTL drain $node --timeout=120s --grace-period=60 --delete-local-data=true || true\n fi\n}\n\n\nuncordon_node() {\n # Un-drain node\n\n node=\"$(get_node)\"\n KUBECTL=\"$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config\"\n if ! [ \"${node}\" == \"no_node_found\" ]\n then\n $KUBECTL uncordon $node || true\n fi\n}\n\n\nfunction valid_ip() {\n# Test an IP address for validity:\n# Usage:\n# valid_ip IP_ADDRESS\n# if [[ $? -eq 0 ]]; then echo good; else echo bad; fi\n# OR\n# if valid_ip IP_ADDRESS; then echo good; else echo bad; fi\n#\n local ip=$1\n local stat=1\n\n if [[ $ip =~ ^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$ ]]; then\n OIFS=$IFS\n IFS='.'\n ip=($ip)\n IFS=$OIFS\n [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \\\n && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]\n stat=$?\n fi\n return $stat\n}\n\n\ninit_cluster() {\n $SNAP/bin/mkdir -p ${SNAP_DATA}/var/kubernetes/backend\n IP=\"127.0.0.1\"\n # To configure dqlite do:\n # echo \"Address: 1.2.3.4:6364\" > $STORAGE_DIR/update.yaml\n # after the initialisation but before connecting other nodes\n echo \"Address: $IP:19001\" > ${SNAP_DATA}/var/kubernetes/backend/init.yaml\n DNS=$($SNAP/bin/hostname)\n $SNAP/bin/mkdir -p $SNAP_DATA/var/tmp/\n $SNAP/bin/cp $SNAP/certs/csr-dqlite.conf.template $SNAP_DATA/var/tmp/csr-dqlite.conf\n $SNAP/bin/sed -i 's/HOSTNAME/'\"${DNS}\"'/g' $SNAP_DATA/var/tmp/csr-dqlite.conf\n $SNAP/bin/sed -i 's/HOSTIP/'\"${IP}\"'/g' $SNAP_DATA/var/tmp/csr-dqlite.conf\n \"${SNAP}/openssl.wrapper\" req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout ${SNAP_DATA}/var/kubernetes/backend/cluster.key -out ${SNAP_DATA}/var/kubernetes/backend/cluster.crt -subj \"/CN=k8s\" -config $SNAP_DATA/var/tmp/csr-dqlite.conf -extensions v3_ext\n $SNAP/bin/chmod -R o-rwX ${SNAP_DATA}/var/kubernetes/backend/\n local group=$(get_microk8s_group)\n if getent group ${group} >/dev/null 2>&1\n then\n $SNAP/bin/chgrp ${group} -R --preserve=mode ${SNAP_DATA}/var/kubernetes/backend/ || true\n fi\n}\n\n\nfunction update_configs {\n # Create the basic tokens\n ca_data=$($SNAP/bin/cat ${SNAP_DATA}/certs/ca.crt | ${SNAP}/usr/bin/base64 -w 0)\n # Create the client kubeconfig\n run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/client.config\n $SNAP/bin/sed -i 's/CADATA/'\"${ca_data}\"'/g' ${SNAP_DATA}/credentials/client.config\n $SNAP/bin/sed -i 's/NAME/admin/g' ${SNAP_DATA}/credentials/client.config\n if $SNAP/bin/grep admin ${SNAP_DATA}/credentials/known_tokens.csv 2>&1 > /dev/null\n then\n admin_token=`$SNAP/bin/grep admin ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1`\n $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/client.config\n $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/client.config\n else\n admin_token=`$SNAP/bin/grep admin ${SNAP_DATA}/credentials/basic_auth.csv | $SNAP/usr/bin/cut -d, -f1`\n $SNAP/bin/sed -i 's/AUTHTYPE/password/g' ${SNAP_DATA}/credentials/client.config\n fi\n $SNAP/bin/sed -i 's/PASSWORD/'\"${admin_token}\"'/g' ${SNAP_DATA}/credentials/client.config\n # Create the known tokens\n proxy_token=`$SNAP/bin/grep kube-proxy ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1`\n hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]')\n kubelet_token=`$SNAP/bin/grep kubelet-0, ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1`\n controller_token=`$SNAP/bin/grep kube-controller-manager ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1`\n scheduler_token=`$SNAP/bin/grep kube-scheduler ${SNAP_DATA}/credentials/known_tokens.csv | $SNAP/usr/bin/cut -d, -f1`\n # Create the client kubeconfig for the controller\n run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/controller.config\n $SNAP/bin/sed -i 's/CADATA/'\"${ca_data}\"'/g' ${SNAP_DATA}/credentials/controller.config\n $SNAP/bin/sed -i 's/NAME/controller/g' ${SNAP_DATA}/credentials/controller.config\n $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/controller.config\n $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/controller.config\n $SNAP/bin/sed -i 's/PASSWORD/'\"${controller_token}\"'/g' ${SNAP_DATA}/credentials/controller.config\n # Create the client kubeconfig for the scheduler\n run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/scheduler.config\n $SNAP/bin/sed -i 's/CADATA/'\"${ca_data}\"'/g' ${SNAP_DATA}/credentials/scheduler.config\n $SNAP/bin/sed -i 's/NAME/scheduler/g' ${SNAP_DATA}/credentials/scheduler.config\n $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/scheduler.config\n $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/scheduler.config\n $SNAP/bin/sed -i 's/PASSWORD/'\"${scheduler_token}\"'/g' ${SNAP_DATA}/credentials/scheduler.config\n # Create the proxy and kubelet kubeconfig\n run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/kubelet.config\n $SNAP/bin/sed -i 's/NAME/kubelet/g' ${SNAP_DATA}/credentials/kubelet.config\n $SNAP/bin/sed -i 's/CADATA/'\"${ca_data}\"'/g' ${SNAP_DATA}/credentials/kubelet.config\n $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/kubelet.config\n $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/kubelet.config\n $SNAP/bin/sed -i 's/PASSWORD/'\"${kubelet_token}\"'/g' ${SNAP_DATA}/credentials/kubelet.config\n run_with_sudo $SNAP/bin/cp ${SNAP}/client.config.template ${SNAP_DATA}/credentials/proxy.config\n $SNAP/bin/sed -i 's/NAME/kubeproxy/g' ${SNAP_DATA}/credentials/proxy.config\n $SNAP/bin/sed -i 's/CADATA/'\"${ca_data}\"'/g' ${SNAP_DATA}/credentials/proxy.config\n $SNAP/bin/sed -i '/username/d' ${SNAP_DATA}/credentials/proxy.config\n $SNAP/bin/sed -i 's/AUTHTYPE/token/g' ${SNAP_DATA}/credentials/proxy.config\n $SNAP/bin/sed -i 's/PASSWORD/'\"${proxy_token}\"'/g' ${SNAP_DATA}/credentials/proxy.config\n $SNAP/microk8s-stop.wrapper || true\n $SNAP/microk8s-start.wrapper\n}\n\nis_apiserver_ready() {\n if (${SNAP}/usr/bin/curl -L --cert ${SNAP_DATA}/certs/server.crt --key ${SNAP_DATA}/certs/server.key --cacert ${SNAP_DATA}/certs/ca.crt https://127.0.0.1:16443/readyz | $SNAP/bin/grep -z \"ok\") &> /dev/null\n then\n return 0\n else\n return 1\n fi\n}\n\nstart_all_containers() {\n for task in $(\"${SNAP}/microk8s-ctr.wrapper\" task ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}')\n do\n \"${SNAP}/microk8s-ctr.wrapper\" task resume $task &>/dev/null || true\n done\n}\n\nstop_all_containers() {\n for task in $(\"${SNAP}/microk8s-ctr.wrapper\" task ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}')\n do\n \"${SNAP}/microk8s-ctr.wrapper\" task pause $task &>/dev/null || true\n \"${SNAP}/microk8s-ctr.wrapper\" task kill -s SIGKILL $task &>/dev/null || true\n done\n}\n\nremove_all_containers() {\n stop_all_containers\n for task in $(\"${SNAP}/microk8s-ctr.wrapper\" task ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}')\n do\n \"${SNAP}/microk8s-ctr.wrapper\" task delete --force $task &>/dev/null || true\n done\n\n for container in $(\"${SNAP}/microk8s-ctr.wrapper\" containers ls | $SNAP/bin/sed -n '1!p' | $SNAP/usr/bin/gawk '{print $1}')\n do\n \"${SNAP}/microk8s-ctr.wrapper\" container delete $container &>/dev/null || true\n done\n\n run_with_sudo iptables-legacy -t nat -F CNI-HOSTPORT-DNAT &>/dev/null || true\n}\n\nget_container_shim_pids() {\n $SNAP/bin/ps -e -o pid= -o args= | $SNAP/bin/grep -v 'grep' | $SNAP/bin/sed -e 's/^ *//; s/\\s\\s*/\\t/;' | $SNAP/bin/grep -w '/snap/microk8s/.*/bin/containerd-shim' | $SNAP/usr/bin/cut -f1\n}\n\nkill_all_container_shims() {\n run_with_sudo systemctl kill snap.microk8s.daemon-kubelite.service --signal=SIGKILL &>/dev/null || true\n run_with_sudo systemctl kill snap.microk8s.daemon-containerd.service --signal=SIGKILL &>/dev/null || true\n}\n\nis_first_boot() {\n # Return 0 if this is the first start after the host booted.\n # The argument $1 is a directory that may contain a last-start-date file\n # The last-start-date file contains a date in seconds\n # if that date is prior to the creation date of /proc/1 we assume this is the first\n # time after the host booted\n # Note, lxc shares the same /proc/stat as the host\n if ! [ -e \"$1/last-start-date\" ] ||\n ! [ -e /proc/1 ]\n then\n return 1\n else\n last_start=$(\"$SNAP/bin/cat\" \"$1/last-start-date\")\n if [ -e /proc/stat ] &&\n $SNAP/bin/grep btime /proc/stat &&\n ! $SNAP/bin/grep lxc /proc/1/environ\n then\n boot_time=$($SNAP/bin/grep btime /proc/stat | $SNAP/usr/bin/cut -d' ' -f2)\n else\n boot_time=$($SNAP/bin/date -r /proc/1 +%s)\n fi\n echo \"Last time service started was $last_start and the host booted at $boot_time\"\n if [ \"$last_start\" -le \"$boot_time\" ]\n then\n return 0\n else\n return 1\n fi\n fi\n}\n\nmark_boot_time() {\n # place the current time in the \"$1\"/last-start-date file\n now=$($SNAP/bin/date +%s)\n echo \"$now\" > \"$1\"/last-start-date\n}\n\ntry_copy_users_to_snap_microk8s() {\n # try copy users from microk8s to snap_microk8s group\n if getent group microk8s >/dev/null 2>&1 &&\n getent group snap_microk8s >/dev/null 2>&1\n then\n for m in $($SNAP/usr/bin/members microk8s)\n do\n echo \"Processing user $m\"\n if ! usermod -a -G snap_microk8s $m\n then\n echo \"Failed to migrate user $m to snap_microk8s group\"\n fi\n done\n else\n echo \"One of the microk8s or snap_microk8s groups is missing\"\n fi\n}\n\ncluster_agent_port() {\n port=\"25000\"\n if $SNAP/bin/grep -e port \"${SNAP_DATA}\"/args/cluster-agent &> /dev/null\n then\n port=$($SNAP/bin/cat \"${SNAP_DATA}\"/args/cluster-agent | \"$SNAP\"/usr/bin/gawk '{print $2}')\n fi\n\n echo \"$port\"\n}\n\nserver_cert_check() {\n \"${SNAP}/openssl.wrapper\" x509 -in \"$SNAP_DATA\"/certs/server.crt -outform der | ${SNAP}/usr/bin/sha256sum | $SNAP/usr/bin/cut -d' ' -f1 | $SNAP/usr/bin/cut -c1-12\n}\n\ngenerate_csr_with_sans() {\n # Description:\n # Generate CSR for component certificates, including hostname and node IP addresses\n # as SubjectAlternateNames. The CSR PEM is printed to stdout. Arguments are:\n # 1. The certificate subject, e.g. \"/CN=system:node:$hostname/O=system:nodes\"\n # 2. The path to write the private key, e.g. \"$SNAP_DATA/certs/kubelet.key\"\n #\n # Notes:\n # - Subject is /CN=system:node:$hostname/O=system:nodes\n # - Node hostname and IP addresses are added as Subject Alternate Names\n #\n # Example usage:\n # generate_csr_with_sans /CN=system:node:$hostname/O=system:nodes $SNAP_DATA/certs/kubelet.key > $SNAP_DATA/certs/kubelet.csr\n\n # Add DNS name and IP addresses as subjectAltName\n hostname=$($SNAP/bin/hostname | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]')\n subjectAltName=\"DNS:$hostname\"\n for ip in $(get_ips); do\n subjectAltName=\"$subjectAltName, IP:$ip\"\n done\n\n # generate key if it does not exist\n if [ ! -f \"$2\" ]; then\n \"${SNAP}/openssl.wrapper\" genrsa -out \"$2\" 2048\n $SNAP/bin/chown 0:0 \"$2\" || true\n $SNAP/bin/chmod 0600 \"$2\" || true\n fi\n\n # generate csr\n \"${SNAP}/openssl.wrapper\" req -new -sha256 -subj \"$1\" -key \"$2\" -addext \"subjectAltName = $subjectAltName\"\n}\n\ngenerate_csr() {\n # Description:\n # Generate CSR for component certificates. The CSR PEM is written to stdout. Arguments are:\n # 1. The certificate subject, e.g. \"/CN=system:kube-scheduler\"\n # 2. The path to write the private key, e.g. \"$SNAP_DATA/certs/scheduler.key\"\n #\n # Example usage:\n # generate_csr /CN=system:kube-scheduler $SNAP_DATA/certs/scheduler.key > $SNAP_DATA/certs/scheduler.csr\n\n # generate key if it does not exist\n if [ ! -f \"$2\" ]; then\n \"${SNAP}/openssl.wrapper\" genrsa -out \"$2\" 2048\n $SNAP/bin/chown 0:0 \"$2\" || true\n $SNAP/bin/chmod 0600 \"$2\" || true\n fi\n\n # generate csr\n \"${SNAP}/openssl.wrapper\" req -new -sha256 -subj \"$1\" -key \"$2\"\n}\n\nsign_certificate() {\n # Description:\n # Sign a certificate signing request (CSR) using the MicroK8s cluster CA.\n # The CSR is read through stdin, and the signed certificate is printed to stdout.\n #\n # Notes:\n # - Read from stdin and write to stdout, so no temporary files are required.\n # - Any SubjectAlternateNames that are included in the CSR are added to the certificate.\n #\n # Example usage:\n # cat component.csr | sign_certificate > component.crt\n\n # We need to use the request more than once, use this trick to grab stdin and save it in '$csr'\n csr=\"$($SNAP/bin/cat)\"\n\n # Parse SANs from the CSR and add them to the certificate extensions (if any)\n extensions=\"\"\n alt_names=\"$(echo \"$csr\" | \"${SNAP}/openssl.wrapper\" req -text | $SNAP/bin/grep \"X509v3 Subject Alternative Name:\" -A1 | $SNAP/usr/bin/tail -n 1 | $SNAP/bin/sed 's,IP Address:,IP:,g')\"\n if test \"x$alt_names\" != \"x\"; then\n extensions=\"subjectAltName = $alt_names\"\n fi\n\n # Sign certificate and print to stdout\n echo \"$csr\" | \"${SNAP}/openssl.wrapper\" x509 -req -sha256 -CA \"${SNAP_DATA}/certs/ca.crt\" -CAkey \"${SNAP_DATA}/certs/ca.key\" -CAcreateserial -days 3650 -extfile <(echo \"${extensions}\")\n}\n\n\nexit_if_low_memory_guard() {\n if [ -e ${SNAP_DATA}/var/lock/low-memory-guard.lock ]\n then\n echo ''\n echo 'This node does not have enough RAM to host the Kubernetes control plane services'\n echo 'and join the database quorum. You may consider joining this node as a worker'\n echo 'node to a cluster.'\n echo ''\n echo 'If you would still like to start the control plane services, start MicroK8s with:'\n echo ''\n echo ' microk8s start --disable-low-memory-guard'\n echo ''\n exit 1\n fi\n}\n\nrefresh_calico_if_needed() {\n # Call the python script that does the calico update if needed\n \"$SNAP/usr/bin/python3\" \"$SNAP/scripts/calico/upgrade.py\"\n}\n\nremove_docker_specific_args() {\n # Remove docker specific arguments and return 0 if kubelet needs to be restarted\n if $SNAP/bin/grep -e \"\\-\\-network-plugin\" ${SNAP_DATA}/args/kubelet ||\n $SNAP/bin/grep -e \"\\-\\-cni-conf-dir\" ${SNAP_DATA}/args/kubelet ||\n $SNAP/bin/grep -e \"\\-\\-cni-bin-dir\" ${SNAP_DATA}/args/kubelet\n then\n skip_opt_in_local_config network-plugin kubelet\n skip_opt_in_local_config cni-conf-dir kubelet\n skip_opt_in_local_config cni-bin-dir kubelet\n return 0\n fi\n\n return 1\n}\n\nfetch_as() {\n # download from location $1 to location $2\n if is_strict\n then\n ARCH=\"$($SNAP/bin/uname -m)\"\n LD_LIBRARY_PATH=\"$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu\" \"${SNAP}/usr/bin/curl\" -L $1 -o $2\n else\n CA_CERT=/snap/core22/current/etc/ssl/certs/ca-certificates.crt\n run_with_sudo \"${SNAP}/usr/bin/curl\" --cacert $CA_CERT -L $1 -o $2\n fi\n}\n\n############################# Strict functions ######################################\n\nlog_init () {\n echo `$SNAP/bin/date +\"[%m-%d %H:%M:%S]\" start logging` > $SNAP_COMMON/var/log/microk8s.log\n}\n\nlog () {\n echo -n `$SNAP/bin/date +\"[%m-%d %H:%M:%S]\"` >> $SNAP_COMMON/var/log/microk8s.log\n echo \": $@\" >> $SNAP_COMMON/var/log/microk8s.log\n}\n\nis_strict() {\n # Return 0 if we are in strict mode\n if $SNAP/bin/cat $SNAP/meta/snap.yaml | $SNAP/bin/grep confinement | $SNAP/bin/grep -q strict\n then\n return 0\n else\n return 1\n fi\n}\n\ncheck_snap_interfaces() {\n # Check whether all of the required interfaces are connected before proceeding.\n # This is to address https://forum.snapcraft.io/t/mimic-sequence-of-hook-calls-with-auto-connected-interfaces/19618\n declare -ra interfaces=(\n \"account-control\"\n \"docker-privileged\"\n \"dot-kube\"\n \"dot-config-helm\"\n \"firewall-control\"\n \"hardware-observe\"\n \"home\"\n \"home-read-all\"\n \"k8s-journald\"\n \"k8s-kubelet\"\n \"k8s-kubeproxy\"\n \"kernel-module-observe\"\n \"kubernetes-support\"\n \"log-observe\"\n \"login-session-observe\"\n \"mount-observe\"\n \"network\"\n \"network-bind\"\n \"network-control\"\n \"network-observe\"\n \"opengl\"\n \"process-control\"\n \"system-observe\"\n )\n\n declare -a missing=()\n\n for interface in ${interfaces[@]}\n do\n if ! snapctl is-connected ${interface}\n then\n missing+=(\"${interface}\")\n fi\n done\n\n if [ ${#missing[@]} -gt 0 ]\n then\n snapctl set-health blocked \"You must connect ${missing[*]} before proceeding\"\n exit 0\n fi\n}\n\nenable_snap() {\n snapctl start --enable ${SNAP_NAME}\n snapctl set-health okay\n}\n\nexit_if_not_root() {\n # test if we run with sudo\n if (is_strict) && [ \"$EUID\" -ne 0 ]\n then echo \"Elevated permissions are needed for this command. Please use sudo.\"\n exit 1\n fi\n}\n\nis_first_boot_on_strict() {\n # Return 0 if this is the first start after the host booted.\n SENTINEL=\"/tmp/.containerd-first-book-check\"\n # We rely on the fact that /tmp is cleared at every boot to determine if\n # this is the first call after boot: if the sentinel file exists, then it\n # means that no reboot occurred since last check; otherwise, return success\n # and create the sentinel file for the future check.\n if [ -f \"$SENTINEL\" ]\n then\n return 1\n else\n $SNAP/bin/touch \"$SENTINEL\"\n return 0\n fi\n}\n\ndefault_route_exists() {\n # test if we have a default route\n ( $SNAP/sbin/ip route; $SNAP/sbin/ip -6 route ) | $SNAP/bin/grep \"^default\" &>/dev/null\n}\n\nwait_for_default_route() {\n # wait 10 seconds for default route to appear\n n=0\n until [ $n -ge 5 ]\n do\n default_route_exists && break\n echo \"Waiting for default route to appear. (attempt $n)\"\n n=$[$n+1]\n sleep 2\n done\n}\n\nis_ec2_instance() {\n if [ -f \"/sys/hypervisor/uuid\" ]\n then\n EC2UID=$($SNAP/usr/bin/head -c 3 /sys/hypervisor/uuid | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]')\n if [[ $EC2UID == *\"ec2\"* ]]\n then\n return 0\n fi\n else\n if [ -f \"/sys/devices/virtual/dmi/id/product_uuid\" ]\n then\n EC2UID=$($SNAP/usr/bin/head -c 3 /sys/devices/virtual/dmi/id/product_uuid | $SNAP/usr/bin/tr '[:upper:]' '[:lower:]')\n if [[ $EC2UID == *\"ec2\"* ]]\n then\n return 0\n fi\n fi\n fi\n return 1\n}\n\nincrease_sysctl_parameter() {\n declare -a conf_locs=(\"/etc/sysctl.d/\" \"/run/sysctl.d/\" \"/usr/local/lib/sysctl.d/\" \"/usr/lib/sysctl.d/\" \"/lib/sysctl.d/\" \"/etc/sysctl.conf\")\n\n local param_key=$1\n local new_val=$2\n\n val_max=\"0\"\n\n if ! is_strict; then\n if [ -f \"/etc/sysctl.d/10-microk8s.conf\" ]; then\n run_with_sudo $SNAP/usr/bin/sort -u /etc/sysctl.d/10-microk8s.conf -o /etc/sysctl.d/10-microk8s.conf\n fi\n\n for loc in \"${conf_locs[@]}\"\n do\n if run_with_sudo $SNAP/bin/grep -qr \"^$param_key=\" $loc; then\n for val in $(run_with_sudo $SNAP/bin/grep -r \"^$param_key=\" $loc | $SNAP/bin/sed 's/^.*=//');\n do\n if [ \"$val\" -ge \"$val_max\" ]; then\n val_max=$val;\n fi\n done\n fi\n done\n\n if [ \"$val_max\" -lt \"$new_val\" ]; then\n echo \"$param_key=$new_val\" | run_with_sudo $SNAP/usr/bin/tee -a /etc/sysctl.d/10-microk8s.conf\n if ! run_with_sudo sysctl --system; then\n echo \"Could not refresh system parameters via sysctl\"\n fi\n fi\n fi\n}\n\nuse_snap_env() {\n # Configure PATH, LD_LIBRARY_PATH and PYTHONPATH\n export PATH=\"$SNAP/usr/bin:$SNAP/bin:$SNAP/usr/sbin:$SNAP/sbin:$REAL_PATH\"\n export LD_LIBRARY_PATH=\"$SNAP_LIBRARY_PATH:$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$SNAPCRAFT_ARCH_TRIPLET:$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET:$SNAP/usr/lib/$SNAPCRAFT_ARCH_TRIPLET/ceph:${REAL_LD_LIBRARY_PATH:-}\"\n export PYTHONPATH=\"$SNAP/usr/lib/python3.10:$SNAP/lib/python3.10/site-packages:$SNAP/usr/lib/python3/dist-packages\"\n\n # Python configuration\n export PYTHONNOUSERSITE=false\n\n # NOTE(neoaggelos/2023-08-14):\n # we cannot list system locales from snap. instead, we attempt\n # well-known locales for Ubuntu/Debian/CentOS and check whether\n # they are available on the system.\n # if they are, set them for the current shell.\n for locale in C.UTF-8 en_US.UTF-8 en_US.utf8; do\n if [ -z \"$(export LC_ALL=$locale 2>&1)\" ]; then\n export LC_ALL=\"${LC_ALL:-$locale}\"\n export LANG=\"${LC_ALL:-$locale}\"\n break\n fi\n done\n\n # Configure XDG_RUNTIME_DIR\n export XDG_RUNTIME_DIR=\"${SNAP_COMMON}/run\"\n mkdir -p \"${XDG_RUNTIME_DIR}\"\n}\n\n# check if this file is run with arguments\nif [[ \"$0\" == \"${BASH_SOURCE}\" ]] &&\n [[ ! -z \"$1\" ]]\nthen\n # call help\n if echo \"$*\" | grep -q -- 'help'; then\n echo \"usage: $0 [function]\"\n echo \"\"\n echo \"Run a utility function and return the output.\"\n echo \"\"\n echo \"available functions:\"\n declare -F | gawk '{print \"- \"$3}'\n exit 0\n fi\n\n if declare -F \"$1\" > /dev/null\n then\n $1 ${@:2}\n exit $?\n else\n echo \"Function does not exist: $1\" >&2\n exit 1\n fi\nfi\n" }, { "path": "microk8s-resources/basic_auth.csv", "content": "bOU1pQYT0Sjfyg6omGb4zzwmTHXH5Yrp,admin,admin,\"system:masters\"\n" }, { "path": "microk8s-resources/certs/csr-dqlite.conf.template", "content": "[ req ]\ndefault_bits = 2048\nprompt = no\ndefault_md = sha256\nreq_extensions = req_ext\ndistinguished_name = dn\n\n[ dn ]\nC = GB\nST = Canonical\nL = Canonical\nO = Canonical\nOU = Canonical\nCN = k8s\n\n[ req_ext ]\nsubjectAltName = @alt_names\n\n[ alt_names ]\nDNS = HOSTNAME\nIP = HOSTIP\n\n[ v3_ext ]\nauthorityKeyIdentifier=keyid,issuer:always\nbasicConstraints=CA:FALSE\nkeyUsage=keyEncipherment,dataEncipherment,digitalSignature\nextendedKeyUsage=serverAuth,clientAuth\nsubjectAltName=@alt_names\n" }, { "path": "microk8s-resources/certs/csr.conf.template", "content": "[ req ]\ndefault_bits = 2048\nprompt = no\ndefault_md = sha256\nreq_extensions = req_ext\ndistinguished_name = dn\n\n[ dn ]\nC = GB\nST = Canonical\nL = Canonical\nO = Canonical\nOU = Canonical\nCN = 127.0.0.1\n\n[ req_ext ]\nsubjectAltName = @alt_names\n\n[ alt_names ]\nDNS.1 = kubernetes\nDNS.2 = kubernetes.default\nDNS.3 = kubernetes.default.svc\nDNS.4 = kubernetes.default.svc.cluster\nDNS.5 = kubernetes.default.svc.cluster.local\nIP.1 = 127.0.0.1\nIP.2 = 10.152.183.1\n#MOREIPS\n\n[ v3_ext ]\nauthorityKeyIdentifier=keyid,issuer:always\nbasicConstraints=CA:FALSE\nkeyUsage=keyEncipherment,dataEncipherment,digitalSignature\nextendedKeyUsage=serverAuth,clientAuth\nsubjectAltName=@alt_names\n" }, { "path": "microk8s-resources/client-x509.config.template", "content": "apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: CADATA\n server: https://127.0.0.1:16443\n name: microk8s-cluster\ncontexts:\n- context:\n cluster: microk8s-cluster\n user: NAME\n name: microk8s\ncurrent-context: microk8s\nkind: Config\npreferences: {}\nusers:\n- name: NAME\n user:\n client-certificate: PATHTOCERT\n client-key: PATHTOKEYCERT\n" }, { "path": "microk8s-resources/client.config", "content": "apiVersion: v1\nclusters:\n- cluster:\n server: http://127.0.0.1:8080\n name: microk8s-cluster\ncontexts:\n- context:\n cluster: microk8s-cluster\n user: admin\n name: microk8s\ncurrent-context: microk8s\nkind: Config\npreferences: {}\nusers:\n- name: admin\n user:\n username: admin\n" }, { "path": "microk8s-resources/client.config.template", "content": "apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: CADATA\n server: https://127.0.0.1:16443\n name: microk8s-cluster\ncontexts:\n- context:\n cluster: microk8s-cluster\n user: NAME\n name: microk8s\ncurrent-context: microk8s\nkind: Config\npreferences: {}\nusers:\n- name: NAME\n user:\n username: NAME\n AUTHTYPE: PASSWORD\n" }, { "path": "microk8s-resources/containerd-profile", "content": "#include \n\n\nprofile cri-containerd.apparmor.d flags=(attach_disconnected,mediate_deleted) {\n\n #include \n\n\n network,\n network inet,\n network inet6,\n network unix,\n network netlink,\n network raw,\n\n capability,\n file,\n umount,\n\n deny @{PROC}/* w, # deny write for all files directly in /proc (not in a subdir)\n # deny write to files not in /proc//** or /proc/sys/**\n deny @{PROC}/{[^1-9],[^1-9][^0-9],[^1-9s][^0-9y][^0-9s],[^1-9][^0-9][^0-9][^0-9]*}/** w,\n deny @{PROC}/sys/[^k]** w, # deny /proc/sys except /proc/sys/k* (effectively /proc/sys/kernel)\n deny @{PROC}/sys/kernel/{?,??,[^s][^h][^m]**} w, # deny everything except shm* in /proc/sys/kernel/\n deny @{PROC}/sysrq-trigger rwklx,\n deny @{PROC}/mem rwklx,\n deny @{PROC}/kmem rwklx,\n deny @{PROC}/kcore rwklx,\n\n deny mount,\n\n deny /sys/[^f]*/** wklx,\n deny /sys/f[^s]*/** wklx,\n deny /sys/fs/[^c]*/** wklx,\n deny /sys/fs/c[^g]*/** wklx,\n deny /sys/fs/cg[^r]*/** wklx,\n deny /sys/firmware/efi/efivars/** rwklx,\n deny /sys/kernel/security/** rwklx,\n\n\n # suppress ptrace denials when using 'docker ps' or using 'ps' inside a container\n ptrace (trace,read) peer=cri-containerd.apparmor.d,\n\n signal (receive) peer=snap.microk8s.daemon-kubelite,\n signal (receive) peer=snap.microk8s.daemon-containerd,\n signal (send,receive) peer=cri-containerd.apparmor.d//&unconfined,\n}\n" }, { "path": "microk8s-resources/default-args/admission-control-config-file.yaml", "content": "apiVersion: apiserver.config.k8s.io/v1\nkind: AdmissionConfiguration\nplugins:\n - name: EventRateLimit\n path: eventconfig.yaml\n" }, { "path": "microk8s-resources/default-args/apiserver-proxy", "content": "--traefik-config ${SNAP_DATA}/args/traefik/traefik.yaml\n--kubeconfig ${SNAP_DATA}/credentials/kubelet.config\n--refresh-interval 30s\n" }, { "path": "microk8s-resources/default-args/certs.d/docker.io/hosts.toml", "content": "server = \"https://docker.io\"\n\n[host.\"https://registry-1.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n\n" }, { "path": "microk8s-resources/default-args/certs.d/localhost__32000/hosts.toml", "content": "server = \"http://localhost:32000\"\n\n[host.\"http://localhost:32000\"]\n capabilities = [\"pull\", \"resolve\"]\n\n" }, { "path": "microk8s-resources/default-args/cluster-agent", "content": "--bind 0.0.0.0:25000\n--keyfile \"${SNAP_DATA}/certs/server.key\"\n--certfile \"${SNAP_DATA}/certs/server.crt\"\n--timeout 240\n" }, { "path": "microk8s-resources/default-args/cni-env", "content": "# Choose CNI to deploy. Only 'calico' is supported at the moment\nCNI=calico\n\n# IPv4 configuration\nIPv4_SUPPORT=true\nIPv4_CLUSTER_CIDR=10.1.0.0/16\nIPv4_SERVICE_CIDR=10.152.183.0/24\n\n# IPv6 configuration\nIPv6_SUPPORT=false\n# IPv6_CLUSTER_CIDR=fd01::/64\n# IPv6_SERVICE_CIDR=fd98::/108\n\n# Calico-specific configuration settings\n# CALICO_VETH_MTU=1450\n" }, { "path": "microk8s-resources/default-args/cni-network/flannel.conflist", "content": "{\n \"name\": \"microk8s-flannel-network\",\n \"plugins\": [\n {\n \"type\": \"flannel\",\n \"delegate\": {\n \"hairpinMode\": true,\n \"isDefaultGateway\": true\n }\n },\n {\n \"type\": \"portmap\",\n \"capabilities\": {\"portMappings\": true},\n \"snat\": true\n }\n ]\n}\n" }, { "path": "microk8s-resources/default-args/containerd", "content": "--config ${SNAP_DATA}/args/containerd.toml\n--root ${SNAP_COMMON}/var/lib/containerd\n--state ${SNAP_COMMON}/run/containerd\n--address ${SNAP_COMMON}/run/containerd.sock\n" }, { "path": "microk8s-resources/default-args/containerd-env", "content": "# Remember to restart MicroK8s after editing this file:\n#\n# sudo microk8s stop; sudo microk8s start\n#\n\n# To start containerd behind a proxy you need to add an HTTPS_PROXY\n# environment variable in this file. HTTPS_PROXY is of the following form:\n# HTTPS_PROXY=http://username:password@proxy:port/\n# where username: and password@ are optional. eg:\n#\n# HTTPS_PROXY=https://squid.internal:3128\n#\n# You may also want to set NO_PROXY to include the cluster-cidr and the services-cidr\n# as specified in /var/snap/microk8s/current/args/kube-proxy and\n# /var/snap/microk8s/current/args/kube-apiserver\n#\n# NO_PROXY=10.1.0.0/16,10.152.183.0/24\n#\n\n# You can set the of the kata containers runtime here.\n#\n# KATA_PATH=\n#\nPATH=$PATH:$KATA_PATH\n\n# Attempt to change the maximum number of open file descriptors\n# this get inherited to the running containers\n#\nulimit -n 65536 || true\n\n# Attempt to change the maximum locked memory limit\n# this get inherited to the running containers\n#\nulimit -l 16384 || true\n\n" }, { "path": "microk8s-resources/default-args/containerd-template.toml", "content": "# Use config version 2 to enable new configuration fields.\nversion = 2\noom_score = 0\n\n[grpc]\n uid = 0\n gid = 0\n max_recv_message_size = 16777216\n max_send_message_size = 16777216\n\n[debug]\n address = \"\"\n uid = 0\n gid = 0\n\n[metrics]\n address = \"127.0.0.1:1338\"\n grpc_histogram = false\n\n[cgroup]\n path = \"\"\n\n\n# The 'plugins.\"io.containerd.grpc.v1.cri\"' table contains all of the server options.\n[plugins.\"io.containerd.grpc.v1.cri\"]\n\n stream_server_address = \"127.0.0.1\"\n stream_server_port = \"0\"\n enable_selinux = false\n sandbox_image = \"registry.k8s.io/pause:3.10\"\n stats_collect_period = 10\n enable_tls_streaming = false\n max_container_log_line_size = 16384\n\n # 'plugins.\"io.containerd.grpc.v1.cri\".containerd' contains config related to containerd\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n # snapshotter is the snapshotter used by containerd.\n snapshotter = \"${SNAPSHOTTER}\"\n\n # no_pivot disables pivot-root (linux only), required when running a container in a RamDisk with runc.\n # This only works for runtime type \"io.containerd.runtime.v1.linux\".\n no_pivot = false\n\n # default_runtime_name is the default runtime name to use.\n default_runtime_name = \"${RUNTIME}\"\n\n # 'plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes' is a map from CRI RuntimeHandler strings, which specify types\n # of runtime configurations, to the matching configurations.\n # In this example, 'runc' is the RuntimeHandler string to match.\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux\n runtime_type = \"${RUNTIME_TYPE}\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.nvidia-container-runtime]\n # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux\n runtime_type = \"${RUNTIME_TYPE}\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.nvidia-container-runtime.options]\n BinaryName = \"nvidia-container-runtime\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.kata]\n runtime_type = \"io.containerd.kata.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.kata.options]\n BinaryName = \"kata-runtime\"\n\n # 'plugins.\"io.containerd.grpc.v1.cri\".cni' contains config related to cni\n [plugins.\"io.containerd.grpc.v1.cri\".cni]\n # bin_dir is the directory in which the binaries for the plugin is kept.\n bin_dir = \"${SNAP_DATA}/opt/cni/bin\"\n\n # conf_dir is the directory in which the admin places a CNI conf.\n conf_dir = \"${SNAP_DATA}/args/cni-network\"\n\n # 'plugins.\"io.containerd.grpc.v1.cri\".registry' contains config related to the registry\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n config_path = \"${SNAP_DATA}/args/certs.d\"\n" }, { "path": "microk8s-resources/default-args/ctr", "content": "--address=${SNAP_COMMON}/run/containerd.sock\n--namespace k8s.io\n" }, { "path": "microk8s-resources/default-args/etcd", "content": "--data-dir=${SNAP_COMMON}/var/run/etcd\n--advertise-client-urls=https://${DEFAULT_INTERFACE_IP_ADDR}:12379\n--listen-client-urls=https://0.0.0.0:12379\n--client-cert-auth\n--trusted-ca-file=${SNAP_DATA}/certs/ca.crt\n--cert-file=${SNAP_DATA}/certs/server.crt\n--key-file=${SNAP_DATA}/certs/server.key\n" }, { "path": "microk8s-resources/default-args/eventconfig.yaml", "content": "apiVersion: eventratelimit.admission.k8s.io/v1alpha1\nkind: Configuration\nlimits:\n - type: Server\n qps: 5000\n burst: 20000\n" }, { "path": "microk8s-resources/default-args/flannel-network-mgr-config", "content": "{\"Network\": \"10.1.0.0/16\", \"Backend\": {\"Type\": \"vxlan\"}}" }, { "path": "microk8s-resources/default-args/flannel-template.conflist", "content": "{\n \"name\": \"microk8s-flannel-network\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": [\n {\n \"type\": \"flannel\",\n \"name\": \"flannel-plugin\",\n \"subnetFile\": \"${SNAP_COMMON}/run/flannel/subnet.env\",\n \"dataDir\": \"${SNAP_COMMON}/var/lib/cni/flannel\",\n \"delegate\": {\n \"hairpinMode\": true,\n \"isDefaultGateway\": true\n }\n },\n {\n \"type\": \"portmap\",\n \"capabilities\": {\"portMappings\": true},\n \"snat\": true\n }\n ]\n}\n" }, { "path": "microk8s-resources/default-args/flanneld", "content": "--iface=\"\"\n--etcd-endpoints=https://127.0.0.1:12379\n--etcd-cafile=${SNAP_DATA}/certs/ca.crt\n--etcd-certfile=${SNAP_DATA}/certs/server.crt\n--etcd-keyfile=${SNAP_DATA}/certs/server.key\n--subnet-file=${SNAP_COMMON}/run/flannel/subnet.env\n--ip-masq=true\n" }, { "path": "microk8s-resources/default-args/git/.gitconfig", "content": "[safe]\n directory = /snap/microk8s/current/addons/community/.git\n directory = /snap/microk8s/current/addons/core/.git\n" }, { "path": "microk8s-resources/default-args/ha-conf", "content": "failure-domain=1\n" }, { "path": "microk8s-resources/default-args/k8s-dqlite", "content": "--storage-dir=${SNAP_DATA}/var/kubernetes/backend/\n--listen=unix://${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379\n" }, { "path": "microk8s-resources/default-args/k8s-dqlite-env", "content": "# Environment variables k8s-dqlite will run with\n#\n# Enable dqlite debugging flags\n# LIBRAFT_TRACE=\"1\"\n# LIBDQLITE_TRACE=\"1\"\n" }, { "path": "microk8s-resources/default-args/kube-apiserver", "content": "--cert-dir=${SNAP_DATA}/certs\n--service-cluster-ip-range=10.152.183.0/24\n--authorization-mode=AlwaysAllow\n--service-account-key-file=${SNAP_DATA}/certs/serviceaccount.key\n--client-ca-file=${SNAP_DATA}/certs/ca.crt\n--tls-cert-file=${SNAP_DATA}/certs/server.crt\n--tls-private-key-file=${SNAP_DATA}/certs/server.key\n--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384\n--kubelet-client-certificate=${SNAP_DATA}/certs/apiserver-kubelet-client.crt\n--kubelet-client-key=${SNAP_DATA}/certs/apiserver-kubelet-client.key\n--secure-port=16443\n--etcd-servers=\"unix://${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379\"\n--allow-privileged=true\n--service-account-issuer='https://kubernetes.default.svc'\n--service-account-signing-key-file=${SNAP_DATA}/certs/serviceaccount.key\n--event-ttl=5m\n--profiling=false\n--feature-gates=ListFromCacheSnapshot=false,SizeBasedListCostEstimate=false,DetectCacheInconsistency=false\n\n# Enable the aggregation layer\n--requestheader-client-ca-file=${SNAP_DATA}/certs/front-proxy-ca.crt\n--requestheader-allowed-names=front-proxy-client\n--requestheader-extra-headers-prefix=X-Remote-Extra-\n--requestheader-group-headers=X-Remote-Group\n--requestheader-username-headers=X-Remote-User\n--proxy-client-cert-file=${SNAP_DATA}/certs/front-proxy-client.crt\n--proxy-client-key-file=${SNAP_DATA}/certs/front-proxy-client.key\n#~Enable the aggregation layer\n--enable-admission-plugins=EventRateLimit\n--admission-control-config-file=${SNAP_DATA}/args/admission-control-config-file.yaml\n--kubelet-certificate-authority=${SNAP_DATA}/certs/ca.crt\n--kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP\n" }, { "path": "microk8s-resources/default-args/kube-controller-manager", "content": "--kubeconfig=${SNAP_DATA}/credentials/controller.config\n--service-account-private-key-file=${SNAP_DATA}/certs/serviceaccount.key\n--root-ca-file=${SNAP_DATA}/certs/ca.crt\n--cluster-signing-cert-file=${SNAP_DATA}/certs/ca.crt\n--cluster-signing-key-file=${SNAP_DATA}/certs/ca.key\n--use-service-account-credentials\n--leader-elect-lease-duration=60s\n--leader-elect-renew-deadline=30s\n--profiling=false\n" }, { "path": "microk8s-resources/default-args/kube-proxy", "content": "--kubeconfig=${SNAP_DATA}/credentials/proxy.config\n--cluster-cidr=10.1.0.0/16\n--healthz-bind-address=127.0.0.1\n--profiling=false\n" }, { "path": "microk8s-resources/default-args/kube-scheduler", "content": "--kubeconfig=${SNAP_DATA}/credentials/scheduler.config\n--leader-elect-lease-duration=60s\n--leader-elect-renew-deadline=30s\n--profiling=false\n" }, { "path": "microk8s-resources/default-args/kubectl", "content": "" }, { "path": "microk8s-resources/default-args/kubectl-env", "content": "export KUBECONFIG=$SNAP_DATA/credentials/client.config" }, { "path": "microk8s-resources/default-args/kubelet", "content": "--kubeconfig=${SNAP_DATA}/credentials/kubelet.config\n--cert-dir=${SNAP_DATA}/certs\n--client-ca-file=${SNAP_DATA}/certs/ca.crt\n--anonymous-auth=false\n--root-dir=${SNAP_COMMON}/var/lib/kubelet\n--fail-swap-on=false\n--eviction-hard=\"memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi\"\n--container-runtime-endpoint=${SNAP_COMMON}/run/containerd.sock\n--containerd=${SNAP_COMMON}/run/containerd.sock\n--node-labels=\"microk8s.io/cluster=true,node.kubernetes.io/microk8s-controlplane=microk8s-controlplane\"\n--authentication-token-webhook=true\n--read-only-port=0\n--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256\n--serialize-image-pulls=false\n" }, { "path": "microk8s-resources/default-args/kubelite", "content": "--scheduler-args-file=$SNAP_DATA/args/kube-scheduler\n--controller-manager-args-file=$SNAP_DATA/args/kube-controller-manager\n--proxy-args-file=$SNAP_DATA/args/kube-proxy\n--kubelet-args-file=$SNAP_DATA/args/kubelet\n--apiserver-args-file=$SNAP_DATA/args/kube-apiserver\n--kubeconfig-file=$SNAP_DATA/credentials/client.config\n--start-control-plane=true\n" }, { "path": "microk8s-resources/default-args/traefik/provider-template.yaml", "content": "tcp:\n routers:\n Router-1:\n rule: \"HostSNI(`*`)\"\n service: \"kube-apiserver\"\n tls:\n passthrough: true\n services:\n kube-apiserver:\n loadBalancer:\n servers:\n# APISERVERS\n# - address: \"10.130.0.2:16443\"\n# - address: \"10.130.0.3:16443\"\n# - address: \"10.130.0.4:16443\"\n" }, { "path": "microk8s-resources/default-args/traefik/traefik-template.yaml", "content": "entryPoints:\n apiserver:\n address: \":16443\"\nproviders:\n file:\n filename: ${SNAP_DATA}/args/traefik/provider.yaml\n watch: true\n" }, { "path": "microk8s-resources/default-hooks/post-refresh.d/30-helm", "content": "#!/bin/bash\n\n# Install helm binaries in SNAP_DATA to maintain backwards-compatibility\nif [ -d \"${SNAP_DATA}/bin\" ]; then\n cp \"${SNAP}/bin/helm\" \"${SNAP_DATA}/bin/helm3\"\nfi\n" }, { "path": "microk8s-resources/default-hooks/reconcile.d/10-pods-restart", "content": "#!/bin/bash\n\n. \"${SNAP}/actions/common/utils.sh\"\n\nif ! [ -e \"${SNAP_DATA}/var/lock/no-cni-reload\" ] &&\n [ -e \"${SNAP_DATA}/var/lock/snapdata-mounts-need-reload\" ]; then\n if (is_apiserver_ready) && \"${SNAP}/scripts/kill-host-pods.py\" \\\n --with-snap-data-mounts --with-owner \\\n -- -A --field-selector spec.nodeName=$(hostname) ; then\n rm \"${SNAP_DATA}/var/lock/snapdata-mounts-need-reload\"\n fi\nfi\n" }, { "path": "microk8s-resources/default-hooks/reconcile.d/90-calico-apply", "content": "#!/usr/bin/env bash\n\n. \"${SNAP}/actions/common/utils.sh\"\n\nuse_snap_env\n\nKUBECTL=\"${SNAP}/microk8s-kubectl.wrapper\"\n\nif [ -e \"${SNAP_DATA}/args/cni-network/cni.yaml\" ] &&\n [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ] &&\n ! [ -e \"${SNAP_DATA}/var/lock/cni-loaded\" ]\nthen\n echo \"Setting up the CNI\"\n if (is_apiserver_ready) && \"${KUBECTL}\" apply -f \"${SNAP_DATA}/args/cni-network/cni.yaml\"\n then\n touch \"${SNAP_DATA}/var/lock/cni-loaded\"\n\n # We just installed Calico, no need to refresh\n rm \"${SNAP_DATA}/var/lock/cni-needs-reload\" || true\n fi\nfi\n" }, { "path": "microk8s-resources/default-hooks/remove.d/10-cni-link", "content": "#!/bin/bash\n\n. \"${SNAP}/actions/common/utils.sh\"\n\nif ! is_strict || (is_strict && snapctl is-connected network-control)\nthen\n for link in cni0\n do\n if \"${SNAP}/sbin/ip\" link show \"${link}\"\n then\n \"${SNAP}/sbin/ip\" link delete \"${link}\" || true\n fi\n done\n for calink in $(\"${SNAP}/sbin/ip\" -j link show |\\\n \"${SNAP}/usr/bin/jq\" -r '.[].ifname | select(test(\"^vxlan[-v6]*.calico|cali[a-f0-9]*$\"))')\n do \"${SNAP}/sbin/ip\" link delete \"${calink}\" || true\n done\nfi\n" }, { "path": "microk8s-resources/default-hooks/remove.d/10-cni-link-cilium", "content": "#!/bin/bash\n\n. \"${SNAP}/actions/common/utils.sh\"\n\nif ! is_strict || (is_strict && snapctl is-connected network-control)\nthen\n for link in cilium_host cilium_vxlan\n do\n if \"${SNAP}/sbin/ip\" link show \"${link}\"\n then\n \"${SNAP}/sbin/ip\" link delete \"${link}\" || true\n fi\n done\nfi\n" }, { "path": "microk8s-resources/default-hooks/remove.d/20-cni-netns", "content": "#!/bin/bash\n\n. \"${SNAP}/actions/common/utils.sh\"\n\nif ! is_strict || (is_strict && snapctl is-connected network-control)\nthen\n for ns in `\"${SNAP}/sbin/ip\" netns list | grep \"^cni-\" | awk '{print $1}'`\n do\n \"${SNAP}/sbin/ip\" netns delete \"${ns}\" || true\n done\nfi\n" }, { "path": "microk8s-resources/default-hooks/remove.d/90-containers", "content": "#!/bin/bash\n\n. \"${SNAP}/actions/common/utils.sh\"\n\nremove_all_containers\nkill_all_container_shims\n" }, { "path": "microk8s-resources/kubelet.config", "content": "apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURQakNDQWlhZ0F3SUJBZ0lKQU8xT0dNWkVmQlJETUEwR0NTcUdTSWIzRFFFQkN3VUFNQmt4RnpBVkJnTlYKQkFNTURqSTFNaTQxTUM0eE9EZ3VNakU0TUI0WERURTRNRFV3TVRFME5UWTFOMW9YRFRJNE1EUXlPREUwTlRZMQpOMW93R1RFWE1CVUdBMVVFQXd3T01qVXlMalV3TGpFNE9DNHlNVGd3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURYUHFsZ3VtMmhWTlI1R3BOdEkzbTF2YnFxNVpwNUNVZHhFZmZPcW1qK201NmUKSzFFQjVNdi9rRVUvWk9FYi9RdHFyeWtCZkFhSFZOY1c1eVJVcCtXTHlEOU5xY0tUQVZ5TFJ3YVRaOWdwTzFhdQo3Qk9VeklTOXZYZ255emUrZ2FPVTZ4Z3NNL2pncklINklmd0tQaDc2NU1rc2UxV2lSL3VuWnhZY0FjOFNwbWdyCm5JVlRidVJoeDZYWDhZdlFqOWoxVXc1RkQrejE0aE1vWHcyTFpYQUJwWW5NaFd1bUdYVldiNzluTkd0VTZPUzcKK2Eybi83Z01nbWdpZlFnWVhRZHBXbUgzWE8xdDVtZHBDSkYrdUxFTkRxUzIxYU1CS0prN080VmNRS1NRd2R2cQp2cVQ5ZmplUmNLMWEreGZiZU95NjNSblh2ZTEreGwwbyt2MndiMm9KQWdNQkFBR2pnWWd3Z1lVd0hRWURWUjBPCkJCWUVGSWoyL0szejlxS1RRSU83V3YvRVY4QzJpSzJBTUVrR0ExVWRJd1JDTUVDQUZJajIvSzN6OXFLVFFJTzcKV3YvRVY4QzJpSzJBb1Iya0d6QVpNUmN3RlFZRFZRUUREQTR5TlRJdU5UQXVNVGc0TGpJeE9JSUpBTzFPR01aRQpmQlJETUF3R0ExVWRFd1FGTUFNQkFmOHdDd1lEVlIwUEJBUURBZ0VHTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCCkFRQ3hUb0dFUXVFSUhwanJEQTIzakxOaVlVcmgxeEFwZlI4bWZxcmR4Ukp2NnN6ZTdIQks2QkRoelY2UURVQlcKRG1tWVNFMDlURkx0WGkyT1VWaGNsVkg3Qm90bVRseUEvTWlZRm5CS2x3ekE1S0VSdGRTU2l2V2tYL2xBQXkvYQpSTTNmZTVnR042RVROS3NvSEI0S1lLaU5DS2poVmR6RjdhajNkUktqanZHTmducEQ1R1hBd1AyY2NONXUxbXBjCkpPV2RzWlh4QmMyRkxyWmFaK1hBdWJWQ0V2amNJMSs0MzlMdEplNU1IdU94Q2UyS1k4N2lOU2gyaTl6cDAxc08KSVUxR0s0VHlCYU1Lc0Nhblh6ZmRyNGFoeXdvdUlVd3p3RDEzeTF6M1Q5RnRJRDltTUlxdlNPKzhOZFNDU3dINQpMUmJIa0xDdFZvMVF2alpDREV2eWFVSWsKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\n server: http://127.0.0.1:8080\n name: microk8s-cluster\ncontexts:\n- context:\n cluster: microk8s-cluster\n user: kubelet\n name: microk8s\ncurrent-context: microk8s\nkind: Config\npreferences: {}\nusers:\n- name: kubelet\n user:\n token: tzviBa7HFzlDaGzVYezXdD528jbQPzgw\n" }, { "path": "microk8s-resources/kubelet.config.template", "content": "apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: CADATA\n server: https://127.0.0.1:16443\n name: microk8s-cluster\ncontexts:\n- context:\n cluster: microk8s-cluster\n user: NAME\n name: microk8s\ncurrent-context: microk8s\nkind: Config\npreferences: {}\nusers:\n- name: NAME\n user:\n token: TOKEN\n" }, { "path": "microk8s-resources/kubeproxy.config", "content": "apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURQakNDQWlhZ0F3SUJBZ0lKQU8xT0dNWkVmQlJETUEwR0NTcUdTSWIzRFFFQkN3VUFNQmt4RnpBVkJnTlYKQkFNTURqSTFNaTQxTUM0eE9EZ3VNakU0TUI0WERURTRNRFV3TVRFME5UWTFOMW9YRFRJNE1EUXlPREUwTlRZMQpOMW93R1RFWE1CVUdBMVVFQXd3T01qVXlMalV3TGpFNE9DNHlNVGd3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURYUHFsZ3VtMmhWTlI1R3BOdEkzbTF2YnFxNVpwNUNVZHhFZmZPcW1qK201NmUKSzFFQjVNdi9rRVUvWk9FYi9RdHFyeWtCZkFhSFZOY1c1eVJVcCtXTHlEOU5xY0tUQVZ5TFJ3YVRaOWdwTzFhdQo3Qk9VeklTOXZYZ255emUrZ2FPVTZ4Z3NNL2pncklINklmd0tQaDc2NU1rc2UxV2lSL3VuWnhZY0FjOFNwbWdyCm5JVlRidVJoeDZYWDhZdlFqOWoxVXc1RkQrejE0aE1vWHcyTFpYQUJwWW5NaFd1bUdYVldiNzluTkd0VTZPUzcKK2Eybi83Z01nbWdpZlFnWVhRZHBXbUgzWE8xdDVtZHBDSkYrdUxFTkRxUzIxYU1CS0prN080VmNRS1NRd2R2cQp2cVQ5ZmplUmNLMWEreGZiZU95NjNSblh2ZTEreGwwbyt2MndiMm9KQWdNQkFBR2pnWWd3Z1lVd0hRWURWUjBPCkJCWUVGSWoyL0szejlxS1RRSU83V3YvRVY4QzJpSzJBTUVrR0ExVWRJd1JDTUVDQUZJajIvSzN6OXFLVFFJTzcKV3YvRVY4QzJpSzJBb1Iya0d6QVpNUmN3RlFZRFZRUUREQTR5TlRJdU5UQXVNVGc0TGpJeE9JSUpBTzFPR01aRQpmQlJETUF3R0ExVWRFd1FGTUFNQkFmOHdDd1lEVlIwUEJBUURBZ0VHTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCCkFRQ3hUb0dFUXVFSUhwanJEQTIzakxOaVlVcmgxeEFwZlI4bWZxcmR4Ukp2NnN6ZTdIQks2QkRoelY2UURVQlcKRG1tWVNFMDlURkx0WGkyT1VWaGNsVkg3Qm90bVRseUEvTWlZRm5CS2x3ekE1S0VSdGRTU2l2V2tYL2xBQXkvYQpSTTNmZTVnR042RVROS3NvSEI0S1lLaU5DS2poVmR6RjdhajNkUktqanZHTmducEQ1R1hBd1AyY2NONXUxbXBjCkpPV2RzWlh4QmMyRkxyWmFaK1hBdWJWQ0V2amNJMSs0MzlMdEplNU1IdU94Q2UyS1k4N2lOU2gyaTl6cDAxc08KSVUxR0s0VHlCYU1Lc0Nhblh6ZmRyNGFoeXdvdUlVd3p3RDEzeTF6M1Q5RnRJRDltTUlxdlNPKzhOZFNDU3dINQpMUmJIa0xDdFZvMVF2alpDREV2eWFVSWsKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==\n server: https://172.31.50.188:16443\n name: microk8s-cluster\ncontexts:\n- context:\n cluster: microk8s-cluster\n user: kube-proxy\n name: microk8s\ncurrent-context: microk8s\nkind: Config\npreferences: {}\nusers:\n- name: kube-proxy\n user:\n token: RPY4OSU13Js323K4FDmXgVW4YWOdS65i\n" }, { "path": "microk8s-resources/microk8s.default.yaml", "content": "# Default launch configuration for MicroK8s.\n---\nversion: 0.1.0\nextraKubeletArgs:\n --cluster-domain: cluster.local\n --cluster-dns: 10.152.183.10\naddons:\n - name: dns\n" }, { "path": "microk8s-resources/wrappers/apiservice-kicker", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"api service kicker will not run on a cluster node\"\n exit 0\nfi\n\nif [ -e \"${SNAP_DATA}/var/lock/low-memory-guard.lock\" ]\nthen\n\techo \"not starting api service kicker because of low memory guard lock\"\n\texit 0\nfi\n\ngroup=$(get_microk8s_or_cis_group)\nrestart_attempt=0\ninstalled_registry_help=0\n\nwhile true\ndo\n if [ $restart_attempt -ge 5 ]\n then\n echo \"Service kicker restarted the apiserver too quickly. Exiting.\"\n exit 1\n fi\n\n # every 5 seconds\n sleep 5\n if [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ] &&\n [ ! -e \"${SNAP_DATA}/var/lock/cis-hardening\" ] &&\n getent group ${group} >/dev/null 2>&1\n then\n chmod -R ug+rwX ${SNAP_DATA}/var/kubernetes/backend || true\n chgrp ${group} -R ${SNAP_DATA}/var/kubernetes/backend || true\n chgrp ${group} ${SNAP_COMMON}/run/containerd.sock || true\n fi\n\n # certificate regeneration on: IP/csr change if not clustered, CA mismatch if clustered\n manage_certs() {\n if ! [ -e \"${SNAP_DATA}/var/lock/no-cert-reissue\" ] &&\n ! grep -E \"(--advertise-address|--bind-address)\" $SNAP_DATA/args/kube-apiserver &> /dev/null &&\n ip route | grep default &> /dev/null\n then\n echo \"$(produce_certs)\"\n else\n echo \"$(ensure_server_ca)\"\n fi\n }\n\n if snapctl services microk8s.daemon-kubelite | grep active &> /dev/null\n then\n certs_modified=\"$(manage_certs)\"\n if [[ \"$certs_modified\" -eq \"1\" ]] &&\n ! [ -e \"${SNAP_DATA}/var/lock/join-in-progress\" ]\n then\n echo \"cert change detected. Restarting the cluster-agent\"\n snapctl restart microk8s.daemon-cluster-agent\n\n echo \"cert change detected. Reconfiguring the kube-apiserver\"\n rm -rf .srl\n snapctl stop microk8s.daemon-kubelite\n remove_all_containers\n kill_all_container_shims\n snapctl restart microk8s.daemon-containerd\n snapctl start microk8s.daemon-kubelite\n start_all_containers\n restart_attempt=$[$restart_attempt+1]\n else\n restart_attempt=0\n fi\n fi\n\n # Run reconcile hooks\n $SNAP/usr/bin/python3 $SNAP/scripts/run-lifecycle-hooks.py reconcile || true\n\n # If no local-registry-hosting documentation has been installed,\n # install a help guide that points to the microk8s registry docs.\n #\n # Wait until the apiserver is up and is successfully responding to\n # namespace checks before we check for the registry configmap.\n if snapctl services microk8s.daemon-kubelite | grep active &> /dev/null\n then\n if [ $installed_registry_help -eq 0 ] &&\n \"$SNAP/kubectl\" \"--kubeconfig=$SNAP_DATA/credentials/client.config\" get namespace kube-public &> /dev/null\n then\n if ! \"$SNAP/kubectl\" \"--kubeconfig=$SNAP_DATA/credentials/client.config\" get configmap local-registry-hosting -n kube-public &> /dev/null\n then\n use_manifest registry/registry-help apply\n fi\n\n installed_registry_help=1\n fi\n fi\ndone\n" }, { "path": "microk8s-resources/wrappers/git.wrapper", "content": "#!/usr/bin/env bash\n\n. $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexport GIT_EXEC_PATH=\"$SNAP/usr/lib/git-core\"\nexport GIT_TEMPLATE_DIR=\"$SNAP/usr/share/git-core/templates\"\nexport GIT_CONFIG_NOSYSTEM=1\n\nHOME=\"${SNAP_DATA}/args/git\"\n\n\"$SNAP/usr/bin/git\" \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-add-node.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s add-node on the master.\"\n exit 1\nfi\n\nif echo \"$*\" | grep -q -- 'help'; then\n # Call add_token.py help\n ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/add_token.py --help\n exit 0\nfi\n\nexit_if_not_root\n\nexit_if_no_permissions\n\nsubject=$(\"${SNAP}/openssl.wrapper\" x509 -sha256 -days 365 -noout -subject -in \"$SNAP_DATA/certs/ca.crt\")\nif [[ $subject == *\"127.0.0.1\"* ]]; then\n echo \"Clustering requires a fresh MicroK8s installation. Reinstall with:\"\n echo \"sudo snap remove microk8s\"\n install_message=\"sudo snap install microk8s\"\n if ! is_strict\n then\n install_message=\"${install_message} --classic\"\n fi\n echo \"$install_message\"\n exit 1\nfi\n\nexit_if_stopped\n\nif [ ! -f \"$SNAP_DATA/credentials/cluster-tokens.txt\" ]; then\n touch $SNAP_DATA/credentials/cluster-tokens.txt\nfi\n\ngroup=$(get_microk8s_or_cis_group)\n\nif getent group ${group} >/dev/null 2>&1\nthen\n chgrp ${group} $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true\n chmod ug+rw $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true\n chmod o-rwX $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true\nfi\n\n# Use python's built-in (3.6+) secrets generator to produce the token.\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/add_token.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-addons.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_no_permissions\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/addons.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-config.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s config on the master.\"\n exit 0\nfi\n\nUSE_LOOPBACK=false\nPARSED=$(getopt --options=lho: --longoptions=use-loopback,help,output: --name \"$@\" -- \"$@\")\neval set -- \"$PARSED\"\nwhile true; do\n case \"$1\" in\n -l|--use-loopback)\n USE_LOOPBACK=true\n shift\n ;;\n -h|--help)\n echo \"Usage: microk8s config [OPTIONS]\"\n echo\n echo \"Retrieve the client config, similar to microk8s kubectl config view --raw\"\n echo\n echo \"Options:\"\n echo \" -h, --help Show this help\"\n echo \" -l, --use-loopback Report the cluster address using the loopback address\"\n echo \" (127.0.0.1) rather than the default interface address\"\n exit 0\n ;;\n --)\n shift\n break\n ;;\n *)\n echo \"microk8s config: invalid option -- $1\"\n exit 1\n esac\ndone\n\nexit_if_no_permissions\n\nif [[ \"$USE_LOOPBACK\" == \"true\" ]]; then\n cat \"$SNAP_DATA/credentials/client.config\"\n \"$SNAP/bin/echo\"\nelse\n IP_ADDR=\"$(get_default_ip)\"\n \"$SNAP/bin/sed\" -e \"s/127.0.0.1/$IP_ADDR/\" \"$SNAP_DATA/credentials/client.config\"\n \"$SNAP/bin/echo\"\nfi\n" }, { "path": "microk8s-resources/wrappers/microk8s-ctr.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_no_permissions\n\nexport CONTAINERD_SNAPSHOTTER=$(snapshotter)\n\nif ! [ -e $SNAP_DATA/args/ctr ]\nthen\n echo \"Arguments file $SNAP_DATA/args/ctr is missing.\"\n exit 1\nfi\n\ndeclare -a args=\"($(cat $SNAP_DATA/args/ctr))\"\n\nrun_with_sudo \"${SNAP}/bin/ctr\" \"${args[@]}\" \"$@\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-dashboard-proxy.wrapper", "content": "#!/usr/bin/env bash\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_not_root\nexit_if_no_permissions\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dashboard_proxy.py\n" }, { "path": "microk8s-resources/wrappers/microk8s-dbctl.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_not_root\n\nexit_if_no_permissions\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dbctl.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-disable.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\n# avoid AppArmor denial in strict mode when running under sudo without -H\n\nif is_strict\nthen\n cd \"$SNAP\"\nfi\n\nexit_if_not_root\n\nexit_if_no_permissions\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/disable.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-enable.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\n# avoid AppArmor denial in strict mode when running under sudo without -H\nif is_strict\nthen\n cd \"$SNAP\"\nfi\n\nexit_if_not_root\n\nexit_if_no_permissions\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/enable.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-helm.wrapper", "content": "#!/usr/bin/env bash\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s helm on the master.\"\n exit 0\nfi\n\nexit_if_stopped\n\n\"${SNAP}/bin/helm\" --kubeconfig=\"${SNAP_DATA}\"/credentials/client.config \"$@\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-helm3.wrapper", "content": "#!/usr/bin/env bash\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s helm on the master.\"\n exit 0\nfi\n\nexit_if_stopped\n\n\"${SNAP}/bin/helm\" --kubeconfig=\"${SNAP_DATA}\"/credentials/client.config \"$@\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-images.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_no_permissions\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/images.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-istioctl.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\n. $SNAP/actions/common/utils.sh\nuse_snap_env\n\nif [ ! -f \"${SNAP_DATA}/bin/istioctl\" ]; then\n echo \"Istio not available, try enabling is with 'microk8s enable istio'\"\n exit 0\nfi\n\nsource $SNAP/actions/common/utils.sh\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s istioctl on the master.\"\n exit 0\nfi\n\nif echo \"$*\" | grep -v -q -- '--kubeconfig'; then\n exit_if_no_permissions\nfi\n\nARCH=$(arch)\nif ! [ \"${ARCH}\" = \"amd64\" ]\nthen\n echo \"Istio is not available for ${ARCH}\"\nelse\n exit_if_stopped\n \"${SNAP_DATA}/bin/istioctl\" --kubeconfig=${SNAP_DATA}/credentials/client.config \"$@\"\nfi\n" }, { "path": "microk8s-resources/wrappers/microk8s-join.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -d \"$SNAP_COMMON/default-storage\" ]\nthen\n echo \"WARNING: Hostpath storage is enabled and is not suitable for multi node clusters.\"\n echo \"\"\nfi\n\nexit_if_no_permissions\n\nrun_with_sudo ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/join.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-kubectl.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s kubectl on the master.\"\n exit 0\nfi\n\nexit_if_stopped\n\nif echo \"$*\" | grep -v -q -- '--kubeconfig'; then\n exit_if_no_permissions\nfi\n\nif [ -e $SNAP_DATA/args/kubectl-env ]\nthen\n source $SNAP_DATA/args/kubectl-env\nfi\n\nif is_strict\nthen\n export EDITOR=\"${SNAP}/bin/nano\"\nfi\n\ndeclare -a args=\"($(cat $SNAP_DATA/args/kubectl))\"\nif [ -n \"${args[@]-}\" ]\nthen\n \"${SNAP}/kubectl\" \"${args[@]}\" \"$@\"\nelse\n \"${SNAP}/kubectl\" \"$@\"\nfi\n" }, { "path": "microk8s-resources/wrappers/microk8s-leave.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_stopped\nexit_if_not_root\nexit_if_no_permissions\n\nif ! [ -e ${SNAP_DATA}/var/lock/no-cert-reissue ] ||\n [[ ! -e ${SNAP_DATA}/var/lock/clustered.lock &&\n ! -e ${SNAP_DATA}/args/k8s-dqlite ]]\nthen\n echo \"This MicroK8s deployment is not acting as a node in a cluster.\"\n exit 1\nfi\n\nrun_with_sudo preserve_env ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/leave.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-linkerd.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\n. $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s linkerd on the master.\"\n exit 0\nfi\n\nif [ ! -f \"${SNAP_DATA}/bin/linkerd\" ]; then\n echo \"Linkerd not available, try enabling Linkerd. 'microk8s enable linkerd' or 'microk8s enable linkerd:--proxy-auto-inject' \"\n exit 0\nfi\n\nif echo \"$*\" | grep -v -q -- '--kubeconfig'; then\n exit_if_no_permissions\nfi\n\nexit_if_stopped\n\n\"${SNAP_DATA}/bin/linkerd\" --kubeconfig=\"${SNAP_DATA}\"/credentials/client.config \"$@\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-refresh-certs.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s refresh-certs command on the master\"\n echo \"and then return to this node to perform a microk8s leave and re-join.\"\n exit 0\nfi\n\nexit_if_not_root\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/refresh_certs.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-remove-node.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_no_permissions\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster. Please use microk8s leave.\"\n exit 1\nfi\n\nif [ \"$#\" -eq 0 ]; then\n echo \"Please provide the node you want to remove.\"\n exit 1\nfi\n\nif [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ] && [ \"$#\" -eq 2 ] && ! [ \"$2\" == \"--force\" ] ; then\n echo \"Please provide the node and the optional --force flag.\"\n exit 1\nfi\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/remove_node.py \"$@\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-reset.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\n. $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/reset.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-start.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster.\"\n echo \"Use 'snap start microk8s' to start services on this node.\"\n exit 0\nfi\n\nexit_if_not_root\nexit_if_no_permissions\n\nPARSED=$(getopt --options=lho: --longoptions=help,output:,disable-low-memory-guard --name \"$@\" -- \"$@\")\neval set -- \"$PARSED\"\nwhile true; do\n case \"$1\" in\n --disable-low-memory-guard)\n rm \"${SNAP_DATA}/var/lock/low-memory-guard.lock\" || true\n shift\n ;;\n -h|--help)\n echo \"Usage: microk8s start [OPTIONS]\"\n echo\n echo \"Start Kubernetes services\"\n echo\n echo \"Options:\"\n echo \" -h, --help Show this help\"\n echo \" --disable-low-memory-guard Start MicroK8s in machines with RAM < 512MB\"\n exit 0\n ;;\n --)\n shift\n break\n ;;\n *)\n echo \"microk8s start: invalid option -- $1\"\n exit 1\n esac\ndone\n\nexit_if_low_memory_guard\n\n\nif ! run_with_sudo preserve_env snapctl start ${SNAP_NAME} --enable\nthen\n echo 'Failed to start microk8s services. Check snapd logs with \"journalctl -u snapd.service\"'\n exit 1\nelse\n start_all_containers\n if run_with_sudo test -e ${SNAP_DATA}/var/lock/stopped.lock\n then\n # Mark the api server as starting\n run_with_sudo rm -f ${SNAP_DATA}/var/lock/stopped.lock &> /dev/null\n fi\nfi\n\nwait_for_node\n" }, { "path": "microk8s-resources/wrappers/microk8s-status.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_no_permissions\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster.\"\n echo \"Please use the control plane node.\"\n exit 0\nfi\n\nexit_if_low_memory_guard\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/status.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s-stop.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"This MicroK8s deployment is acting as a node in a cluster.\"\n echo \"Use 'snap stop microk8s' to stop services on this node.\"\n exit 0\nfi\n\nexit_if_not_root\nexit_if_no_permissions\n\nFORCE=false\nPARSED=$(getopt --options=lho: --longoptions=force,help,output: --name \"$@\" -- \"$@\")\neval set -- \"$PARSED\"\nwhile true; do\n case \"$1\" in\n -h|--help)\n echo \"Usage: microk8s stop [OPTIONS]\"\n echo\n echo \"Stop Kubernetes services\"\n echo\n echo \"Options:\"\n echo \" -h, --help Show this help\"\n exit 0\n ;;\n --)\n shift\n break\n ;;\n *)\n echo \"microk8s stop: invalid option -- $1\"\n exit 1\n esac\ndone\n\nprefix_cmd=\"run_with_sudo snap\"\nif is_strict\nthen\n prefix_cmd=\"snapctl\"\nfi\n\n$prefix_cmd stop microk8s.daemon-kubelite --disable\nstop_status=$?\n\nif ! [ $stop_status -eq 0 ]\nthen\n echo 'Failed to stop microk8s services. Check snapd logs with \"journalctl -u snapd.service\"'\n exit 1\nelse\n remove_all_containers\n kill_all_container_shims\n $prefix_cmd stop microk8s --disable\n run_with_sudo touch ${SNAP_DATA}/var/lock/stopped.lock\nfi\n" }, { "path": "microk8s-resources/wrappers/microk8s-version.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_no_permissions\n\n${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/version.py \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/microk8s.wrapper", "content": "#!/usr/bin/env bash\n\nset -e\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nfunction help() {\n echo \"Available subcommands are:\"\n for i in ${SNAP}/microk8s-*.wrapper; do\n echo -e \"\\t$(basename ${i} | sed 's/microk8s-//g' | sed 's/.wrapper//g')\"\n done\n echo -e \"\\tinspect\"\n plugins=\"$(find ${SNAP_COMMON}/plugins/* 2> /dev/null)\"\n if [ ! -z \"$plugins\" ]; then\n echo \"Available subcommands from addons are:\"\n for i in \"${SNAP_COMMON}/plugins/\"*; do\n echo -e \"\\t$(basename \"${i}\")\"\n done\n fi\n}\n\nif [ -z \"$1\" ]; then\n help\n exit 1\nfi\n\nreadonly APP=\"$1\"\nshift\n\nif [ -f \"${SNAP}/microk8s-${APP}.wrapper\" ]; then\n \"${SNAP}/microk8s-${APP}.wrapper\" \"$@\"\n readonly EXIT=\"$?\"\nelif [ \"${APP}\" == \"inspect\" ]; then\n run_with_sudo preserve_env ${SNAP}/inspect.sh \"$@\"\n readonly EXIT=\"$?\"\nelif [ \"${APP}\" == \"help\" ] || [ \"${APP}\" == \"--help\" ] || [ \"$APP\" == \"-h\" ]; then\n help\n readonly EXIT=\"0\"\nelif [ -f \"${SNAP_COMMON}/plugins/${APP}\" ]; then\n \"${SNAP_COMMON}/plugins/${APP}\" \"$@\"\n\n readonly EXIT=\"$?\"\nelse\n echo \"'${APP}' is not a valid MicroK8s subcommand.\"\n help\n readonly EXIT=\"1\"\nfi\n\nexit ${EXIT}\n" }, { "path": "microk8s-resources/wrappers/openssl.wrapper", "content": "#!/usr/bin/env bash\n\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexport OPENSSL_CONF=\"${SNAP}/etc/ssl/openssl.cnf\"\n\n\"${SNAP}/usr/bin/openssl\" \"${@}\"\n" }, { "path": "microk8s-resources/wrappers/run-apiserver-proxy-with-args", "content": "#!/usr/bin/env bash\nset -ex\n\nsource $SNAP/actions/common/utils.sh\nuse_snap_env\n\nif ! [ -e \"$SNAP_DATA/var/lock/clustered.lock\" ]; then\n echo \"Not a worker node, exiting\"\n exit 0\nfi\n\nif ! [ -e \"$SNAP_DATA/args/traefik\" ]\nthen\n exit 0\nfi\n\nif ! [ -e \"$SNAP_DATA/args/apiserver-proxy\" ]\nthen\n exit 0\nfi\n\nsed 's@${SNAP}@'\"${SNAP}\"'@g;s@${SNAP_DATA}@'\"${SNAP_DATA}\"'@g' $SNAP_DATA/args/traefik/traefik-template.yaml > $SNAP_DATA/args/traefik/traefik.yaml\n\n# This is really the only way I could find to get the args passed in correctly.\ndeclare -a args=\"($(cat $SNAP_DATA/args/apiserver-proxy))\"\nexec \"$SNAP/bin/cluster-agent\" apiserver-proxy \"${args[@]}\"\n" }, { "path": "microk8s-resources/wrappers/run-cluster-agent-with-args", "content": "#!/usr/bin/env bash\n\nif [ -z \"$HOME\" ]\nthen\n mkdir -p $SNAP_DATA/var/tmp/\n export HOME=$SNAP_DATA/var/tmp/\nfi\n\nset -eu\n\n. $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\n# This is really the only way I could find to get the args passed in correctly.\ndeclare -a args=\"($(cat $SNAP_DATA/args/cluster-agent))\"\n\n${SNAP}/bin/cluster-agent cluster-agent \"${args[@]}\"\n" }, { "path": "microk8s-resources/wrappers/run-containerd-with-args", "content": "#!/usr/bin/env bash\n\nset -ex\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\n# Re-exec outside of apparmor confinement\nif ! is_strict && [ -d /sys/kernel/security/apparmor ]; then\n status_file=/proc/self/attr/apparmor/current\n # Fallback for older kernels, check the legacy interfaces\n # https://www.kernel.org/doc/html/latest/admin-guide/LSM/index.html\n if [ ! -f $status_file ]; then\n status_file=/proc/self/attr/current\n fi\n if [ \"$(cat $status_file)\" != \"unconfined\" ]; then\n exec aa-exec -p unconfined -- \"$0\" \"$@\"\n fi\nfi\n\n# Why we put the /snap/microk8s/current in the path?\n# containerd-shims need to call runc. They inherit their PATH from containerd.\n# As the snap refreshes runc changes location, eg moves from\n# /snap/microk8s/123/usr/bin/runc to /snap/microk8s/124/usr/runc.\n# containerd-shims need to look for runc in /snap/microk8s/current/usr/bin/runc\nSNAP_CURRENT=`echo \"${SNAP}\" | sed -e \"s,${SNAP_REVISION},current,\"`\nCURRENT_PATH=\"$SNAP_CURRENT/usr/sbin:$SNAP_CURRENT/usr/bin:$SNAP_CURRENT/sbin:$SNAP_CURRENT/bin\"\nexport PATH=\"$CURRENT_PATH:$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH\"\n\nif is_strict\nthen\n apparmor_parser -r $SNAP/containerd-profile\nelse\n if [ -d \"/etc/apparmor.d\" ]; then\n echo \"Using a default profile template\"\n cp ${SNAP}/containerd-profile /etc/apparmor.d/cri-containerd.apparmor.d\n echo \"Reloading AppArmor profiles\"\n if ! service apparmor reload\n then\n echo \"AppArmor profiles loading failed. AppArmor may be unavailable on this host.\"\n fi\n fi\nfi\n\napp=containerd\n\n\nRUNTIME=\"runc\"\nRUNTIME_TYPE=\"io.containerd.runc.v1\"\nSNAPSHOTTER=$(snapshotter)\n\nif mount -t cgroup2 | grep -q 'type cgroup2'; then\n RUNTIME_TYPE=\"io.containerd.runc.v2\"\nfi\n\nsed 's@${SNAP}@'\"${SNAP}\"'@g;s@${SNAP_DATA}@'\"${SNAP_DATA}\"'@g;s@${SNAPSHOTTER}@'\"${SNAPSHOTTER}\"'@g;s@${RUNTIME}@'\"${RUNTIME}\"'@g' $SNAP_DATA/args/containerd-template.toml > $SNAP_DATA/args/containerd.toml\nsed -i 's@${RUNTIME_TYPE}@'\"${RUNTIME_TYPE}\"'@g' $SNAP_DATA/args/containerd.toml\n\n\nrun_flanneld=\"$(is_service_expected_to_start flanneld)\"\nif [ \"${run_flanneld}\" == \"1\" ]\nthen\n sed 's@${SNAP}@'\"${SNAP}\"'@g;s@${SNAP_DATA}@'\"${SNAP_DATA}\"'@g;s@${SNAP_COMMON}@'\"${SNAP_COMMON}\"'@g' $SNAP_DATA/args/flannel-template.conflist > $SNAP_DATA/args/cni-network/flannel.conflist\nfi\n\n# clean leftover container state if we just booted\nif (is_strict && is_first_boot_on_strict) || (! is_strict && is_first_boot \"${SNAP_COMMON}/run/containerd\")\nthen\n rm -rf \"${SNAP_COMMON}/run/containerd\" || true\nfi\n\nmkdir -p \"${SNAP_COMMON}/run/containerd\"\nif ! is_strict\nthen\n mark_boot_time \"${SNAP_COMMON}/run/containerd\"\nfi\n\n# This is really the only way I could find to get the args passed in correctly.\ndeclare -a args=\"($(cat $SNAP_DATA/args/$app))\"\nset -a\n. \"${SNAP_DATA}/args/${app}-env\"\nset +a\n\n# check if there is a default route available\nif ! default_route_exists\nthen\n echo \"WARNING: No default route exists. MicroK8s might not work properly.\"\n echo \"Refer to https://microk8s.io/docs for instructions on air-gap deployments.\"\nfi\n\n# NOTE(neoaggelos): See https://github.com/canonical/microk8s/issues/423 for context\nwait_for_default_route\n\n# Set the path to the Cilium socket correctly for CNI\nexport CILIUM_SOCK=\"${SNAP_DATA}/var/run/cilium/cilium.sock\"\n\nexec \"$SNAP/bin/$app\" \"${args[@]}\"\n" }, { "path": "microk8s-resources/wrappers/run-etcd-with-args", "content": "#!/usr/bin/env bash\n\nset -e\n\nsource $SNAP/actions/common/utils.sh\nuse_snap_env\n\nexit_if_service_not_expected_to_start etcd\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n echo \"etcd will not run on a cluster node\"\n exit 0\nfi\n\n# etcd will not start if the socket already exists.\nif [ -S \"${SNAP_DATA}/etcd.socket:2379\" ]; then\n rm \"${SNAP_DATA}/etcd.socket:2379\"\nfi\n\nARCH=$(arch)\n\nif ! [ \"$ARCH\" = \"amd64\" ]; then\n export ETCD_UNSUPPORTED_ARCH=\"$ARCH\"\nfi\n\nexport DEFAULT_INTERFACE_IP_ADDR=\"$(get_default_ip)\"\n\n# This is really the only way I could find to get the args passed in correctly.\ndeclare -a args=\"($(cat $SNAP_DATA/args/etcd))\"\nexec \"$SNAP/etcd\" \"${args[@]}\"\n" }, { "path": "microk8s-resources/wrappers/run-flanneld-with-args", "content": "#!/usr/bin/env bash\nset -e\n\nsource $SNAP/actions/common/utils.sh\nuse_snap_env\n\nexit_if_service_not_expected_to_start flanneld\n\n# Allow some slack for containerd and etcd to start\n# so we avoid this edge case: https://forum.snapcraft.io/t/restarting-services-from-configure-hook-race-condition/2513/13\nsleep 5\n\nn=0\nuntil [ $n -ge 10 ]\ndo\n test -e \"$SNAP_DATA/args/flannel-network-mgr-config\" && test -e \"$SNAP_DATA/args/flanneld\" && break\n echo \"Waiting for flannled configuration to appear. (attempt $n)\"\n n=$[$n+1]\n sleep 2\ndone\n\n# TODO rewrite for snaps\netcd_endpoints=\"$(cat $SNAP_DATA/args/flanneld | grep \"etcd-endpoints\" | tr \"=\" \" \"| awk '{print $2}')\"\ncert_file=\"$(cat $SNAP_DATA/args/flanneld | grep \"etcd-certfile\" | tr \"=\" \" \"| awk '{print $2}')\"\ncert_file=\"$(eval echo $cert_file)\"\nkey_file=\"$(cat $SNAP_DATA/args/flanneld | grep \"etcd-keyfile\" | tr \"=\" \" \"| awk '{print $2}')\"\nkey_file=\"$(eval echo $key_file)\"\nca_file=\"$(cat $SNAP_DATA/args/flanneld | grep \"etcd-cafile\" | tr \"=\" \" \"| awk '{print $2}')\"\nca_file=\"$(eval echo $ca_file)\"\nexport ETCDCTL_API=3\n\n# TODO get this from a file\ndata=\"$(cat $SNAP_DATA/args/flannel-network-mgr-config)\"\n\n# Prepare etcd configuration for flannel, iff an etcd endpoint is set.\n# Skip this part if an alternate data store is used (e.g. Kubernetes).\nif [ ! -z \"$etcd_endpoints\" ]; then\n if ! \"${SNAP}/etcdctl\" --endpoints \"${etcd_endpoints}\" --cert \"${cert_file}\" --key \"${key_file}\" --cacert \"${ca_file}\" del \"/coreos.com/network/config\"; then\n echo \"/coreos.com/network/config is not in etcd. Probably a first time run.\"\n fi\n \"${SNAP}/etcdctl\" --endpoints \"${etcd_endpoints}\" --cert \"${cert_file}\" --key \"${key_file}\" --cacert \"${ca_file}\" put \"/coreos.com/network/config\" \"$data\"\nfi\n\nset -a\nif [ -e \"${SNAP_DATA}/args/flanneld-env\" ]\nthen\n . \"${SNAP_DATA}/args/flanneld-env\"\nfi\nset +a\n\n# This is really the only way I could find to get the args passed in correctly.\ndeclare -a args=\"($(cat $SNAP_DATA/args/flanneld))\"\nexec \"$SNAP_DATA/opt/cni/bin/flanneld\" \"${args[@]}\"\n" }, { "path": "microk8s-resources/wrappers/run-k8s-dqlite-with-args", "content": "#!/usr/bin/env bash\n\nset -ex\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nexit_if_service_not_expected_to_start k8s-dqlite\n\nif [ -e \"${SNAP_DATA}/var/lock/low-memory-guard.lock\" ]\nthen\n\techo \"not starting dqlite because of low memory guard lock\"\n\texit 0\nfi\n\napp=k8s-dqlite\n\nif ! [ -e \"$SNAP_DATA/args/${app}\" ]\nthen\n exit 0\nfi\n\n# We add some delay so that systemd really retries the restarts\nsleep 6\n\nif [ ! -e \"${SNAP_DATA}/var/lock/skip-aio-tune.lock\" ]\nthen\n increase_sysctl_parameter \"fs.aio-max-nr\" \"1048576\"\nfi\n\nif [ ! -e \"${SNAP_DATA}/var/lock/skip-inotify-tune.lock\" ]\nthen\n increase_sysctl_parameter \"fs.inotify.max_user_instances\" \"1024\"\n increase_sysctl_parameter \"fs.inotify.max_user_watches\" \"1048576\"\nfi\n\nset -a\nif [ -e \"${SNAP_DATA}/args/${app}-env\" ]\nthen\n . \"${SNAP_DATA}/args/${app}-env\"\nfi\nset +a\n\ndeclare -a args=\"($(cat $SNAP_DATA/args/$app))\"\nexec \"$SNAP/bin/$app\" \"${args[@]}\"\n" }, { "path": "microk8s-resources/wrappers/run-kubelite-with-args", "content": "#!/usr/bin/env bash\n\nset -ex\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\napp=kubelite\n\nif ! [ -e ${SNAP_DATA}/var/lock/lite.lock ]\nthen\n echo \"${app} will not run\"\n exit 0\nfi\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n refresh_opt_in_local_config \"start-control-plane\" \"false\" kubelite\nelif [ -e ${SNAP_DATA}/var/lock/low-memory-guard.lock ]\nthen\n echo \"${app} will not run, memory guard is enabled\"\n exit 0\nelse\n refresh_opt_in_local_config \"start-control-plane\" \"true\" kubelite\nfi\n\nif [ -e ${SNAP_DATA}/var/lock/stopped.lock ]\nthen\n # Mark the cluster as starting. This is needed in case you\n # microk8s stop and then snap start microk8s\n rm -f ${SNAP_DATA}/var/lock/stopped.lock &> /dev/null\nfi\n\n## API server configuration\n\n# Check if we advertise an address. If we do we do not need to wait for a default network interface.\nif ! grep -E \"(--advertise-address|--bind-address)\" $SNAP_DATA/args/kube-apiserver &> /dev/null\nthen\n # we keep this command to cleanup the file, as it is no longer needed.\n rm -f ${SNAP_DATA}/external_ip.txt\n\n # check if there is a default route available\n if ! default_route_exists\n then\n echo \"WARNING: No default route exists. MicroK8s might not work properly.\"\n echo \"Refer to https://microk8s.io/docs for instructions on air-gap deployments.\"\n fi\n\n wait_for_default_route\nfi\n\nif [ -e ${SNAP_DATA}/args/ha-conf ]\nthen\n storage_param=\"$(get_opt_in_config '--storage-dir' 'k8s-dqlite')\"\n storage_dir=\"$(eval echo $storage_param)\"\n if $(grep -qE \"^failure-domain\" \"${SNAP_DATA}/args/ha-conf\"); then\n val=\"$(get_opt_in_config 'failure-domain' 'ha-conf')\"\n echo \"$val\" > $storage_dir/failure-domain\n fi\nfi\n\n# Sanitise arguments\nif [ -e $SNAP_DATA/var/lock/no-arg-sanitisation ]\nthen\n echo \"Skipping argument sanitisation.\"\nelse\n echo \"Sanitise arguments.\"\n sanitise_args_kubeapi_server\n sanitise_args_kubelet\n sanitise_args_kube_proxy\n sanitise_args_kube_scheduler\n sanitise_args_kube_controller_manager\nfi\n\n## Kubelet configuration\npod_cidr=\"$(cat $SNAP_DATA/args/kube-proxy | grep \"cluster-cidr\" | tr \"=\" \" \"| gawk '{print $2}')\"\nif [ -z \"$pod_cidr\" ]\nthen\n pod_cidr=\"$(cat $SNAP_DATA/args/kubelet | grep \"pod-cidr\" | tr \"=\" \" \"| gawk '{print $2}')\"\n if [ -z \"$pod_cidr\" ]\n then\n pod_cidr=\"$(jq .Network $SNAP_DATA/args/flannel-network-mgr-config | tr -d '\\\"')\"\n fi\nfi\nif ! [ -z \"$pod_cidr\" ]\nthen\n if ! iptables -C FORWARD -s \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT\n then\n # The point of \"|| true\" is that if for any reason this is failing we \"fallback\"\n # to the previous manual approach of pointing people to the troubleshooting guide.\n iptables -t filter -A FORWARD -s \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT || true\n iptables -t filter -A FORWARD -d \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT || true\n iptables-nft -t filter -A FORWARD -s \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT || true\n iptables-nft -t filter -A FORWARD -d \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT || true\n fi\nfi\n\n# Configure host resolv.conf file with non-loopback upstream nameservers\nresolv_conf=\"$(cat $SNAP_DATA/args/kubelet | grep -- \"--resolv-conf\" | tr \"=\" \" \" | gawk '{print $2}')\"\nif [ -z \"$resolv_conf\" ]\nthen\n host_resolv_conf=\"$(\"$SNAP/usr/bin/python3\" \"$SNAP/scripts/find-resolv-conf.py\")\"\n if [ ! -z \"$host_resolv_conf\" ]; then\n refresh_opt_in_local_config \"resolv-conf\" \"${host_resolv_conf}\" \"kubelet\"\n echo \"Configured kubelet to use ${host_resolv_conf} for DNS configuration\"\n fi\nfi\n\n#UFW configuration\nif ! is_strict && (ufw version &> /dev/null)\nthen\n if ufw status | grep -q \"Status: active\" &&\n ! [ -e ${SNAP_DATA}/var/lock/skip.ufw ]\n then\n # These succeed regardless of whether the rule exists already or not\n echo \"Found enabled UFW: adding rules to allow in/out traffic on 'cali+' and 'vxlan.calico' devices\"\n if ! ufw allow in on vxlan.calico ||\n ! ufw allow out on vxlan.calico ||\n ! ufw allow in on cali+ ||\n ! ufw allow out on cali+\n then\n echo \"Failed to update UFW rules. You may want to set them manually.\"\n fi\n fi\nfi\n\nif ! is_strict &&\n (systemctl show snap.microk8s.daemon-kubelite.service -p NRestarts | grep -qv \"NRestarts=0\") &&\n (grep -qv cpuset /sys/fs/cgroup/cgroup.subtree_control) &&\n [ ! -e /etc/systemd/system/snap.microk8s.daemon-kubelite.service.d/delegate.conf ]\nthen\n mkdir -p /etc/systemd/system/snap.microk8s.daemon-kubelite.service.d\n tee /etc/systemd/system/snap.microk8s.daemon-kubelite.service.d/delegate.conf > /dev/null < /dev/null\nthen\n socket=$(grep -e \"--address \" $SNAP_DATA/args/containerd | gawk '{print $2}')\n # socket_file may be of the form ${SNAP_DATA}/containerd.sock\n # we need to replace any variables\n socket_file_expand=$(eval echo ${socket})\n # wait up until 20 seconds for the docker socket to appear\n n=0\n until [ $n -ge 10 ]\n do\n test -S \"${socket_file_expand}\" && break\n echo \"Waiting for containerd socket ${socket_file_expand} to appear. (attempt $n)\"\n n=$[$n+1]\n sleep 2\n done\nfi\n\n# Handle non-unified cgroups https://github.com/canonical/microk8s/issues/519\nif [ -e /proc/$$/cgroup ] &&\n [[ $(gawk -F '[:]' '(/cpu/ && !/cpuset/) || /memory/ {print $3}' /proc/$$/cgroup | uniq | wc -l) -eq \"2\" ]] &&\n ! grep -e \"runtime-cgroups\" $SNAP_DATA/args/kubelet &> /dev/null &&\n ! grep -e \"kubelet-cgroups\" $SNAP_DATA/args/kubelet &> /dev/null &&\n [ -e /sys/fs/cgroup/systemd/system.slice ]\nthen\n refresh_opt_in_local_config \"runtime-cgroups\" \"/systemd/system.slice\" kubelet\n refresh_opt_in_local_config \"kubelet-cgroups\" \"/systemd/system.slice\" kubelet\nfi\n\nif [ -L /var/lib/kubelet ]\nthen\n echo \"\\`/var/lib/kubelet\\` is a symbolic link\"\n ls -l /var/lib/kubelet\nelse\n echo \"\\`/var/lib/kubelet\\` already exists. CSI add-ons have to point to $SNAP_COMMON for kubelet.\"\nfi\n\n## Kube-proxy configuration\n\nif [ -e \"$SNAP_DATA/args/cni-network/cni.yaml\" ]\nthen\n ipvs=\"ipv4 ipv6\"\n for ipv in $ipvs\n do\n if [ -e \"/proc/sys/net/$ipv/conf/all/forwarding\" ] &&\n ! grep -e \"1\" \"/proc/sys/net/$ipv/conf/all/forwarding\"\n then\n echo \"Enable ip forwarding for $ipv\"\n echo \"1\" > \"/proc/sys/net/$ipv/conf/all/forwarding\"\n fi\n done\nfi\n\nif [ -f \"${SNAP_DATA}/var/lock/host-access-enabled\" ] &&\n ! ip link show lo:microk8s &> /dev/null\nthen\n IP_ADDRESS=$(<\"${SNAP_DATA}/var/lock/host-access-enabled\")\n if ! \"${SNAP}/sbin/ip\" addr add \"${IP_ADDRESS}\" dev lo label lo:microk8s\n then\n echo \"Failed to enable host-access\"\n else\n echo \"Host-access enabled [${IP_ADDRESS}]\"\n fi\nfi\n\n# kube-proxy reads some values related to the 'nf_conntrack' kernel\n# module from procfs on startup, so we must ensure it is loaded:\nif ! [ -f /proc/sys/net/netfilter/nf_conntrack_max ]\nthen\n if /sbin/modprobe nf_conntrack || modprobe nf_conntrack\n then\n echo \"Successfully loaded nf_conntrack module.\"\n else\n echo -n \"Failed to load nf_conntrack kernel module. \"\n echo \"ProxyServer will fail to start until it's loaded.\"\n fi\nfi\n\n# on lxc containers do not try to change the conntrack configuration\n# see https://github.com/canonical/microk8s/issues/1438\nif grep -E lxc /proc/1/environ &&\n ! grep -E \"conntrack-max-per-core\" $SNAP_DATA/args/kube-proxy\nthen\n refresh_opt_in_local_config \"conntrack-max-per-core\" \"0\" kube-proxy\nfi\n\nif ! [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]\nthen\n # NOTE(neoaggelos): https://github.com/canonical/microk8s/issues/3085\n # Attempt to use modprobe from the host, otherwise fallback to the one\n # provided with the snap.\n if /sbin/modprobe br_netfilter || modprobe br_netfilter\n then\n echo \"Successfully loaded br_netfilter module.\"\n else\n echo \"Failed to load br_netfilter. Calico might not work properly.\"\n fi\nfi\n\nif [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ] &&\n grep 0 /proc/sys/net/bridge/bridge-nf-call-iptables\nthen\n echo \"1\" > /proc/sys/net/bridge/bridge-nf-call-iptables\nfi\n\ndeclare -a args=\"($(cat $SNAP_DATA/args/$app))\"\nexec \"$SNAP/$app\" \"${args[@]}\"\n" }, { "path": "pyproject.toml", "content": "[tool.black]\nline-length = 100\ntarget-version = ['py35']\n" }, { "path": "scripts/calico/upgrade.py", "content": "#!/usr/bin/python3\nimport os\nimport yaml\nimport shutil\nimport sys\n\n\ndef get_calico_node_spec(cni_file):\n \"\"\"\n Extract the section of the calico node container\n\n return: The container section of the calico node, None otherwise\n \"\"\"\n try:\n with open(cni_file, \"r\", encoding=\"utf8\") as f:\n for doc in yaml.safe_load_all(f):\n if doc and doc[\"kind\"] == \"DaemonSet\" and doc[\"metadata\"][\"name\"] == \"calico-node\":\n # Reach for the containers\n if (\n doc[\"spec\"]\n and doc[\"spec\"][\"template\"]\n and doc[\"spec\"][\"template\"][\"spec\"]\n and doc[\"spec\"][\"template\"][\"spec\"][\"containers\"]\n ):\n containers = doc[\"spec\"][\"template\"][\"spec\"][\"containers\"]\n for c in containers:\n if c[\"name\"] == \"calico-node\":\n return c\n except (yaml.YAMLError, TypeError) as e:\n print(e, file=sys.stderr)\n return None\n return None\n\n\ndef is_calico_cni_manifest(cni_file):\n \"\"\"\n Check if this is a Calico CNI manifest\n\n return: True if the provided manifest is a Calico one, False otherwise\n \"\"\"\n try:\n with open(cni_file, \"r\", encoding=\"utf8\") as f:\n for doc in yaml.safe_load_all(f):\n if doc and doc[\"kind\"] == \"DaemonSet\" and doc[\"metadata\"][\"name\"] == \"calico-node\":\n return True\n except (yaml.YAMLError, TypeError) as e:\n print(e, file=sys.stderr)\n return False\n return False\n\n\ndef get_installed_version_of_calico(cni_file):\n \"\"\"\n Extract the Calico version in the provided CNI manifest\n\n return: A string with the version, None otherwise\n \"\"\"\n try:\n c = get_calico_node_spec(cni_file)\n if c:\n parts = c[\"image\"].split(\":\")\n return parts[-1]\n else:\n return None\n except (yaml.YAMLError, TypeError) as e:\n print(e, file=sys.stderr)\n return None\n\n\ndef get_calicos_autodetection_method(cni_file):\n \"\"\"\n Extract the IP autodetection method\n\n return: A string with the IP autodetection method, None otherwise\n \"\"\"\n try:\n c = get_calico_node_spec(cni_file)\n if c:\n methods = [i[\"value\"] for i in c[\"env\"] if i[\"name\"] == \"IP_AUTODETECTION_METHOD\"]\n if len(methods) > 0:\n return methods[0]\n return None\n except (yaml.YAMLError, TypeError) as e:\n print(e, file=sys.stderr)\n return None\n\n\ndef patch_manifest(cni_file, autodetection):\n \"\"\"\n Patch the CNI manifest with the IP autodetection method provided\n \"\"\"\n yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str\n\n def repr_str(dumper, data):\n if \"\\n\" in data:\n return dumper.represent_scalar(\"tag:yaml.org,2002:str\", data, style=\"|\")\n return dumper.org_represent_str(data)\n\n yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)\n\n try:\n with open(cni_file, \"r\", encoding=\"utf8\") as f:\n to_remove = []\n docs = list(yaml.safe_load_all(f))\n for doc in docs:\n if not doc:\n to_remove.append(doc)\n if doc and doc[\"kind\"] == \"DaemonSet\" and doc[\"metadata\"][\"name\"] == \"calico-node\":\n # Reach for the containers\n containers = doc[\"spec\"][\"template\"][\"spec\"][\"containers\"]\n for c in containers:\n if c[\"name\"] == \"calico-node\":\n env = c[\"env\"]\n for variable in env:\n if variable[\"name\"] == \"IP_AUTODETECTION_METHOD\":\n variable[\"value\"] = autodetection\n\n # remove empty yaml documents\n for d in to_remove:\n docs.remove(d)\n\n with open(cni_file, \"w\", encoding=\"utf8\") as fout:\n yaml.safe_dump_all(docs, fout)\n\n except (yaml.YAMLError, TypeError) as e:\n print(e, file=sys.stderr)\n\n\ndef backup_old_cni(cni_file):\n \"\"\"\n Creating a backup of the provided file\n \"\"\"\n backup_cni_file = f\"{cni_file}.backup\"\n shutil.copyfile(cni_file, backup_cni_file)\n\n\ndef try_upgrade(cni_file, new_cni_file, cni_no_manage=None):\n \"\"\"\n Perform the upgrade if possible.\n\n return: True if the CNI needs to be reloaded\n \"\"\"\n\n # If cni auto management is disabled by lock file do nothing\n if cni_no_manage is not None and os.path.exists(cni_no_manage):\n return False\n\n # If cni files are not in place do nothing\n if not (os.path.exists(cni_file) and os.path.exists(new_cni_file)):\n return False\n\n # If the current cni.yaml is not calico do nothing\n if not is_calico_cni_manifest(cni_file):\n return False\n\n # If the current cni.yaml is not from 3.21 do nothing\n # s390x will be filtered out because it is in 3.15\n current_version = get_installed_version_of_calico(cni_file)\n if \"3.21\" not in current_version:\n return False\n\n backup_old_cni(cni_file)\n autodetection = get_calicos_autodetection_method(cni_file)\n shutil.copyfile(new_cni_file, cni_file)\n if autodetection and \"can-reach\" in autodetection:\n patch_manifest(cni_file, autodetection)\n\n return True\n\n\ndef mark_apply_needed(lock_file):\n \"\"\"\n Remove the lock file provided so the apiserver kicker will re apply the CNI manifest\n \"\"\"\n try:\n os.remove(lock_file)\n except OSError:\n pass\n\n\ndef main():\n \"\"\"\n Run the upgrade.\n\n :return: None\n \"\"\"\n cni_reapply_lock_file = os.path.expandvars(\"${SNAP_DATA}/var/lock/cni-loaded\")\n cni_file = os.path.expandvars(\"${SNAP_DATA}/args/cni-network/cni.yaml\")\n cni_no_manage = os.path.expandvars(\"${SNAP_DATA}/var/lock/no-manage-calico\")\n new_cni_file = os.path.expandvars(\n \"${SNAP}/upgrade-scripts/000-switch-to-calico/resources/calico.yaml\"\n )\n\n if try_upgrade(cni_file, new_cni_file, cni_no_manage):\n # we mark the CNI needs to be updated so the api service kicker will take over\n mark_apply_needed(cni_reapply_lock_file)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "scripts/find-resolv-conf.py", "content": "#!/usr/bin/env python3\n\nimport ipaddress\nimport re\n\nimport click\n\n\n# https://regex101.com/r/RWZH94/1\nNAMESERVER_REGEX = r\"^\\s*nameserver\\s+(\\S*)\\s*$\"\n\nDEFAULT_RESOLV_CONFS = [\n \"/etc/resolv.conf\",\n \"/run/systemd/resolve/resolv.conf\",\n]\n\n\ndef safe_is_non_loopback_address(address: str):\n \"\"\"\n Return true if the given address is a valid non loopback address. Returns\n false if the address is loopback or invalid\n \"\"\"\n try:\n return not ipaddress.ip_address(address).is_loopback\n except ValueError:\n # NOTE(neoaggelos): https://github.com/canonical/microk8s/issues/4327\n # Python 3.8 fails with scoped IPv6 address, e.g. \"fe80::5054:ff:fe00:b61d%2\"\n # Try to remove the scope suffix, and accept if value is an IPv6 address\n if \"%\" not in address:\n return False\n\n try:\n ip = ipaddress.ip_address(address[: address.find(\"%\")])\n return ip.version == 6 and not ip.is_loopback\n except (ValueError, IndexError):\n return False\n\n\ndef find_resolv_conf_with_non_loopback_address(resolv_confs: list):\n \"\"\"\n Given a list of resolv.conf file paths, return the first one that contains non-loopback\n upstream nameservers.\n \"\"\"\n for path in resolv_confs:\n try:\n with open(path) as fin:\n contents = fin.read()\n\n nameservers = re.findall(NAMESERVER_REGEX, contents, re.MULTILINE)\n\n if nameservers and all(map(safe_is_non_loopback_address, nameservers)):\n return path\n except (OSError, ValueError):\n # ignore invalid resolv.conf files\n pass\n\n\n@click.command(\"find-resolv-conf\")\n@click.argument(\"resolv_confs\", nargs=-1)\ndef main(resolv_confs):\n \"\"\"\n find-resolv-conf looks in the system for a resolv.conf file that contains non-loopback\n upstream nameservers. If there are any, the first one found is printed to stdout.\n\n Paths to resolv.conf files may be given as arguments. By default, known system locations\n defined in DEFAULT_RESOLV_CONFS are checked.\n\n If no resolv.conf file with non-loopback nameservers is found, nothing is printed to stdout.\n \"\"\"\n path = find_resolv_conf_with_non_loopback_address(resolv_confs or DEFAULT_RESOLV_CONFS)\n if path:\n print(path)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "scripts/generate-cni.sh", "content": "#!/usr/bin/env bash\n\nset -x\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nif [ -e \"${SNAP_DATA}/args/cni-env\" ]; then\n source \"${SNAP_DATA}/args/cni-env\"\nfi\n\nCNI_YAML=\"${SNAP_DATA}/args/cni-network/cni.yaml\"\nCALICO_RESOURCES=\"${SNAP}/upgrade-scripts/000-switch-to-calico/resources\"\n\nCLUSTER_CIDR=\"\"\nSERVICE_CIDR=\"\"\n\n\nfunction handle_calico {\n # Start with the default CNI and patch it based on the user needs\n cp \"${CALICO_RESOURCES}/calico.yaml\" \"$CNI_YAML\"\n\n # IPv4 configuration\n if test \"x${IPv4_SUPPORT}\" = \"xtrue\"; then\n # Update the calico cni.yaml with IPv4 specific configurations\n sed -i '/\"type\": \"calico-ipam\"/i \\ \"assign_ipv4\": \"true\",' \"$CNI_YAML\"\n sed -i 's%10.1.0.0/16%'\"${IPv4_CLUSTER_CIDR}\"'%g' \"$CNI_YAML\"\n fi\n\n # IPv6 configuration\n if test \"x${IPv6_SUPPORT}\" = \"xtrue\"; then\n sed -i '/\"type\": \"calico-ipam\"/i \\ \"assign_ipv6\": \"true\",' \"$CNI_YAML\"\n sed -i '/FELIX_IPV6SUPPORT/{n;s/.*/\\ value: \"true\"/}' \"$CNI_YAML\"\n sed -i '/CALICO_IPV6POOL_VXLAN/{n;s/.*/\\ value: \"Always\"/}' \"$CNI_YAML\"\n sed -i '/Enable or Disable VXLAN on the default IPv6 IP pool./a \\ - name: IP6' \"$CNI_YAML\"\n sed -i '/- name: IP6/a \\ value: \"autodetect\"' \"$CNI_YAML\"\n\n sed -i '/Enable or Disable VXLAN on the default IPv6 IP pool./a \\ - name: CALICO_IPV6POOL_CIDR' \"$CNI_YAML\"\n sed -i '/CALICO_IPV6POOL_CIDR/a \\ value: '\"${IPv6_CLUSTER_CIDR}\" \"$CNI_YAML\"\n\n sed -i '/Enable or Disable VXLAN on the default IPv6 IP pool./a \\ - name: IP6_AUTODETECTION_METHOD' \"$CNI_YAML\"\n sed -i '/IP6_AUTODETECTION_METHOD/a \\ value: \"first-found\"' \"$CNI_YAML\"\n fi\n\n # Other configuration\n if [ ! -z \"${CALICO_VETH_MTU}\" ]; then\n sed -i 's,veth_mtu: \"0\",veth_mtu: \"'\"${CALICO_VETH_MTU}\"'\",' \"$CNI_YAML\"\n fi\n}\n\n\nfunction validate_configuration {\n if test \"x${IPv4_SUPPORT}\" = \"xtrue\"; then\n if test \"x${IPv4_CLUSTER_CIDR}\" = \"x\"; then\n echo \"Failed: IPv4_SUPPORT is true, but no IPv4_CLUSTER_CIDR was set\"\n exit 1\n fi\n if test \"x${IPv4_SERVICE_CIDR}\" = \"x\"; then\n echo \"Failed: IPv4_SUPPORT is true, but no IPv4_SERVICE_CIDR was set\"\n exit 1\n fi\n fi\n\n if test \"x${IPv6_SUPPORT}\" = \"xtrue\"; then\n if test \"x${IPv6_CLUSTER_CIDR}\" = \"x\"; then\n echo \"Failed: IPv6_SUPPORT is true, but no IPv6_CLUSTER_CIDR was set\"\n exit 1\n fi\n if test \"x${IPv6_SERVICE_CIDR}\" = \"x\"; then\n echo \"Failed: IPv6_SUPPORT is true, but no IPv6_SERVICE_CIDR was set\"\n exit 1\n fi\n fi\n\n if test x\"${IPv4_SUPPORT}\" = \"xtrue\" && test x\"${IPv6_SUPPORT}\" = \"xtrue\"; then\n CLUSTER_CIDR=\"${IPv4_CLUSTER_CIDR},${IPv6_CLUSTER_CIDR}\"\n SERVICE_CIDR=\"${IPv4_SERVICE_CIDR},${IPv6_SERVICE_CIDR}\"\n elif test x\"${IPv4_SUPPORT}\" = \"xtrue\"; then\n CLUSTER_CIDR=\"${IPv4_CLUSTER_CIDR}\"\n SERVICE_CIDR=\"${IPv4_SERVICE_CIDR}\"\n elif test x\"${IPv6_SUPPORT}\" = \"xtrue\"; then\n CLUSTER_CIDR=\"${IPv6_CLUSTER_CIDR}\"\n SERVICE_CIDR=\"${IPv6_SERVICE_CIDR}\"\n else\n echo \"Failed: At least one of IPv4_SUPPORT or IPv6_SUPPORT must be true\"\n exit 1\n fi\n}\n\nfunction setup_service_arguments {\n # Setup arguments in the microk8s services\n\n refresh_opt_in_local_config service-cluster-ip-range \"${SERVICE_CIDR}\" kube-controller-manager\n refresh_opt_in_local_config cluster-cidr \"${CLUSTER_CIDR}\" kube-controller-manager\n\n refresh_opt_in_local_config service-cluster-ip-range \"${SERVICE_CIDR}\" kube-apiserver\n\n refresh_opt_in_local_config cluster-cidr \"${CLUSTER_CIDR}\" kube-proxy\n}\n\n# Common setup, validation and setup service arguments\nvalidate_configuration\nsetup_service_arguments\n\n# Do setup of the chosen CNI\ncase \"${CNI}\" in\n calico)\n handle_calico\n ;;\n *)\n echo \"CNI must be set to 'calico'\"\n exit 1\n ;;\nesac\n" }, { "path": "scripts/inspect.sh", "content": "#!/usr/bin/env bash\n\nINSPECT_DUMP=${SNAP_DATA}/inspection-report\nRETURN_CODE=0\nJOURNALCTL_LIMIT=100000\n\nsource $SNAP/actions/common/utils.sh\n\nfunction print_help {\n # Print the help message\n printf -- 'This script will inspect your microk8s installation. It will report any issue it finds,\\n';\n printf -- 'and create a tarball of logs and traces which can be attached to an issue filed against\\n';\n printf -- 'the microk8s project.\\n';\n}\n\n\nfunction check_service {\n # Check the service passed as the first argument is up and running and collect its logs.\n local service=$1\n mkdir -p $INSPECT_DUMP/$service\n\n status=\"inactive\"\n if is_strict\n then\n journalctl -n $JOURNALCTL_LIMIT -u snap.$service &> $INSPECT_DUMP/$service/journal.log\n snapctl services $service &> $INSPECT_DUMP/$service/snapctl.log\n if ! snapctl services $service | grep inactive &> /dev/null\n then\n status=\"active\"\n fi\n else\n journalctl -n $JOURNALCTL_LIMIT -u $service &> $INSPECT_DUMP/$service/journal.log\n systemctl status $service &> $INSPECT_DUMP/$service/systemctl.log\n if systemctl status $service &> /dev/null\n then\n status=\"active\"\n fi\n fi\n\n if [ \"$status\" == \"active\" ]\n then\n printf -- ' Service %s is running\\n' \"$service\"\n else\n printf -- '\\033[31m FAIL: \\033[0m Service %s is not running\\n' \"$service\"\n printf -- 'For more details look at: sudo journalctl -u %s\\n' \"$service\"\n RETURN_CODE=1\n fi\n}\n\n\nfunction check_apparmor {\n # Collect apparmor info.\n mkdir -p $INSPECT_DUMP/apparmor\n if is_strict\n then\n journalctl -k &> $INSPECT_DUMP/apparmor/dmesg\n else\n if [ -f /etc/apparmor.d/containerd ]\n then\n cp /etc/apparmor.d/containerd $INSPECT_DUMP/apparmor/\n fi\n dmesg &> $INSPECT_DUMP/apparmor/dmesg\n aa-status &> $INSPECT_DUMP/apparmor/aa-status\n fi\n}\n\n\nfunction store_args {\n # Collect the services arguments.\n printf -- ' Copy service arguments to the final report tarball\\n'\n cp -r ${SNAP_DATA}/args $INSPECT_DUMP\n}\n\n\nfunction store_network {\n # Collect network setup.\n printf -- ' Copy network configuration to the final report tarball\\n'\n mkdir -p $INSPECT_DUMP/network\n ip addr &> $INSPECT_DUMP/network/ip-addr\n ip route &> $INSPECT_DUMP/network/ip-route\n iptables -t nat -L -n -v &> $INSPECT_DUMP/network/iptables\n iptables -S &> $INSPECT_DUMP/network/iptables-S\n iptables -L &> $INSPECT_DUMP/network/iptables-L\n if ! is_strict\n then\n ss -pln &> $INSPECT_DUMP/network/ss\n fi\n}\n\n\nfunction store_sys {\n # Generate sys directory\n mkdir -p $INSPECT_DUMP/sys\n # collect the processes running\n printf -- ' Copy processes list to the final report tarball\\n'\n ps -ef > $INSPECT_DUMP/sys/ps\n # Store disk usage information\n printf -- ' Copy disk usage information to the final report tarball\\n'\n df -h | grep ^/ &> $INSPECT_DUMP/sys/disk_usage # remove the grep to also include virtual in-memory filesystems\n # Store memory usage information\n printf -- ' Copy memory usage information to the final report tarball\\n'\n free -m &> $INSPECT_DUMP/sys/memory_usage\n # Store server's uptime.\n printf -- ' Copy server uptime to the final report tarball\\n'\n uptime &> $INSPECT_DUMP/sys/uptime\n # Store openssl information.\n printf -- ' Copy openSSL information to the final report tarball\\n'\n openssl version -v -d -e &> $INSPECT_DUMP/sys/openssl\n if ! is_strict\n then\n printf -- ' Copy snap list to the final report tarball\\n'\n snap version > $INSPECT_DUMP/sys/snap-version\n snap list > $INSPECT_DUMP/sys/snap-list\n # Stores VM name (or none, if we are not on a VM)\n printf -- ' Copy VM name (or none) to the final report tarball\\n'\n systemd-detect-virt &> $INSPECT_DUMP/sys/vm_name\n # Store the current linux distro.\n printf -- ' Copy current linux distribution to the final report tarball\\n'\n lsb_release -a &> $INSPECT_DUMP/sys/lsb_release\n # Store the aio request limits and current number\n # Returns sysctl: permission denied on key 'fs.aio-max-nr' on strict\n printf -- ' Copy asnycio usage and limits to the final report tarball\\n'\n sysctl fs.aio-max-nr &> $INSPECT_DUMP/sys/aio-max-nr\n sysctl fs.aio-nr &> $INSPECT_DUMP/sys/aio-nr\n # Store the max inotify parameters\n printf -- ' Copy inotify max_user_instances and max_user_watches to the final report tarball\\n'\n sysctl fs.inotify.max_user_instances &> $INSPECT_DUMP/sys/inotify_max_user_instances\n sysctl fs.inotify.max_user_watches &> $INSPECT_DUMP/sys/inotify_max_user_watches\n fi\n}\n\n\nfunction store_kubernetes_info {\n # Collect some in-k8s details\n printf -- ' Inspect kubernetes cluster\\n'\n mkdir -p $INSPECT_DUMP/k8s\n if is_strict\n then\n export KUBECONFIG=$SNAP_DATA/credentials/client.config\n $SNAP/kubectl version > $INSPECT_DUMP/k8s/version 2>&1\n $SNAP/kubectl cluster-info > $INSPECT_DUMP/k8s/cluster-info 2>&1\n $SNAP/kubectl cluster-info dump -A > $INSPECT_DUMP/k8s/cluster-info-dump 2>&1\n $SNAP/kubectl get all --all-namespaces -o wide > $INSPECT_DUMP/k8s/get-all 2>&1\n $SNAP/kubectl get pv > $INSPECT_DUMP/k8s/get-pv 2>&1\n $SNAP/kubectl get pvc --all-namespaces > $INSPECT_DUMP/k8s/get-pvc 2>&1\n else\n run_with_sudo /snap/bin/microk8s kubectl version 2>&1 | sudo tee $INSPECT_DUMP/k8s/version > /dev/null\n run_with_sudo /snap/bin/microk8s kubectl cluster-info 2>&1 | sudo tee $INSPECT_DUMP/k8s/cluster-info > /dev/null\n run_with_sudo /snap/bin/microk8s kubectl cluster-info dump -A 2>&1 | sudo tee $INSPECT_DUMP/k8s/cluster-info-dump > /dev/null\n run_with_sudo /snap/bin/microk8s kubectl get all --all-namespaces -o wide 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-all > /dev/null\n run_with_sudo /snap/bin/microk8s kubectl get pv 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-pv > /dev/null # 2>&1 redirects stderr and stdout to /dev/null if no resources found\n run_with_sudo /snap/bin/microk8s kubectl get pvc --all-namespaces 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-pvc > /dev/null # 2>&1 redirects stderr and stdout to /dev/null if no resources found\n fi\n\n # Collect bill of materials\n cp $SNAP/bom.json $INSPECT_DUMP/bom.json\n}\n\nfunction check_storage_addon {\n export KUBECONFIG=$SNAP_DATA/credentials/client.config\n image=\"$($SNAP/kubectl get deploy -n kube-system hostpath-provisioner -o jsonpath='{.spec.template.spec.containers[0].image}' 2>&1)\"\n\n case \"$image\" in\n cdkbot/hostpath-provisioner-amd64:1.0.0|cdkbot/hostpath-provisioner-arm64:1.0.0)\n printf -- '\\n'\n printf -- '\\033[0;33mWARNING: \\033[0m You are using an outdated version of the hostpath-provisioner.\\n'\n printf -- 'Existing PersistentVolumes are not affected, but you may be unable to create new ones.\\n'\n printf -- \"Image currently in use: ${image}\\n\"\n printf -- '\\n'\n printf -- 'Consider updating the hostpath-provisioner with:\\n'\n printf -- ' microk8s disable hostpath-storage\\n'\n printf -- ' microk8s enable hostpath-storage\\n'\n printf -- '\\n'\n ;;\n esac\n}\n\n\nfunction store_dqlite_info {\n # Collect some dqlite details\n printf -- ' Inspect dqlite\\n'\n mkdir -p $INSPECT_DUMP/dqlite\n run_with_sudo preserve_env cp ${SNAP_DATA}/var/kubernetes/backend/cluster.yaml $INSPECT_DUMP/dqlite/\n run_with_sudo preserve_env cp ${SNAP_DATA}/var/kubernetes/backend/info.yaml $INSPECT_DUMP/dqlite/\n run_with_sudo preserve_env ls -lh ${SNAP_DATA}/var/kubernetes/backend/ 2>&1 > $INSPECT_DUMP/dqlite/list.out\n}\n\n\nfunction suggest_fixes {\n # Propose fixes\n printf '\\n'\n # This if is verbose but I hope is clear\n status=\"inactive\"\n if is_strict\n then\n if ! snapctl services microk8s.daemon-kubelite | grep inactive &> /dev/null\n then\n status=\"active\"\n fi\n else\n if systemctl status snap.microk8s.daemon-kubelite &> /dev/null\n then\n status=\"active\"\n fi\n fi\n\n if [ \"$status\" == \"inactive\" ]\n then\n if lsof -Pi :16443 -sTCP:LISTEN -t &> /dev/null\n then\n printf -- '\\033[0;33m WARNING: \\033[0m Port 16443 seems to be in use by another application.\\n'\n fi\n fi\n\n if iptables -L 2>&1 | grep FORWARD | grep DROP &> /dev/null\n then\n printf -- '\\033[0;33m WARNING: \\033[0m IPtables FORWARD policy is DROP. '\n printf -- 'Consider enabling traffic forwarding with: sudo iptables -P FORWARD ACCEPT \\n'\n printf -- 'The change can be made persistent with: sudo apt-get install iptables-persistent\\n'\n fi\n\n if ! is_strict && [ /snap/core22/current/usr/bin/which ufw &> /dev/null ]\n then\n if ufw status | grep -q \"Status: active\"\n then\n header='\\033[0;33m WARNING: \\033[0m Firewall is enabled. Consider allowing pod traffic with: \\n'\n content=''\n if ! echo $ufw | grep -q vxlan.calico\n then\n content+=' sudo ufw allow in on vxlan.calico && sudo ufw allow out on vxlan.calico\\n'\n fi\n if ! echo $ufw | grep 'cali+' &> /dev/null\n then\n content+=' sudo ufw allow in on cali+ && sudo ufw allow out on cali+\\n'\n fi\n\n if [[ ! -z \"$content\" ]]\n then\n echo printing\n printf -- \"$header\"\n printf -- \"$content\"\n fi\n fi\n fi\n\n # check for selinux. if enabled, print warning.\n if getenforce 2>&1 | grep 'Enabled' > /dev/null\n then\n printf -- '\\033[0;33m WARNING: \\033[0m SElinux is enabled. Consider disabling it.\\n'\n fi\n\n # check for docker\n # if docker is installed\n if [ -d \"/etc/docker/\" ] && ! [ -z \"$(which dockerd)\" ]; then\n # if docker/daemon.json file doesn't exist print prompt to create it and mark the registry as insecure\n if [ ! -f \"/etc/docker/daemon.json\" ]; then\n printf -- '\\033[0;33mWARNING: \\033[0m Docker is installed. \\n'\n printf -- 'File \"/etc/docker/daemon.json\" does not exist. \\n'\n printf -- 'You should create it and add the following lines: \\n'\n printf -- '{\\n'\n printf -- ' \"insecure-registries\" : [\"localhost:32000\"] \\n'\n printf -- '}\\n'\n printf -- 'and then restart docker with: sudo systemctl restart docker\\n'\n # else if the file docker/daemon.json exists\n else\n # if it doesn't include the registry as insecure, prompt to add the following lines\n if ! grep -qs localhost:32000 /etc/docker/daemon.json\n then\n printf -- '\\033[0;33mWARNING: \\033[0m Docker is installed. \\n'\n printf -- 'Add the following lines to /etc/docker/daemon.json: \\n'\n printf -- '{\\n'\n printf -- ' \"insecure-registries\" : [\"localhost:32000\"] \\n'\n printf -- '}\\n'\n printf -- 'and then restart docker with: sudo systemctl restart docker\\n'\n fi\n fi\n fi\n\n if ! grep -q 'cgroup/memory' /proc/self/mountinfo; then\n if ! grep -q 'cgroup2' /proc/self/mountinfo; then\n if ! mount -t cgroup2 | grep -q 'type cgroup2'; then\n printf -- '\\033[0;33mWARNING: \\033[0m The memory cgroup is not enabled. \\n'\n printf -- 'The cluster may not be functioning properly. Please ensure cgroups are enabled \\n'\n printf -- 'See for example: https://microk8s.io/docs/install-alternatives#heading--arm \\n'\n fi\n fi\n fi\n\n # Fedora Specific Checks\n if ! is_strict && fedora_release\n then\n\n # Check if appropriate cgroup libraries for Fedora are installed\n if ! rpm -q libcgroup &> /dev/null\n then\n printf -- '\\033[31m FAIL: \\033[0m libcgroup v1 is not installed. Please install it\\n'\n printf -- '\\twith: dnf install libcgroup libcgroup-tools \\n'\n fi\n\n # check if cgroups v1 is supported\n if [ ! -d \"/sys/fs/cgroup/memory\" ] &> /dev/null\n then\n printf -- '\\033[31m FAIL: \\033[0m Cgroup v1 seems not to be enabled. Please enable it \\n'\n printf -- '\\tby executing the following command and reboot: \\n'\n printf -- '\\tgrubby --update-kernel=ALL --args=\"systemd.unified_cgroup_hierarchy=0\" \\n'\n fi\n fi\n\n # Debian 9 checks\n if ! is_strict && debian9_release\n then\n\n # Check if snapctl is fresh\n if ! [ -L \"/usr/bin/snapctl\" ]\n then\n printf -- '\\033[0;33mWARNING: \\033[0m On Debian 9 the snapctl binary if outdated. Replace it with: \\n'\n printf -- '\\t sudo snap install core \\n'\n printf -- '\\t sudo mv /usr/bin/snapctl /usr/bin/snapctl.old \\n'\n printf -- '\\t sudo ln -s /snap/core/current/usr/bin/snapctl /usr/bin/snapctl \\n'\n fi\n fi\n\n # LXD Specific Checks\n if ! is_strict && cat /proc/1/environ | grep \"container=lxc\" &> /dev/null\n then\n\n # make sure the /dev/kmsg is available, indicating a potential missing profile\n if [ ! -c \"/dev/kmsg\" ] # kmsg is a character device\n then\n printf -- '\\033[0;33mWARNING: \\033[0m the lxc profile for MicroK8s might be missing. \\n'\n printf -- '\\t Refer to this help document to get MicroK8s working in with LXD: \\n'\n printf -- '\\t https://microk8s.io/docs/lxd \\n'\n fi\n fi\n\n # node name\n nodename=\"$(hostname)\"\n if [[ \"$nodename\" =~ [A-Z|_] ]] && ! grep -e \"hostname-override\" \"${SNAP_DATA}/args/kubelet\" &> /dev/null\n then\n printf -- \"\\033[0;33mWARNING: \\033[0m This machine's hostname contains capital letters and/or underscores. \\n\"\n printf -- \"\\t This is not a valid name for a Kubernetes node, causing node registration to fail.\\n\"\n printf -- \"\\t Please change the machine's hostname or refer to the documentation for more details: \\n\"\n printf -- \"\\t https://microk8s.io/docs/troubleshooting#heading--common-issues \\n\"\n fi\n\n if grep Raspberry /proc/cpuinfo -q &&\n [ -e /etc/os-release ] &&\n grep impish /etc/os-release -q &&\n ! dpkg -l | grep linux-modules-extra-raspi -q\n then\n printf -- \"\\033[0;33mWARNING: \\033[0m On Raspberry Pi consider installing the linux-modules-extra-raspi package with: \\n\"\n printf -- \"\\t 'sudo apt install linux-modules-extra-raspi' and reboot.\\n\"\n fi\n\n if ! [ -f /proc/sys/net/bridge/bridge-nf-call-iptables ]\n then\n printf -- \"\\033[0;33mWARNING: \\033[0m Please ensure br_netfilter kernel module is loaded on the host. \\n\"\n fi\n\n if ! ( ip route; ip -6 route ) | grep \"^default\" &>/dev/null\n then\n printf -- \"\n\\033[0;33mWARNING: \\033[0m No default route exists. MicroK8s might not work properly.\nRefer to https://microk8s.io/docs for instructions on air-gap deployments.\\n\\n\n\"\n fi\n\n if ! is_strict\n then\n AIO_MAX_NR=$(sysctl -n fs.aio-max-nr)\n AIO_NR=$(sysctl -n fs.aio-nr)\n if [ \"$AIO_NR\" -ge \"$AIO_MAX_NR\" ]; then\n printf -- \"\\033[0;33mWARNING: \\033[0m Available asyncio requests are exhausted. This might lead to dqlite being unresponsive. \\n\"\n printf -- \"\\t Increase the limit and restart the k8s-dqlite service with: \\n\"\n printf -- \"\\t \\t echo fs.aio.max-nr=1048576 | sudo tee -a /etc/sysctl.conf\\n\"\n printf -- \"\\t \\t sudo sysctl --system\\n\"\n printf -- \"\\t \\t sudo snap restart microk8s.daemon-k8s-dqlite\\n\"\n fi\n\n MAX_USER_INSTANCES=$(sysctl -n fs.inotify.max_user_instances)\n if [ \"$MAX_USER_INSTANCES\" -lt 1024 ]; then\n printf -- \"\\033[0;33mWARNING: \\033[0m Maximum number of inotify user instances is less than the recommended value of 1024. \\n\"\n printf -- \"\\t Increase the limit with: \\n\"\n printf -- \"\\t \\t echo fs.inotify.max_user_instances=1024 | sudo tee -a /etc/sysctl.conf\\n\"\n printf -- \"\\t \\t sudo sysctl --system\\n\"\n fi\n\n MAX_USER_WATCHES=$(sysctl -n fs.inotify.max_user_watches)\n if [ \"$MAX_USER_WATCHES\" -lt 1048576 ]; then\n printf -- \"\\033[0;33mWARNING: \\033[0m Maximum number of inotify user watches is less than the recommended value of 1048576. \\n\"\n printf -- \"\\t Increase the limit with: \\n\"\n printf -- \"\\t \\t echo fs.inotify.max_user_watches=1048576 | sudo tee -a /etc/sysctl.conf\\n\"\n printf -- \"\\t \\t sudo sysctl --system\\n\"\n fi\n fi\n}\n\nfunction fedora_release {\n local RELEASE=`cat /etc/os-release | grep \"^NAME=\" | cut -f2 -d=`\n if [ \"${RELEASE}\" == \"Fedora\" ]\n then\n return 0\n else\n return 1\n fi\n}\n\nfunction debian9_release {\n if [ -e /etc/debian_version ] &&\n grep \"9\\.\" /etc/debian_version -q\n then\n return 0\n else\n return 1\n fi\n}\n\nfunction build_report_tarball {\n # Tar and gz the report\n local now_is=$(date +\"%Y%m%d_%H%M%S\")\n tar -C ${SNAP_DATA} -cf ${SNAP_DATA}/inspection-report-${now_is}.tar inspection-report &> /dev/null\n gzip ${SNAP_DATA}/inspection-report-${now_is}.tar\n printf -- ' Report tarball is at %s/inspection-report-%s.tar.gz\\n' \"${SNAP_DATA}\" \"${now_is}\"\n}\n\nfunction check_certificates {\n exp_date_str=\"$(openssl x509 -enddate -noout -in \"${SNAP_DATA}/certs/ca.crt\" | cut -d= -f 2)\"\n exp_date_secs=\"$(date -d \"$exp_date_str\" +%s)\"\n now_secs=$(date +%s)\n difference=$(($exp_date_secs-$now_secs))\n days=$(($difference/(3600*24)))\n if [ \"3\" -ge $days ];\n then\n printf -- '\\033[0;33mWARNING: \\033[0m This deployments certificates will expire in $days days. \\n'\n printf -- 'Either redeploy MicroK8s or attempt a refresh with \"microk8s refresh-certs\"\\n'\n fi\n}\n\nfunction check_memory {\n MEMORY=`cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'`\n if [ $MEMORY -le 524288 ]\n then\n printf -- \"\\033[0;33mWARNING: \\033[0m This system has ${MEMORY} bytes of RAM available.\\n\"\n printf -- \"It may not be enough to run the Kubernetes control plane services.\\n\"\n printf -- \"Consider joining as a worker-only to a cluster.\\n\"\n fi\n}\n\nfunction check_low_memory_guard {\n if [ -e \"${SNAP_DATA}/var/lock/low-memory-guard.lock\" ]\n then\n printf -- '\\033[0;33mWARNING: \\033[0m The low memory guard is enabled.\\n'\n printf -- 'This is to protect the server from running out of memory.\\n'\n printf -- 'Consider joining as a worker-only to a cluster.\\n'\n printf -- '\\n'\n printf -- 'Alternatively, to disable the low memory guard, start MicroK8s with:\\n'\n printf -- '\\n'\n printf -- ' microk8s start --disable-low-memory-guard\\n'\n fi\n}\n\nfunction check_hostname {\n HOST=$(hostname)\n if echo \"${HOST}\" | grep -q \"[A-Z]\" 2> /dev/null\n then\n printf -- \"\\033[0;33mWARNING: \\033[0m The hostname of this server is '${HOST}'.\\n\"\n printf -- \"Having uppercase letters in the hostname may cause issues with RBAC.\\n\"\n printf -- \"Consider changing the hostname to only have lowercase letters with:\\n\"\n printf -- \"\\n\"\n printf -- \" hostnamectl set-hostname $(echo ${HOST} | tr '[:upper:]' '[:lower:]')\\n\"\n fi\n}\n\nif [[ (${#@} -ne 0) && ((\"$*\" == \"--help\") || (\"$*\" == \"-h\")) ]]; then\n print_help\n exit 0;\nfi;\n\nexit_if_not_root\n\nrm -rf ${SNAP_DATA}/inspection-report\nmkdir -p ${SNAP_DATA}/inspection-report\n\nprintf -- 'Inspecting system\\n'\ncheck_memory\ncheck_low_memory_guard\ncheck_hostname\n\nprintf -- 'Inspecting Certificates\\n'\ncheck_certificates\n\nprintf -- 'Inspecting services\\n'\nsvc_cluster_agent=\"microk8s.daemon-cluster-agent\"\nsvc_containerd=\"microk8s.daemon-containerd\"\nsvc_kubelite=\"microk8s.daemon-kubelite\"\nsvc_flanneld=\"microk8s.daemon-flanneld\"\nsvc_etcd=\"microk8s.daemon-etcd\"\nsvc_dqlite=\"microk8s.daemon-k8s-dqlite\"\nsvc_api_server_proxy=\"microk8s.daemon-apiserver-proxy\"\nsvc_api_server_kicker=\"microk8s.daemon-apiserver-kicker\"\n\nif ! is_strict\nthen\n svc_cluster_agent=\"snap.${svc_cluster_agent}\"\n svc_containerd=\"snap.${svc_containerd}\"\n svc_kubelite=\"snap.${svc_kubelite}\"\n svc_flanneld=\"snap.${svc_flanneld}\"\n svc_etcd=\"snap.${svc_etcd}\"\n svc_dqlite=\"snap.${svc_dqlite}\"\n svc_api_server_proxy=\"snap.${svc_api_server_proxy}\"\n svc_api_server_kicker=\"snap.${svc_api_server_kicker}\"\nfi\n\ncheck_service $svc_cluster_agent\ncheck_service $svc_containerd\ncheck_service $svc_kubelite\n\nif ! [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ]\nthen\n check_service $svc_flanneld\n check_service $svc_etcd\nelse\n if ! [ -e \"${SNAP_DATA}/var/lock/no-k8s-dqlite\" ]\n then\n # workers do not run dqlite\n check_service $svc_dqlite\n fi\nfi\n\nif ! [ -e \"${SNAP_DATA}/var/lock/no-apiserver-proxy\" ]\nthen\n check_service $svc_api_server_proxy\nfi\n\nif ! [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n check_service $svc_api_server_kicker\nfi\n\nstore_args\n\nprintf -- 'Inspecting AppArmor configuration\\n'\ncheck_apparmor\n\nprintf -- 'Gathering system information\\n'\nstore_sys\nstore_network\n\nprintf -- 'Inspecting kubernetes cluster\\n'\nstore_kubernetes_info\ncheck_storage_addon\n\nif [ -e \"${SNAP_DATA}/var/lock/ha-cluster\" ]\nthen\n printf -- 'Inspecting dqlite\\n'\n store_dqlite_info\nfi\n\nsuggest_fixes\n\nprintf -- 'Building the report tarball\\n'\nbuild_report_tarball\n\nexit $RETURN_CODE\n" }, { "path": "scripts/kill-host-pods.py", "content": "#!/usr/bin/env python3\n\nimport json\nimport subprocess\nimport os\nfrom pathlib import Path\n\nimport click\n\nCTR = os.path.expandvars(\"$SNAP/microk8s-ctr.wrapper\")\nSNAP = os.getenv(\"SNAP\")\nSNAP_DATA = os.getenv(\"SNAP_DATA\")\nSNAP_DATA_BASE = os.path.abspath(f\"{SNAP_DATA}/..\")\nSNAP_DATA_CURRENT = os.path.abspath(f\"{SNAP_DATA_BASE}/current\")\nSNAP_COMMON = os.path.abspath(f\"{SNAP_DATA_BASE}/common\")\nKUBECTL = [f\"{SNAP}/kubectl\", f\"--kubeconfig={SNAP_DATA}/credentials/kubelet.config\"]\n\n\ndef post_filter_has_known_containers(pod, containers: list) -> bool:\n \"\"\"\n Return true if any of the container IDs on the pod match the list of\n containers passed on the second argument.\n \"\"\"\n for container in pod[\"status\"][\"containerStatuses\"] or []:\n try:\n _, container_id = container[\"containerID\"].split(\"containerd://\", 2)\n if container_id in containers:\n return True\n except (KeyError, ValueError, TypeError, AttributeError):\n continue\n\n return False\n\n\ndef post_filter_has_snap_data_mounts(pod) -> bool:\n \"\"\"\n Return true if a pod definition contains any volumes that mount paths\n from `/var/snap/microk8s/current/...`, e.g. CNI pods\n \"\"\"\n for volume in pod[\"spec\"].get(\"volumes\", []):\n hostpath_volume = volume.get(\"hostPath\", {})\n host_path = hostpath_volume.get(\"path\", \"\")\n if not host_path:\n continue\n if host_path.startswith(SNAP_DATA_CURRENT):\n return True\n\n # handle the case where a symlink to a SNAP_DATA path is used\n try:\n resolved = Path(host_path).resolve().as_posix()\n if resolved.startswith(SNAP_DATA_BASE) and not resolved.startswith(SNAP_COMMON):\n return True\n except OSError:\n pass\n\n return False\n\n\ndef post_filter_has_owner(pod: dict):\n \"\"\"\n Return true if a pod definition has an ownerReference (i.e. is it managed by a\n Deployment or a DaemonSet)\n \"\"\"\n owner_references = pod[\"metadata\"].get(\"ownerReferences\") or []\n return len(owner_references) > 0\n\n\n@click.command(\"kill-host-pods\")\n@click.argument(\"selector\", nargs=-1)\n@click.option(\"--dry-run\", is_flag=True, default=False)\n@click.option(\"--with-snap-data-mounts\", is_flag=True, default=False)\n@click.option(\"--with-owner\", is_flag=True, default=False)\ndef main(selector: list, dry_run: bool, with_snap_data_mounts: bool, with_owner: bool):\n \"\"\"\n Delete pods running on the local node based on Kubernetes selectors.\n\n Example usage:\n\n $ ./kill-host-pods.py -- -n kube-system -l k8s-app=calico-node\n $ ./kill-host-pods.py --with-snap-data-mounts -- -n kube-system\n $ ./kill-host-pods.py --with-snap-data-mounts -- -A\n \"\"\"\n containers = subprocess.check_output([CTR, \"container\", \"ls\", \"-q\"]).decode().split(\"\\n\")\n out = subprocess.check_output([*KUBECTL, \"get\", \"pod\", \"-o\", \"json\", *selector])\n\n pods = json.loads(out)\n for pod in pods[\"items\"]:\n if not post_filter_has_known_containers(pod, containers):\n continue\n if with_snap_data_mounts and not post_filter_has_snap_data_mounts(pod):\n continue\n if with_owner and not post_filter_has_owner(pod):\n continue\n\n meta = pod[\"metadata\"]\n cmd = [*KUBECTL, \"delete\", \"pod\", \"-n\", meta[\"namespace\"], meta[\"name\"]]\n if dry_run:\n cmd = [\"echo\", *cmd]\n\n subprocess.check_call(cmd)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "scripts/run-lifecycle-hooks.py", "content": "#!/usr/bin/env python3\n\nfrom pathlib import Path\nimport os\nimport click\nimport subprocess\n\nSNAP_COMMON = Path(os.getenv(\"SNAP_COMMON\"))\nTIMEOUT = 120\n\n\n@click.command(\"run-lifecycle-hooks\")\n@click.argument(\"hook\")\ndef main(hook: str):\n hooks_dir = SNAP_COMMON / \"hooks\" / f\"{hook}.d\"\n hooks = os.listdir(hooks_dir)\n for hook in sorted(hooks):\n try:\n if not (os.stat(hooks_dir / hook).st_mode & 0o100):\n # ignore non-executable files\n continue\n\n subprocess.run([hooks_dir / hook], timeout=TIMEOUT)\n except (subprocess.CalledProcessError, OSError):\n pass\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "scripts/wrappers/add_token.py", "content": "import json\nimport yaml\nimport os\nimport sys\nimport time\nimport argparse\nimport subprocess\n\nfrom common.cluster.utils import is_node_running_dqlite, TOKEN_ΜΙΝ_LEN\n\ntry:\n from secrets import token_hex\nexcept ImportError:\n from os import urandom\n\n def token_hex(nbytes=None):\n return urandom(nbytes).hex()\n\n\ncluster_tokens_file = os.path.expandvars(\"${SNAP_DATA}/credentials/cluster-tokens.txt\")\nutils_sh_file = os.path.expandvars(\"${SNAP}/actions/common/utils.sh\")\ntoken_with_expiry = \"{}|{}\\n\"\ntoken_without_expiry = \"{}\\n\"\n\n\ndef add_token_with_expiry(token, file, ttl):\n \"\"\"\n This method will add a token to the token file with or without expiry\n Expiry time is in seconds.\n\n Format of the item in the file: |\n\n :param str token: The token to add to the file\n :param str file: The file name for which the token will be written to\n :param ttl: How long the token should last before expiry, represented in seconds.\n \"\"\"\n\n with open(file, \"a+\") as fp:\n if ttl != -1:\n expiry = int(round(time.time())) + ttl\n fp.write(token_with_expiry.format(token, expiry))\n else:\n fp.write(token_without_expiry.format(token))\n\n\ndef run_util(*args, debug=False):\n env = os.environ.copy()\n prog = [\"bash\", utils_sh_file]\n prog.extend(args)\n\n if debug:\n print(\"\\033[;1;32m+ %s\\033[;0;0m\" % \" \".join(prog))\n\n result = subprocess.run(\n prog,\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n env=env,\n )\n\n try:\n result.check_returncode()\n except subprocess.CalledProcessError:\n print(\"Failed to call utility function.\")\n sys.exit(1)\n\n return result.stdout.decode(\"utf-8\").strip()\n\n\ndef get_network_info():\n \"\"\"\n Obtain machine IP address(es) and cluster agent port.\n :return: tuple of default IP, all IPs, and cluster agent port\n \"\"\"\n default_ip = run_util(\"get_default_ip\")\n all_ips = run_util(\"get_ips\").split(\" \")\n port = run_util(\"cluster_agent_port\")\n\n return (default_ip, all_ips, port)\n\n\ndef print_pretty(token, check):\n default_ip, all_ips, port = get_network_info()\n\n print(\"From the node you wish to join to this cluster, run the following:\")\n print(f\"microk8s join {default_ip}:{port}/{token}/{check}\\n\")\n\n if is_node_running_dqlite():\n print(\n \"Use the '--worker' flag to join a node as a worker not running the control plane, eg:\"\n )\n print(f\"microk8s join {default_ip}:{port}/{token}/{check} --worker\\n\")\n\n print(\n \"If the node you are adding is not reachable through the default interface you can use one of the following:\"\n )\n for ip in all_ips:\n print(f\"microk8s join {ip}:{port}/{token}/{check}\")\n\n\ndef get_output_dict(token, check):\n _, all_ips, port = get_network_info()\n info = {\n \"token\": f\"{token}/{check}\",\n \"urls\": [f\"{ip}:{port}/{token}/{check}\" for ip in all_ips],\n }\n return info\n\n\ndef print_json(token, check):\n info = get_output_dict(token, check)\n print(json.dumps(info, indent=2))\n\n\ndef print_yaml(token, check):\n info = get_output_dict(token, check)\n print(yaml.dump(info, indent=2))\n\n\ndef print_short(token, check):\n default_ip, all_ips, port = get_network_info()\n print(f\"microk8s join {default_ip}:{port}/{token}/{check}\")\n for ip in all_ips:\n if ip != default_ip:\n print(f\"microk8s join {ip}:{port}/{token}/{check}\")\n\n\nif __name__ == \"__main__\":\n\n # initiate the parser with a description\n parser = argparse.ArgumentParser(\n description=\"Produce a connection string for a node to join the cluster.\",\n prog=\"microk8s add-node\",\n )\n parser.add_argument(\n \"--token-ttl\",\n \"-l\",\n help=\"Specify how long the token is valid, before it expires. \"\n 'Value of \"-1\" indicates that the token is usable only once '\n \"(i.e. after joining a node, the token becomes invalid)\",\n type=int,\n default=\"-1\",\n )\n parser.add_argument(\n \"--token\",\n \"-t\",\n help=\"Specify the bootstrap token to add, must be 32 characters long. \"\n \"Auto generates when empty.\",\n )\n parser.add_argument(\n \"--format\",\n help=\"Format the output of the token in pretty, short, token, or token-check\",\n default=\"pretty\",\n choices={\"pretty\", \"short\", \"token\", \"token-check\", \"json\", \"yaml\"},\n )\n\n # read arguments from the command line\n args = parser.parse_args()\n\n ttl = args.token_ttl\n\n if args.token is not None:\n token = args.token\n else:\n token = token_hex(16)\n\n if len(token) < TOKEN_ΜΙΝ_LEN:\n print(\"Invalid token size. It must be 32 characters long.\")\n exit(1)\n\n add_token_with_expiry(token, cluster_tokens_file, ttl)\n check = run_util(\"server_cert_check\")\n\n if args.format == \"pretty\":\n print_pretty(token, check)\n elif args.format == \"short\":\n print_short(token, check)\n elif args.format == \"token-check\":\n print(f\"{token}/{check}\")\n elif args.format == \"json\":\n print_json(token, check)\n elif args.format == \"yaml\":\n print_yaml(token, check)\n else:\n print(token)\n" }, { "path": "scripts/wrappers/addons.py", "content": "#!/usr/bin/python3\nimport json\nimport os\nimport shutil\nimport subprocess\nimport sys\nfrom pathlib import Path\nfrom typing import List\n\nimport click\nimport jsonschema\nimport yaml\n\nfrom common.utils import get_current_arch, snap_common, snap, exit_if_no_root\nfrom common.cluster.utils import get_group\n\nGIT = os.path.expandvars(\"$SNAP/git.wrapper\")\n\naddons = click.Group()\n\nrepository = click.Group(\"repo\")\naddons.add_command(repository)\n\n\nclass RepoValidationError(Exception):\n @property\n def message(self) -> str:\n raise NotImplementedError()\n\n\nclass AddonsYamlNotFoundError(RepoValidationError):\n def __init__(self, repo_name: str):\n self.repo_name = repo_name\n\n @property\n def message(self) -> str:\n return f\"Error: repository {self.repo_name} does not contain an addons.yaml file\"\n\n\nclass AddonsYamlFormatError(RepoValidationError):\n def __init__(self, message):\n self._message = message\n\n @property\n def message(self) -> str:\n return self._message\n\n\nclass MissingHookError(RepoValidationError):\n def __init__(self, hook_name: str, addon: str):\n self.hook_name = hook_name\n self.addon = addon\n\n @property\n def message(self) -> str:\n return f\"Missing {self.hook_name} hook for {self.addon} addon\"\n\n\nclass WrongHookPermissionsError(RepoValidationError):\n def __init__(self, hook_name: str, addon: str):\n self.hook_name = hook_name\n self.addon = addon\n\n @property\n def message(self) -> str:\n return f\"{self.hook_name} hook for {self.addon} addon needs execute permissions\"\n\n\ndef validate_addons_repo(repo_dir: Path) -> None:\n \"\"\"\n Runs some checks on an addons repository.\n Inner validations raise SystemExit if any of the validations fail.\n \"\"\"\n validate_addons_file(repo_dir)\n validate_hooks(repo_dir)\n\n\ndef validate_addons_file(repo_dir: Path) -> None:\n \"\"\"\n Checks that the addons.yaml file exists and that it has the appropriate format.\n \"\"\"\n contents = load_addons_yaml(repo_dir)\n schema = {\n \"type\": \"object\",\n \"properties\": {\n \"microk8s-addons\": {\n \"type\": \"object\",\n \"properties\": {\n \"addons\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"name\": {\"type\": \"string\"},\n \"description\": {\"type\": \"string\"},\n \"version\": {\"type\": \"string\"},\n \"check_status\": {\"type\": \"string\"},\n \"supported_architectures\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"enum\": [\"amd64\", \"arm64\", \"s390x\", \"ppc64le\"],\n },\n },\n },\n \"required\": [\n \"name\",\n \"description\",\n \"version\",\n \"check_status\",\n \"supported_architectures\",\n ],\n },\n }\n },\n \"required\": [\"addons\"],\n }\n },\n \"required\": [\"microk8s-addons\"],\n }\n try:\n jsonschema.validate(contents, schema=schema)\n except jsonschema.ValidationError as err:\n message = f\"Invalid addons.yaml file: {err.message}\"\n raise AddonsYamlFormatError(message)\n\n\ndef load_addons_yaml(repo_dir: Path):\n addons_yaml = repo_dir / \"addons.yaml\"\n try:\n with open(addons_yaml, mode=\"r\") as f:\n return yaml.safe_load(f.read())\n except FileNotFoundError:\n raise AddonsYamlNotFoundError(repo_dir.name)\n except yaml.YAMLError as err:\n message = f\"Yaml format error in addons.yaml file: {err.context} {err.problem}\"\n raise AddonsYamlFormatError(message)\n\n\ndef validate_hooks(repo_dir: Path) -> None:\n \"\"\"\n Check that, for each registered addon, the enable and disable hooks\n are present in the repository, and that they have execute permissions.\n \"\"\"\n for addon in get_addons_list(repo_dir):\n addon_folder = repo_dir / \"addons\" / addon\n for hook_name in (\"enable\", \"disable\"):\n hook = addon_folder / hook_name\n if not hook.exists():\n raise MissingHookError(hook_name, addon)\n if not os.access(hook, os.X_OK):\n raise WrongHookPermissionsError(hook_name, addon)\n\n\ndef get_addons_list(repo_dir: Path) -> List[str]:\n contents = load_addons_yaml(repo_dir)\n return [addon[\"name\"] for addon in contents[\"microk8s-addons\"][\"addons\"]]\n\n\ndef pull_and_validate(name: str, repo_dir: Path):\n commit_before_pull = git_current_commit(repo_dir)\n subprocess.check_call([GIT, \"pull\"], cwd=repo_dir)\n\n try:\n validate_addons_repo(repo_dir)\n except RepoValidationError as err:\n click.echo(err.message, err=True)\n click.echo(f\"Rolling back repository {name}\")\n git_rollback(commit_before_pull, repo_dir)\n sys.exit(1)\n\n\ndef clone_and_validate(remote_url: str, repo_dir: Path):\n if repo_dir.exists():\n shutil.rmtree(repo_dir)\n subprocess.check_call([GIT, \"clone\", remote_url, repo_dir])\n subprocess.check_call([\"chgrp\", get_group(), \"-R\", repo_dir])\n\n try:\n validate_addons_repo(repo_dir)\n except RepoValidationError as err:\n click.echo(err.message, err=True)\n click.echo(f\"Removing {repo_dir}\")\n shutil.rmtree(repo_dir)\n sys.exit(1)\n\n\n@repository.command(\"add\", help=\"Add a MicroK8s addons repository\")\n@click.argument(\"name\")\n@click.argument(\"repository\")\n@click.option(\"--reference\")\n@click.option(\"--force\", is_flag=True, default=False)\ndef add(name: str, repository: str, reference: str, force: bool):\n repo_dir = snap_common() / \"addons\" / name\n if repo_dir.exists():\n if not force:\n click.echo(\"Error: repository '{}' already exists!\".format(name), err=True)\n click.echo(\"Use the --force flag to overwrite it\", err=True)\n sys.exit(1)\n\n click.echo(\"Removing {}\".format(repo_dir))\n shutil.rmtree(repo_dir)\n\n cmd = [GIT, \"clone\", repository, repo_dir]\n if reference is not None:\n cmd += [\"-b\", reference]\n subprocess.check_call(cmd)\n subprocess.check_call([\"chgrp\", get_group(), \"-R\", repo_dir])\n\n try:\n validate_addons_repo(repo_dir)\n except RepoValidationError as err:\n click.echo(err.message, err=True)\n click.echo(f\"Removing {repo_dir}\")\n shutil.rmtree(repo_dir)\n sys.exit(1)\n\n\n@repository.command(\"remove\", help=\"Remove a MicroK8s addons repository\")\n@click.argument(\"name\")\ndef remove(name: str):\n repo_dir = snap_common() / \"addons\" / name\n if not repo_dir.exists():\n click.echo(\"Error: repository '{}' does not exist\".format(name), err=True)\n sys.exit(1)\n\n click.echo(\"Removing {}\".format(repo_dir))\n shutil.rmtree(repo_dir)\n\n\n@repository.command(\"update\", help=\"Update a MicroK8s addons repository\")\n@click.argument(\"name\")\n@click.option(\"--skip-check-root/--no-skip-check-root\", is_flag=True, default=False)\ndef update(name: str, skip_check_root: bool):\n if not skip_check_root:\n exit_if_no_root()\n\n repo_dir = snap_common() / \"addons\" / name\n if not repo_dir.exists():\n click.echo(\"Error: repository '{}' does not exist\".format(name), err=True)\n sys.exit(1)\n\n if not (repo_dir / \".git\").exists():\n click.echo(\"Error: built-in repository '{}' cannot be updated\".format(name), err=True)\n sys.exit(1)\n\n click.echo(\"Updating repository {}\".format(name))\n remote_url = (\n subprocess.check_output(\n [GIT, \"remote\", \"get-url\", \"origin\"], cwd=repo_dir, stderr=subprocess.DEVNULL\n )\n .decode()\n .strip()\n )\n if remote_url.startswith(str(snap().parent)):\n # This is a repository that we have in the snap.\n # If the branch name we follow has not changed a simple git pull is enough\n # If the branch name changed we need to git repo add --force\n followed_branch_name = subprocess.check_output(\n [GIT, \"rev-parse\", \"--abbrev-ref\", \"HEAD\"], cwd=repo_dir, stderr=subprocess.DEVNULL\n ).decode()\n snapped_branch_name = subprocess.check_output(\n [GIT, \"rev-parse\", \"--abbrev-ref\", \"HEAD\"], cwd=remote_url, stderr=subprocess.DEVNULL\n ).decode()\n if followed_branch_name != snapped_branch_name:\n clone_and_validate(remote_url, repo_dir)\n else:\n pull_and_validate(name, repo_dir)\n else:\n pull_and_validate(name, repo_dir)\n\n\nclass GettingGitCommitError(Exception):\n def __init__(self, exit_code, stderr):\n self.exit_code = exit_code\n self.stderr = stderr\n\n\ndef git_current_commit(repository: Path) -> str:\n \"\"\"\n Returns the current commit hash of a given git repository\n \"\"\"\n cmd = [GIT, \"rev-parse\", \"--verify\", \"HEAD\"]\n process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repository)\n stdout, stderr = process.communicate()\n if process.returncode != 0:\n raise GettingGitCommitError(exit_code=process.returncode, stderr=stderr)\n return stdout\n\n\ndef git_rollback(commit: str, repository: Path):\n \"\"\"\n Resets the git repository to a particular commit hash\n \"\"\"\n cmd = [GIT, \"reset\", \"--hard\", commit]\n subprocess.check_call(cmd, cwd=repository)\n\n\n@repository.command(\"list\", help=\"List configured MicroK8s addons repositories\")\n@click.option(\"--format\", default=\"table\", type=click.Choice([\"json\", \"yaml\", \"table\"]))\ndef list(format: str):\n arch = get_current_arch()\n repositories = []\n for dir in os.listdir(snap_common() / \"addons\"):\n try:\n repo_dir = snap_common() / \"addons\" / dir\n addons_yaml = repo_dir / \"addons.yaml\"\n with open(addons_yaml, \"r\") as fin:\n addons = yaml.safe_load(fin)\n\n count = 0\n for addon in addons[\"microk8s-addons\"][\"addons\"]:\n if arch in addon[\"supported_architectures\"]:\n count += 1\n\n source = \"(built-in)\"\n try:\n remote_url = subprocess.check_output(\n [GIT, \"remote\", \"get-url\", \"origin\"], cwd=repo_dir, stderr=subprocess.DEVNULL\n ).decode()\n revision = subprocess.check_output(\n [GIT, \"rev-parse\", \"HEAD\"], cwd=repo_dir, stderr=subprocess.DEVNULL\n ).decode()[:6]\n source = \"{}@{}\".format(remote_url.strip(), revision.strip())\n except (subprocess.CalledProcessError, TypeError, ValueError):\n pass\n\n repositories.append(\n {\n \"name\": dir,\n \"addons\": count,\n \"source\": source,\n \"description\": addons[\"microk8s-addons\"][\"description\"],\n }\n )\n\n except Exception as e:\n click.echo(\"could not load addons from {}: {}\".format(addons_yaml, e), err=True)\n\n if format == \"json\":\n click.echo(json.dumps(repositories))\n elif format == \"yaml\":\n click.echo(yaml.safe_dump(repositories))\n elif format == \"table\":\n click.echo((\"{:10} {:>6} {}\").format(\"REPO\", \"ADDONS\", \"SOURCE\"))\n for repo in repositories:\n click.echo(\"{:10} {:>6} {}\".format(repo[\"name\"], repo[\"addons\"], repo[\"source\"]))\n\n\nif __name__ == \"__main__\":\n addons(prog_name=\"microk8s addons\")\n" }, { "path": "scripts/wrappers/common/__init__.py", "content": "" }, { "path": "scripts/wrappers/common/cluster/__init__.py", "content": "" }, { "path": "scripts/wrappers/common/cluster/utils.py", "content": "import base64\nimport datetime\nimport ipaddress\nimport json\nimport os\nimport random\nimport re\nimport shutil\nimport socket\nimport string\nimport subprocess\nimport time\nfrom pathlib import Path\nfrom subprocess import CalledProcessError, check_output\n\nimport yaml\n\nFINGERPRINT_MIN_LEN = 12\nTOKEN_ΜΙΝ_LEN = 32\n\n\nclass InvalidConnectionError(Exception):\n pass\n\n\ndef is_strict():\n snap_yaml = snap() / \"meta/snap.yaml\"\n with open(snap_yaml) as f:\n snap_meta = yaml.safe_load(f)\n return snap_meta[\"confinement\"] == \"strict\"\n\n\ndef get_group():\n return \"snap_microk8s\" if is_strict() else \"microk8s\"\n\n\ndef snap() -> Path:\n try:\n return Path(os.environ[\"SNAP\"])\n except KeyError:\n return Path(\"/snap/microk8s/current\")\n\n\ndef snap_data() -> Path:\n try:\n return Path(os.environ[\"SNAP_DATA\"])\n except KeyError:\n return Path(\"/var/snap/microk8s/current\")\n\n\ndef try_set_file_permissions(file):\n \"\"\"\n Try setting the ownership group and permission of the file\n\n :param file: full path and filename\n \"\"\"\n mode = 0o660\n group = get_group()\n\n if os.path.exists(snap_data() / \"var\" / \"lock\" / \"cis-hardening\"):\n group = \"root\"\n mode = 0o600\n\n os.chmod(file, mode)\n try:\n shutil.chown(file, group=group)\n except LookupError:\n # not setting the group means only the current user can access the file\n pass\n\n\ndef remove_expired_token_from_file(file):\n \"\"\"\n Remove expired token from the valid tokens set\n\n :param file: the file to be removed from\n \"\"\"\n backup_file = \"{}.backup\".format(file)\n # That is a critical section. We need to protect it.\n # We are safe for now because flask serves one request at a time.\n with open(backup_file, \"w\") as back_fp:\n with open(file, \"r\") as fp:\n for _, line in enumerate(fp):\n if is_token_expired(line):\n continue\n back_fp.write(\"{}\".format(line))\n\n try_set_file_permissions(backup_file)\n shutil.copyfile(backup_file, file)\n\n\ndef remove_token_from_file(token, file):\n \"\"\"\n Remove a token from the valid tokens set\n\n :param token: the token to be removed\n :param file: the file to be removed from\n \"\"\"\n backup_file = \"{}.backup\".format(file)\n # That is a critical section. We need to protect it.\n # We are safe for now because flask serves one request at a time.\n with open(backup_file, \"w\") as back_fp:\n with open(file, \"r\") as fp:\n for _, line in enumerate(fp):\n # Not considering cluster tokens with expiry in this method.\n if \"|\" not in line:\n if line.strip() == token:\n continue\n back_fp.write(\"{}\".format(line))\n\n try_set_file_permissions(backup_file)\n shutil.copyfile(backup_file, file)\n\n\ndef is_token_expired(token_line):\n \"\"\"\n Checks if the token in the file is expired, when using the TTL based.\n\n :returns: True if the token is expired, otherwise False\n \"\"\"\n if \"|\" in token_line:\n expiry = token_line.strip().split(\"|\")[1]\n if int(round(time.time())) > int(expiry):\n return True\n\n return False\n\n\ndef get_callback_token():\n \"\"\"\n Generate a token and store it in the callback token file\n\n :returns: the token\n \"\"\"\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n callback_token_file = \"{}/credentials/callback-token.txt\".format(snapdata_path)\n if os.path.exists(callback_token_file):\n with open(callback_token_file) as fp:\n token = fp.read()\n else:\n token = \"\".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(64))\n with open(callback_token_file, \"w\") as fp:\n fp.write(\"{}\\n\".format(token))\n try_set_file_permissions(callback_token_file)\n\n return token\n\n\ndef is_node_running_dqlite():\n \"\"\"\n Check if we should use the dqlite joining process (join api version 2.0)\n\n :returns: True if dqlite is to be used, otherwise False\n \"\"\"\n ha_lock = os.path.expandvars(\"${SNAP_DATA}/var/lock/ha-cluster\")\n return os.path.isfile(ha_lock)\n\n\ndef is_node_dqlite_worker():\n \"\"\"\n Check if this is a worker only node\n\n :returns: True if this is a worker node, otherwise False\n \"\"\"\n ha_lock = os.path.expandvars(\"${SNAP_DATA}/var/lock/ha-cluster\")\n clustered_lock = os.path.expandvars(\"${SNAP_DATA}/var/lock/clustered.lock\")\n no_apiserver_proxy_lock = os.path.expandvars(\"${SNAP_DATA}/var/lock/no-apiserver-proxy\")\n return (\n os.path.isfile(ha_lock)\n and os.path.isfile(clustered_lock)\n and not os.path.exists(no_apiserver_proxy_lock)\n )\n\n\ndef is_low_memory_guard_enabled():\n \"\"\"\n Check if the low memory guard is enabled on this Node\n\n :returns: True if enabled, otherwise False\n \"\"\"\n lock = os.path.expandvars(\"${SNAP_DATA}/var/lock/low-memory-guard.lock\")\n return os.path.isfile(lock)\n\n\ndef get_dqlite_port():\n \"\"\"\n What is the port dqlite listens on\n\n :return: the dqlite port\n \"\"\"\n # We get the dqlite port from the already existing deployment\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n cluster_dir = \"{}/var/kubernetes/backend\".format(snapdata_path)\n dqlite_info = \"{}/info.yaml\".format(cluster_dir)\n port = 19001\n if os.path.exists(dqlite_info):\n with open(dqlite_info) as f:\n data = yaml.safe_load(f)\n if \"Address\" in data:\n port = data[\"Address\"].split(\":\")[1]\n\n return port\n\n\ndef get_cluster_agent_port():\n \"\"\"\n What is the cluster agent port\n\n :return: the port\n \"\"\"\n cluster_agent_port = \"25000\"\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n filename = \"{}/args/cluster-agent\".format(snapdata_path)\n with open(filename) as fp:\n for _, line in enumerate(fp):\n if line.startswith(\"--bind\"):\n port_parse = line.split(\" \")\n port_parse = port_parse[-1].split(\"=\")\n port_parse = port_parse[-1].split(\":\")\n if len(port_parse) > 1:\n cluster_agent_port = port_parse[1].rstrip()\n return cluster_agent_port\n\n\ndef get_cluster_cidr():\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n filename = \"{}/args/kube-proxy\".format(snapdata_path)\n with open(filename) as fp:\n for _, line in enumerate(fp):\n if line.startswith(\"--cluster-cidr\"):\n cidr_parse = line.split(\"=\")\n if len(cidr_parse) > 1:\n return cidr_parse[1].rstrip()\n return \"\"\n\n\ndef get_control_plane_nodes_internal_ips():\n \"\"\"\n Return the internal IP of the nodes labeled running the control plane.\n\n :return: list of node internal IPs\n \"\"\"\n snap_path = os.environ.get(\"SNAP\")\n control_plane_label = \"node.kubernetes.io/microk8s-controlplane=microk8s-controlplane\"\n nodes_info = subprocess.check_output(\n \"{}/microk8s-kubectl.wrapper get no -o json -l {}\".format(\n snap_path, control_plane_label\n ).split()\n )\n info = json.loads(nodes_info.decode())\n node_ips = []\n for node_info in info[\"items\"]:\n node_ip = get_internal_ip_from_get_node(node_info)\n node_ips.append(node_ip)\n return node_ips\n\n\ndef get_internal_ip_from_get_node(node_info):\n \"\"\"\n Retrieves the InternalIp returned by kubectl get no -o json\n \"\"\"\n for status_addresses in node_info[\"status\"][\"addresses\"]:\n if status_addresses[\"type\"] == \"InternalIP\":\n return status_addresses[\"address\"]\n\n\ndef is_same_server(hostname, ip):\n \"\"\"\n Check if the hostname is the same as the current node's hostname\n \"\"\"\n try:\n hname, _, _ = socket.gethostbyaddr(ip)\n if hname == hostname:\n return True\n except socket.error:\n # Ignore any unresolvable IP by host, surely this is not from the same node.\n pass\n\n return False\n\n\ndef apply_cni_manifest(timeout_insec=60):\n \"\"\"\n Apply the CNI yaml. If applying the manifest fails an exception is raised.\n :param timeout_insec: Try up to timeout seconds to apply the manifest.\n \"\"\"\n yaml = \"{}/args/cni-network/cni.yaml\".format(os.environ.get(\"SNAP_DATA\"))\n snap_path = os.environ.get(\"SNAP\")\n cmd = \"{}/microk8s-kubectl.wrapper apply -f {}\".format(snap_path, yaml)\n deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec)\n while True:\n try:\n check_output(cmd.split()).strip().decode(\"utf8\")\n break\n except CalledProcessError as err:\n output = err.output.strip().decode(\"utf8\").replace(\"\\\\n\", \"\\n\")\n print(\"Applying {} failed with {}\".format(yaml, output))\n if datetime.datetime.now() > deadline:\n raise\n print(\"Retrying {}\".format(cmd))\n time.sleep(3)\n\n\ndef cni_is_patched():\n \"\"\"\n Detect if the cni.yaml manifest already has the hint for detecting nodes routing paths\n :return: True if calico knows where the rest of the nodes are.\n \"\"\"\n yaml = \"{}/args/cni-network/cni.yaml\".format(os.environ.get(\"SNAP_DATA\"))\n with open(yaml) as f:\n if \"can-reach\" in f.read():\n return True\n else:\n return False\n\n\ndef cni_yaml_exists():\n \"\"\"\n Detect if the cni.yaml manifest exists.\n :return: True if calico cni.yaml exists.\n \"\"\"\n yaml = \"{}/args/cni-network/cni.yaml\".format(os.environ.get(\"SNAP_DATA\"))\n return os.path.exists(yaml)\n\n\ndef patch_cni(ip):\n \"\"\"\n Patch the cni.yaml manifest with the proper hint on where the rest of the nodes are\n :param ip: The IP another k8s node has.\n \"\"\"\n cni_yaml = \"{}/args/cni-network/cni.yaml\".format(os.environ.get(\"SNAP_DATA\"))\n backup_file = \"{}.backup\".format(cni_yaml)\n with open(backup_file, \"w\") as back_fp:\n with open(cni_yaml, \"r\") as fp:\n for _, line in enumerate(fp):\n if \"first-found\" in line:\n line = line.replace(\"first-found\", \"can-reach={}\".format(ip))\n back_fp.write(\"{}\".format(line))\n\n try_set_file_permissions(backup_file)\n shutil.copyfile(backup_file, cni_yaml)\n\n\ndef try_initialise_cni_autodetect_for_clustering(ip, apply_cni=True):\n \"\"\"\n Try to initialise the calico route autodetection based on the IP\n provided, see https://docs.projectcalico.org/networking/ip-autodetection.\n If the cni manifest got changed by default it gets reapplied.\n :param ip: The IP another k8s node has.\n :param apply_cni: Should we apply the the manifest\n \"\"\"\n if not cni_yaml_exists() or cni_is_patched():\n return True\n\n patch_cni(ip)\n if apply_cni:\n apply_cni_manifest()\n\n\ndef is_kubelite():\n \"\"\"\n Do we run kubelite?\n \"\"\"\n snap_data = os.environ.get(\"SNAP_DATA\")\n if not snap_data:\n snap_data = \"/var/snap/microk8s/current/\"\n kubelite_lock = \"{}/var/lock/lite.lock\".format(snap_data)\n return os.path.exists(kubelite_lock)\n\n\ndef service(operation, service_name):\n \"\"\"\n Restart a service. Handle case where kubelite is enabled.\n\n :param service_name: The service name\n :param operation: Operation to perform on the service\n \"\"\"\n if service_name in [\"apiserver\", \"proxy\", \"kubelet\", \"scheduler\", \"controller-manager\"]:\n daemon = \"microk8s.daemon-kubelite\"\n else:\n daemon = \"microk8s.daemon-{}\".format(service_name)\n\n subprocess.check_call([\"snapctl\", operation, daemon])\n\n\ndef mark_no_cert_reissue():\n \"\"\"\n Mark a node as being part of a cluster that should not re-issue certs\n on network changes\n \"\"\"\n snap_data = os.environ.get(\"SNAP_DATA\")\n lock_file = \"{}/var/lock/no-cert-reissue\".format(snap_data)\n open(lock_file, \"a\").close()\n os.chmod(lock_file, 0o700)\n\n\ndef unmark_no_cert_reissue():\n \"\"\"\n Unmark a node as being part of a cluster. The node should now re-issue certs\n on network changes\n \"\"\"\n snap_data = os.environ.get(\"SNAP_DATA\")\n lock_file = \"{}/var/lock/no-cert-reissue\".format(snap_data)\n if os.path.exists(lock_file):\n os.unlink(lock_file)\n\n\ndef restart_all_services():\n \"\"\"\n Restart all services\n \"\"\"\n snap_path = os.environ.get(\"SNAP\")\n waits = 10\n while waits > 0:\n try:\n subprocess.check_call(\n \"{}/microk8s-stop.wrapper\".format(snap_path).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n break\n except subprocess.CalledProcessError:\n time.sleep(5)\n waits -= 1\n waits = 10\n while waits > 0:\n try:\n subprocess.check_call(\n \"{}/microk8s-start.wrapper\".format(snap_path).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n break\n except subprocess.CalledProcessError:\n time.sleep(5)\n waits -= 1\n\n\ndef get_token(name, tokens_file=\"known_tokens.csv\"):\n \"\"\"\n Get token from known_tokens file\n\n :param name: the name of the node\n :param tokens_file: the file where the tokens should go\n :returns: the token or None(if name doesn't exist)\n \"\"\"\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n file = \"{}/credentials/{}\".format(snapdata_path, tokens_file)\n with open(file) as fp:\n for line in fp:\n if name in line:\n parts = line.split(\",\")\n return parts[0].rstrip()\n return None\n\n\ndef get_arg(key, file):\n \"\"\"\n Get an argument from a file\n\n :param key: argument name\n :param file: the arguments file\n :return: value\n \"\"\"\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n\n filename = \"{}/args/{}\".format(snapdata_path, file)\n with open(filename, \"r+\") as fp:\n for _, line in enumerate(fp):\n if line.startswith(key):\n parts = re.split(r\" |=\", line)\n return parts[-1]\n return None\n\n\ndef set_arg(key, value, file):\n \"\"\"\n Set an argument to a file\n\n :param key: argument name\n :param value: value\n :param file: the arguments file\n \"\"\"\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n\n filename = \"{}/args/{}\".format(snapdata_path, file)\n filename_remote = \"{}/args/{}.remote\".format(snapdata_path, file)\n done = False\n with open(filename_remote, \"w+\") as back_fp:\n with open(filename, \"r+\") as fp:\n for _, line in enumerate(fp):\n if line.startswith(key):\n done = True\n if value is not None:\n back_fp.write(\"{}={}\\n\".format(key, value))\n else:\n back_fp.write(\"{}\".format(line))\n if not done and value is not None:\n back_fp.write(\"{}={}\\n\".format(key, value))\n\n shutil.copyfile(filename, \"{}.backup\".format(filename))\n try_set_file_permissions(\"{}.backup\".format(filename))\n shutil.copyfile(filename_remote, filename)\n try_set_file_permissions(filename)\n os.remove(filename_remote)\n\n\ndef is_token_auth_enabled():\n \"\"\"\n Return True if token auth is enabled\n \"\"\"\n if get_arg(\"--token-auth-file\", \"kube-apiserver\"):\n return True\n else:\n return False\n\n\ndef enable_token_auth(token):\n \"\"\"\n Turn on token auth and inject the admin token\n\n :param token: the admin token\n \"\"\"\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n\n file = \"{}/credentials/known_tokens.csv\".format(snapdata_path)\n with open(file, \"w\") as fp:\n fp.write(f'{token},admin,admin,\"system:masters\"\\n')\n\n try_set_file_permissions(file)\n set_arg(\"--token-auth-file\", \"${SNAP_DATA}/credentials/known_tokens.csv\", \"kube-apiserver\")\n\n\ndef ca_one_line(ca):\n \"\"\"\n The CA in one line\n :param ca: the ca\n :return: one line\n \"\"\"\n return base64.b64encode(ca.encode(\"utf-8\")).decode(\"utf-8\")\n\n\ndef rebuild_x509_auth_client_configs():\n \"\"\"\n Recreate all the client configs\n \"\"\"\n if is_token_auth_enabled():\n set_arg(\"--token-auth-file\", None, \"kube-apiserver\")\n\n subprocess.check_call(\n [f\"{snap()}/actions/common/utils.sh\", \"create_user_certs_and_configs\"],\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n\n\ndef get_valid_connection_parts(connection):\n \"\"\"\n Ensure that connection has a valid format :/[/]\n\n :param connection: the connection string\n :return: connection parts\n :raise:\n InvalidConnectionError: if connection string is not valid\n \"\"\"\n connection_parts = connection.split(\"/\")\n\n if len(connection_parts) not in [2, 3]:\n raise InvalidConnectionError(\n \"Expected format: :/[/]\"\n )\n\n master_ep = connection_parts[0].split(\":\")\n if len(master_ep) != 2:\n raise InvalidConnectionError(\n \"Expected format: :/[/]\"\n )\n\n try:\n ipaddress.ip_address(master_ep[0])\n except ValueError:\n raise InvalidConnectionError(\"Invalid master IP\")\n\n try:\n port = int(master_ep[1])\n if port < 1 or port > 65535:\n raise InvalidConnectionError(\"Master PORT not in range 1:65535\")\n except ValueError:\n raise InvalidConnectionError(\"Master PORT not a number\")\n\n if len(connection_parts[1]) < TOKEN_ΜΙΝ_LEN:\n raise InvalidConnectionError(f\"Cluster token size should be at least {TOKEN_ΜΙΝ_LEN} bytes\")\n\n if len(connection_parts) == 3 and len(connection_parts[2]) < FINGERPRINT_MIN_LEN:\n raise InvalidConnectionError(f\"Fingerprint should be at least {FINGERPRINT_MIN_LEN} bytes\")\n\n return connection_parts\n" }, { "path": "scripts/wrappers/common/utils.py", "content": "import fcntl\nimport getpass\nimport json\nimport os\nimport platform\nimport subprocess\nimport sys\nimport time\nfrom pathlib import Path\nimport logging\n\nimport click\nimport yaml\n\nfrom common.cluster.utils import (\n try_set_file_permissions,\n is_strict,\n)\n\nLOG = logging.getLogger(__name__)\n\nKUBECTL = os.path.expandvars(\"$SNAP/microk8s-kubectl.wrapper\")\n\n\ndef get_current_arch():\n # architecture mapping\n arch_mapping = {\n \"aarch64\": \"arm64\",\n \"armv7l\": \"armhf\",\n \"x86_64\": \"amd64\",\n \"s390x\": \"s390x\",\n \"ppc64le\": \"ppc64le\",\n \"ppc64el\": \"ppc64le\",\n }\n\n return arch_mapping[platform.machine()]\n\n\ndef snap() -> Path:\n try:\n return Path(os.environ[\"SNAP\"])\n except KeyError:\n return Path(\"/snap/microk8s/current\")\n\n\ndef snap_data() -> Path:\n try:\n return Path(os.environ[\"SNAP_DATA\"])\n except KeyError:\n return Path(\"/var/snap/microk8s/current\")\n\n\ndef snap_common() -> Path:\n try:\n return Path(os.environ[\"SNAP_COMMON\"])\n except KeyError:\n return Path(\"/var/snap/microk8s/common\")\n\n\ndef run(*args, die=True):\n # Add wrappers to $PATH\n env = os.environ.copy()\n env[\"PATH\"] += \":%s\" % os.environ[\"SNAP\"]\n result = subprocess.run(\n args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env\n )\n\n try:\n result.check_returncode()\n except subprocess.CalledProcessError as err:\n if die:\n if result.stderr:\n print(result.stderr.decode(\"utf-8\"))\n print(err)\n sys.exit(1)\n else:\n raise\n\n return result.stdout.decode(\"utf-8\")\n\n\ndef is_cluster_ready(with_ready_node=True):\n try:\n return \"service/kubernetes\" in kubectl_get(\"all\") and (\n not with_ready_node or \" Ready \" in kubectl_get(\"nodes\")\n )\n except Exception:\n return False\n\n\ndef is_ha_enabled():\n ha_lock = os.path.expandvars(\"${SNAP_DATA}/var/lock/ha-cluster\")\n return os.path.isfile(ha_lock)\n\n\ndef get_dqlite_info():\n cluster_dir = os.path.expandvars(\"${SNAP_DATA}/var/kubernetes/backend\")\n snap_path = os.environ.get(\"SNAP\")\n\n info = []\n\n if not is_ha_enabled():\n return info\n\n waits = 10\n while waits > 0:\n try:\n with open(\"{}/info.yaml\".format(cluster_dir), mode=\"r\") as f:\n data = yaml.safe_load(f)\n out = subprocess.check_output(\n \"{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt \"\n \"-k {dbdir}/cluster.key -f json k8s .cluster\".format(\n snappath=snap_path, dbdir=cluster_dir\n ).split(),\n timeout=4,\n stderr=subprocess.DEVNULL,\n )\n if data[\"Address\"] in out.decode():\n break\n else:\n time.sleep(5)\n waits -= 1\n except (subprocess.CalledProcessError, subprocess.TimeoutExpired):\n time.sleep(2)\n waits -= 1\n\n if waits == 0:\n return info\n\n nodes = json.loads(out.decode())\n for n in nodes:\n if n[\"Role\"] == 0:\n info.append((n[\"Address\"], \"voter\"))\n if n[\"Role\"] == 1:\n info.append((n[\"Address\"], \"standby\"))\n if n[\"Role\"] == 2:\n info.append((n[\"Address\"], \"spare\"))\n return info\n\n\ndef get_etcd_info():\n kube_apiserver_args = os.path.expandvars(\"${SNAP_DATA}/args/kube-apiserver\")\n with open(kube_apiserver_args, \"r\") as f:\n kube_apiserver_args_content = f.read()\n etcd_endpoints = []\n for line in kube_apiserver_args_content.split(\"\\n\"):\n if \"etcd-servers\" in line:\n server_url = get_server_urls(line)\n # list all etcd endpointss\n for endpoint in server_url.split(\",\"):\n if \"//\" in endpoint:\n ip_port = endpoint.split(\"//\")[1]\n etcd_endpoints.append(ip_port)\n break\n\n return etcd_endpoints\n\n\ndef get_server_urls(args):\n server_url = None\n parts = args.split(\"=\")\n if len(parts) == 2:\n # Argument has an equals sign, e.g. \"--etcd-servers=http://10.0.0.1:2379\"\n server_url = parts[1]\n elif len(parts) == 1:\n # Argument has a space, e.g. \"--etcd-servers http://10.0.0.1:2379\"\n server_url = args.split(\"--etcd-servers\")[1].strip()\n return server_url\n\n\ndef is_external_etcd():\n kube_apiserver_args = os.path.expandvars(\"${SNAP_DATA}/args/kube-apiserver\")\n with open(kube_apiserver_args, \"r\") as f:\n kube_apiserver_args_content = f.read()\n for line in kube_apiserver_args_content.split(\"\\n\"):\n if \"var/kubernetes/backend/kine.sock\" in line:\n return False\n return True\n\n\ndef is_cluster_locked():\n if (snap_data() / \"var/lock/clustered.lock\").exists():\n click.echo(\"This MicroK8s deployment is acting as a node in a cluster.\")\n click.echo(\"Please use the master node.\")\n sys.exit(1)\n\n\ndef wait_for_ready(timeout, with_ready_node=True):\n start_time = time.time()\n end_time = start_time + timeout\n\n while True:\n if is_cluster_ready(with_ready_node=with_ready_node):\n return True\n elif timeout and time.time() > end_time:\n return False\n else:\n time.sleep(2)\n\n\ndef exit_if_no_root():\n \"\"\"\n Exit if the user is not root\n \"\"\"\n if not os.geteuid() == 0:\n click.echo(\n \"Elevated permissions is needed for this operation. Please run this command with sudo.\"\n )\n exit(50)\n\n\ndef exit_if_stopped():\n stoppedLockFile = os.path.expandvars(\"${SNAP_DATA}/var/lock/stopped.lock\")\n if os.path.isfile(stoppedLockFile):\n print(\"microk8s is not running, try microk8s start\")\n exit(0)\n\n\ndef exit_if_no_permission():\n user = getpass.getuser()\n # test if we can access the default kubeconfig\n client_config_file = os.path.expandvars(\"${SNAP_DATA}/credentials/client.config\")\n if not os.access(client_config_file, os.R_OK):\n print(\"Insufficient permissions to access MicroK8s.\")\n print(\n \"You can either try again with sudo or add the user {} to the 'microk8s' group:\".format(\n user\n )\n )\n print(\"\")\n print(\" sudo usermod -a -G microk8s {}\".format(user))\n print(\" sudo chown -R $USER ~/.kube\")\n print(\"\")\n print(\n \"After this, reload the user groups either via a reboot or by running 'newgrp microk8s'.\"\n )\n exit(1)\n\n\ndef ensure_started():\n if (snap_data() / \"var/lock/stopped.lock\").exists():\n click.echo(\"microk8s is not running, try microk8s start\", err=True)\n sys.exit(1)\n\n\ndef kubectl_get(cmd, namespace=\"--all-namespaces\"):\n if namespace == \"--all-namespaces\":\n return run(KUBECTL, \"get\", cmd, \"--all-namespaces\", die=False)\n else:\n return run(KUBECTL, \"get\", cmd, \"-n\", namespace, die=False)\n\n\ndef kubectl_get_clusterroles():\n return run(\n KUBECTL,\n \"get\",\n \"clusterroles\",\n \"--show-kind\",\n \"--no-headers\",\n die=False,\n )\n\n\ndef is_community_addon(arch, addon_name):\n \"\"\"\n Check if an addon is part of the community repo.\n\n :param arch: architecture of the addon we are looking for\n :param addon_name: name of the addon we are looking for\n :return: True if the addon is in the community repo\n \"\"\"\n try:\n addons_yaml = f\"{os.environ['SNAP']}/addons/community/addons.yaml\"\n with open(addons_yaml, \"r\") as fin:\n addons = yaml.safe_load(fin)\n\n for addon in addons[\"microk8s-addons\"][\"addons\"]:\n if arch in addon[\"supported_architectures\"]:\n if addon_name == addon[\"name\"]:\n return True\n except Exception:\n LOG.exception(\"could not load addons from %s\", addons_yaml)\n\n return False\n\n\ndef get_available_addons(arch):\n available = []\n strict = is_strict()\n for dir in os.listdir(snap_common() / \"addons\"):\n try:\n addons_yaml = snap_common() / \"addons\" / dir / \"addons.yaml\"\n with open(addons_yaml, \"r\") as fin:\n addons = yaml.safe_load(fin)\n\n for addon in addons[\"microk8s-addons\"][\"addons\"]:\n if arch not in addon[\"supported_architectures\"]:\n continue\n\n if \"confinement\" in addon:\n if strict and \"strict\" not in addon[\"confinement\"]:\n continue\n if not strict and \"classic\" not in addon[\"confinement\"]:\n continue\n\n available.append({**addon, \"repository\": dir})\n\n except Exception:\n LOG.exception(\"could not load addons from %s\", addons_yaml)\n\n available = sorted(available, key=lambda k: (k[\"repository\"], k[\"name\"]))\n return available\n\n\ndef get_addon_by_name(addons, name):\n filtered_addon = []\n\n parts = name.split(\"/\")\n if len(parts) == 1:\n repo_name, addon_name = None, parts[0]\n elif len(parts) == 2:\n repo_name, addon_name = parts[0], parts[1]\n else:\n # just fallback to the addon name\n repo_name, addon_name = None, name\n\n for addon in addons:\n if addon_name == addon[\"name\"] and (repo_name == addon[\"repository\"] or not repo_name):\n filtered_addon.append(addon)\n\n return filtered_addon\n\n\ndef is_service_expected_to_start(service):\n \"\"\"\n Check if a service is supposed to start\n :param service: the service name\n :return: True if the service is meant to start\n \"\"\"\n lock_path = os.path.expandvars(\"${SNAP_DATA}/var/lock\")\n lock = \"{}/{}\".format(lock_path, service)\n return os.path.exists(lock_path) and not os.path.isfile(lock)\n\n\ndef set_service_expected_to_start(service, start=True):\n \"\"\"\n Check if a service is not expected to start.\n :param service: the service name\n :param start: should the service start or not\n \"\"\"\n lock_path = os.path.expandvars(\"${SNAP_DATA}/var/lock\")\n lock = \"{}/{}\".format(lock_path, service)\n if start:\n os.remove(lock)\n else:\n fd = os.open(lock, os.O_CREAT, mode=0o700)\n os.close(fd)\n\n\ndef check_help_flag(addons: list) -> bool:\n \"\"\"Checks to see if a help message needs to be printed for an addon.\n\n Not all addons check for help flags themselves. Until they do, intercept\n calls to print help text and print out a generic message to that effect.\n \"\"\"\n addon = addons[0]\n if any(help_arg in addons for help_arg in (\"-h\", \"--help\")):\n print(\"Addon %s does not yet have a help message.\" % addon)\n print(\"For more information about it, visit https://microk8s.io/docs/addons\")\n return True\n return False\n\n\ndef parse_xable_addon_args(addon_args: list, available_addons: list):\n \"\"\"\n Parse the list of addons passed into the microk8s enable or disable commands.\n Further, it will infer the repository name for addons when possible.\n If any errors are encountered, we print them to stderr and exit.\n\n :param addon_args: The parameters passed to the microk8s enable command\n :param available_addons: List of available addons as (repo_name, addon_name) tuples\n\n Handles the following cases:\n - microk8s enable foo bar:--baz # enable many addons, inline arguments\n - microk8s enable bar --baz # enable one addon, unix style command line arguments\n\n :return: a list of (repo_name, addon_name, args) tuples\n \"\"\"\n\n # Backwards compatibility with enabling multiple addons at once, e.g.\n # `microk8s.enable foo bar:\"baz\"`\n available_addon_names = [addon_name for (_, addon_name) in available_addons]\n available_addon_names += [\n \"/\".join([repo_name, addon_name]) for (repo_name, addon_name) in available_addons\n ]\n addon_names = [arg.split(\":\")[0] for arg in addon_args]\n if set(addon_names) < set(available_addon_names):\n return [parse_xable_single_arg(addon_arg, available_addons) for addon_arg in addon_args]\n\n # The new way of xabling addons, that allows for unix-style argument passing,\n # such as `microk8s.enable foo --bar`.\n repo_name, addon_name, args = parse_xable_single_arg(addon_args[0], available_addons)\n if args and addon_args[1:]:\n click.echo(\n \"Can't pass string arguments and flag arguments simultaneously!\\n\"\n \"Enable or disable addons with only one argument style at a time:\\n\"\n \"\\n\"\n \" microk8s enable foo:'bar'\\n\"\n \"or\\n\"\n \" microk8s enable foo --bar\\n\"\n )\n sys.exit(1)\n\n return [(repo_name, addon_name, addon_args[1:])]\n\n\ndef parse_xable_single_arg(addon_arg: str, available_addons: list):\n \"\"\"\n Parse an addon arg of the following form: `(repo_name/)addon_name(:args)`\n It will automatically infer the repository name if not specified. If multiple repositories\n are found for the addon, we print an error and exit.\n\n :param addon_arg: A parameter passed to the microk8s enable command\n :param available_addons: List of available addons as (repo_name, addon_name) tuples\n\n :return: a (repo_name, addon_name, args) tuple\n \"\"\"\n addon_name, *args = addon_arg.split(\":\")\n parts = addon_name.split(\"/\")\n if len(parts) == 2:\n return (parts[0], parts[1], args)\n elif len(parts) == 1:\n matching_repos = [repo for (repo, addon) in available_addons if addon == addon_name]\n if len(matching_repos) == 0:\n click.echo(\"Addon {} was not found in any repository\".format(addon_name), err=True)\n if is_community_addon(get_current_arch(), addon_name):\n click.echo(\n \"To use the community maintained flavor enable the respective repository:\"\n )\n click.echo(\"\")\n click.echo(\" microk8s enable community\")\n click.echo(\"\")\n\n sys.exit(1)\n elif len(matching_repos) == 1:\n click.echo(\n \"Infer repository {} for addon {}\".format(matching_repos[0], addon_name), err=True\n )\n return (matching_repos[0], addon_name, args)\n else:\n click.echo(\n \"Addon {} exists in more than repository. Please explicitly specify\\n\"\n \"the repository using any of:\\n\".format(addon_name),\n err=True,\n )\n for repo in matching_repos:\n click.echo(\" {}/{}\".format(repo, addon_name), err=True)\n click.echo(\"\", err=True)\n sys.exit(1)\n\n else:\n click.echo(\"Invalid addon name {}\".format(addon_name))\n sys.exit(1)\n\n\ndef xable(action: str, addon_args: list):\n if os.getenv(\"MICROK8S_ADDONS_SKIP_LOCK\") == \"1\":\n unprotected_xable(action, addon_args)\n else:\n protected_xable(action, addon_args)\n\n\ndef protected_xable(action: str, addon_args: list):\n \"\"\"\n Get an exclusive lock file and then perform enable/disable of addons.\n\n Ensure that the lock file is always unlocked on exit.\n \"\"\"\n\n lock_file_path = snap_data() / \"var/lock/.microk8s-addon-lock\"\n with open(lock_file_path, \"w\") as f:\n # set file permissions so non-root users do not fail\n try:\n try_set_file_permissions(lock_file_path)\n except OSError:\n pass\n\n try:\n fcntl.lockf(f, fcntl.LOCK_EX)\n\n # NOTE(neoaggelos): We now have the lock, ensure any recursive\n # invocations will not deadlock. One example is addons that\n # enable other addons as requirements.\n #\n # See the relevant check in xable().\n os.environ[\"MICROK8S_ADDONS_SKIP_LOCK\"] = \"1\"\n\n unprotected_xable(action, addon_args)\n finally:\n fcntl.lockf(f, fcntl.LOCK_UN)\n\n\ndef unprotected_xable(action: str, addon_args: list):\n \"\"\"Enables or disables the given addons.\n\n Collated into a single function since the logic is identical other than\n the script names.\n\n :param action: \"enable\" or \"disable\"\n :param addons: List of addons to enable. Each addon may be prefixed with `repository/`\n to specify which addon repository it will be sourced from.\n \"\"\"\n available_addons_info = get_available_addons(get_current_arch())\n enabled_addons_info, disabled_addons_info = get_status(available_addons_info, True)\n if action == \"enable\":\n xabled_addons_info = enabled_addons_info\n elif action == \"disable\":\n xabled_addons_info = disabled_addons_info\n else:\n click.echo(\"Invalid action {}. Only enable and disable are supported\".format(action))\n sys.exit(1)\n\n # available_addons is a list of (repo_name, addon_name) tuples for all available addons\n available_addons = [(addon[\"repository\"], addon[\"name\"]) for addon in available_addons_info]\n # xabled_addons is a list (repo_name, addon_name) tuples of already xabled addons\n xabled_addons = [(addon[\"repository\"], addon[\"name\"]) for addon in xabled_addons_info]\n\n addons = parse_xable_addon_args(addon_args, available_addons)\n if len(addons) > 1:\n click.echo(\n \"WARNING: Do not enable or disable multiple addons in one command.\\n\"\n \" This form of chained operations on addons will be DEPRECATED in the future.\\n\"\n f\" Please, {action} one addon at a time: 'microk8s {action} '\"\n )\n\n for repo_name, addon_name, args in addons:\n if (repo_name, addon_name) not in available_addons:\n click.echo(\"Addon {}/{} not found\".format(repo_name, addon_name))\n continue\n if (repo_name, addon_name) in xabled_addons:\n click.echo(\"Addon {}/{} is already {}d\".format(repo_name, addon_name, action))\n continue\n\n wait_for_ready(timeout=30, with_ready_node=False)\n p = subprocess.run(\n [snap_common() / \"addons\" / repo_name / \"addons\" / addon_name / action, *args]\n )\n if p.returncode:\n sys.exit(p.returncode)\n wait_for_ready(timeout=30, with_ready_node=False)\n\n\ndef is_enabled(addon, item):\n if addon in item:\n return True\n else:\n filepath = os.path.expandvars(addon)\n return os.path.isfile(filepath)\n\n\ndef get_status(available_addons, isReady):\n enabled = []\n disabled = []\n if isReady:\n # 'all' does not include ingress\n kube_output = kubectl_get(\"all,ingress,ingressclass\")\n cluster_output = kubectl_get_clusterroles()\n kube_output = kube_output + cluster_output\n for addon in available_addons:\n found = False\n for row in kube_output.split(\"\\n\"):\n if is_enabled(addon[\"check_status\"], row):\n enabled.append(addon)\n found = True\n break\n if not found:\n disabled.append(addon)\n\n return enabled, disabled\n\n\ndef is_within_directory(directory, target):\n abs_directory = os.path.abspath(directory)\n abs_target = os.path.abspath(target)\n\n prefix = os.path.commonprefix([abs_directory, abs_target])\n\n return prefix == abs_directory\n\n\ndef safe_extract(tar, path=\".\", members=None, *, numeric_owner=False):\n for member in tar.getmembers():\n member_path = os.path.join(path, member.name)\n if not is_within_directory(path, member_path):\n raise Exception(\"Attempted Path Traversal in Tar File\")\n\n tar.extractall(path, members, numeric_owner=numeric_owner)\n" }, { "path": "scripts/wrappers/dashboard_proxy.py", "content": "#!/usr/bin/python3\n\nimport base64\nimport os\nfrom subprocess import check_output, run, CalledProcessError\nimport time\n\nimport click\n\nMICROK8S_ENABLE = os.path.expandvars(\"$SNAP/microk8s-enable.wrapper\")\nKUBECTL = os.path.expandvars(\"$SNAP/microk8s-kubectl.wrapper\")\nSECRET_YAML = \"\"\"\napiVersion: v1\nkind: Secret\nmetadata:\n name: microk8s-dashboard-token\n namespace: kube-system\n annotations:\n kubernetes.io/service-account.name: \"default\"\ntype: kubernetes.io/service-account-token\n\"\"\"\n\n\ndef get_token(secret):\n \"\"\"\n Get a token to be used\n \"\"\"\n token = None\n print(\"Trying to get token from {}\".format(secret))\n for attempt in range(3):\n print(\"Waiting for secret token (attempt {})\".format(attempt))\n command = [\n KUBECTL,\n \"-n\",\n \"kube-system\",\n \"get\",\n \"secret\",\n secret,\n \"-o\",\n \"jsonpath={.data.token}\",\n ]\n try:\n output = check_output(command)\n except CalledProcessError:\n time.sleep(5)\n continue\n\n if output:\n token = base64.b64decode(output).decode()\n break\n\n time.sleep(5)\n\n return token\n\n\n@click.command(context_settings={\"help_option_names\": [\"-h\", \"--help\"]})\ndef dashboard_proxy():\n \"\"\"\n Enable the dashboard add-on and configures port-forwarding\n to allow accessing the dashboard from the local machine.\n \"\"\"\n print(\"Checking if Dashboard is running.\")\n command = [MICROK8S_ENABLE, \"dashboard\"]\n output = check_output(command)\n\n ns = \"kubernetes-dashboard\"\n deploy = \"kubernetes-dashboard-kong\"\n service = \"kubernetes-dashboard-kong-proxy\"\n namespaces = check_output([KUBECTL, \"get\", \"ns\"])\n if b\"kubernetes-dashboard\" not in namespaces:\n # NOTE(Hue): Backwards compatibility for installs in kube-system ns\n ns = \"kube-system\"\n deploy = \"kubernetes-dashboard\"\n service = \"kubernetes-dashboard\"\n\n if b\"Addon dashboard is already enabled.\" not in output:\n print(\"Waiting for Dashboard to come up.\")\n command = [\n KUBECTL,\n \"-n\",\n ns,\n \"wait\",\n \"--timeout=240s\",\n \"deployment\",\n deploy,\n \"--for\",\n \"condition=available\",\n ]\n check_output(command)\n\n # Get token created by the dashboard enable script\n token = get_token(\"microk8s-dashboard-token\")\n if not token:\n # Create a token. This is needed in case the enable script is pre-1.25.\n print(\"Create token for accessing the dashboard\")\n run([KUBECTL, \"apply\", \"-f\", \"-\"], input=SECRET_YAML.encode(\"ascii\"))\n token = get_token(\"microk8s-dashboard-token\")\n\n print(\"Dashboard will be available at https://127.0.0.1:10443\")\n print(\"Use the following token to login:\")\n print(token)\n\n command = [\n KUBECTL,\n \"port-forward\",\n \"-n\",\n ns,\n f\"service/{service}\",\n \"10443:443\",\n \"--address\",\n \"0.0.0.0\",\n ]\n\n try:\n check_output(command)\n except KeyboardInterrupt:\n exit(0)\n\n\nif __name__ == \"__main__\":\n dashboard_proxy()\n" }, { "path": "scripts/wrappers/dbctl.py", "content": "#!/usr/bin/python3\nimport os\nimport argparse\n\nimport tempfile\nimport datetime\nimport subprocess\nimport tarfile\nimport os.path\n\nfrom common.utils import (\n exit_if_no_permission,\n is_cluster_locked,\n is_ha_enabled,\n snap_data,\n safe_extract,\n)\n\n\ndef get_kine_endpoint():\n \"\"\"\n Return the default kine endpoint\n \"\"\"\n return \"unix://{}/var/kubernetes/backend/kine.sock:12379\".format(snap_data())\n\n\ndef kine_exists():\n \"\"\"\n Check the existence of the kine socket\n :return: True if the kine socket exists\n \"\"\"\n kine_socket = get_kine_endpoint()\n kine_socket_path = kine_socket.replace(\"unix://\", \"\")\n return os.path.exists(kine_socket_path)\n\n\ndef generate_backup_name():\n \"\"\"\n Generate a filename based on the current time and date\n :return: a generated filename\n \"\"\"\n now = datetime.datetime.now()\n return \"backup-{}\".format(now.strftime(\"%Y-%m-%d-%H-%M-%S\"))\n\n\ndef run_command(command):\n \"\"\"\n Run a command while printing the output\n :param command: the command to run\n :return: the return code of the command\n \"\"\"\n process = subprocess.Popen(command.split(), stdout=subprocess.PIPE)\n while True:\n output = process.stdout.readline()\n if (not output or output == \"\") and process.poll() is not None:\n break\n if output:\n print(output.decode().strip())\n rc = process.poll()\n return rc\n\n\ndef backup(fname=None, debug=False):\n \"\"\"\n Backup the database to a provided file\n :param fname_tar: the tar file\n :param debug: show debug output\n \"\"\"\n snap_path = os.environ.get(\"SNAP\")\n kine_ep = get_kine_endpoint()\n\n if not fname:\n fname = generate_backup_name()\n if fname.endswith(\".tar.gz\"):\n fname = fname[:-7]\n fname_tar = \"{}.tar.gz\".format(fname)\n\n with tempfile.TemporaryDirectory() as tmpdirname:\n backup_cmd = (\n \"{}/bin/k8s-dqlite migrator --endpoint {} --mode backup-dqlite --db-dir {}\".format(\n snap_path, kine_ep, \"{}/{}\".format(tmpdirname, fname)\n )\n )\n if debug:\n backup_cmd = \"{} {}\".format(backup_cmd, \"--debug\")\n try:\n rc = run_command(backup_cmd)\n if rc > 0:\n print(\"Backup process failed. {}\".format(rc))\n exit(1)\n with tarfile.open(fname_tar, \"w:gz\") as tar:\n tar.add(\n \"{}/{}\".format(tmpdirname, fname),\n arcname=os.path.basename(\"{}/{}\".format(tmpdirname, fname)),\n )\n\n print(\"The backup is: {}\".format(fname_tar))\n except subprocess.CalledProcessError as e:\n print(\"Backup process failed. {}\".format(e))\n exit(2)\n\n\ndef restore(fname_tar, debug=False):\n \"\"\"\n Restore the database from the provided file\n :param fname_tar: the tar file\n :param debug: show debug output\n \"\"\"\n snap_path = os.environ.get(\"SNAP\")\n kine_ep = get_kine_endpoint()\n with tempfile.TemporaryDirectory() as tmpdirname:\n with tarfile.open(fname_tar, \"r:gz\") as tar:\n safe_extract(tar, path=tmpdirname)\n if fname_tar.endswith(\".tar.gz\"):\n fname = fname_tar[:-7]\n else:\n fname = fname_tar\n fname = os.path.basename(fname)\n restore_cmd = (\n \"{}/bin/k8s-dqlite migrator --endpoint {} --mode restore-to-dqlite --db-dir {}\".format(\n snap_path, kine_ep, \"{}/{}\".format(tmpdirname, fname)\n )\n )\n if debug:\n restore_cmd = \"{} {}\".format(restore_cmd, \"--debug\")\n try:\n rc = run_command(restore_cmd)\n if rc > 0:\n print(\"Restore process failed. {}\".format(rc))\n exit(3)\n except subprocess.CalledProcessError as e:\n print(\"Restore process failed. {}\".format(e))\n exit(4)\n\n\nif __name__ == \"__main__\":\n exit_if_no_permission()\n is_cluster_locked()\n\n if not kine_exists() or not is_ha_enabled():\n print(\"Please ensure the kubernetes apiserver is running and HA is enabled.\")\n exit(10)\n\n # initiate the parser with a description\n parser = argparse.ArgumentParser(\n description=\"backup and restore the Kubernetes datastore.\", prog=\"microk8s dbctl\"\n )\n parser.add_argument(\"--debug\", action=\"store_true\", help=\"print debug output\")\n commands = parser.add_subparsers(title=\"commands\", help=\"backup and restore operations\")\n restore_parser = commands.add_parser(\"restore\")\n restore_parser.add_argument(\"backup-file\", help=\"name of file with the backup\")\n backup_parser = commands.add_parser(\"backup\")\n backup_parser.add_argument(\"-o\", metavar=\"backup-file\", help=\"output filename\")\n args = parser.parse_args()\n\n if \"backup-file\" in args:\n fname = vars(args)[\"backup-file\"]\n print(\"Restoring from {}\".format(fname))\n restore(fname, args.debug)\n elif \"o\" in args:\n print(\"Backing up the datastore\")\n backup(vars(args)[\"o\"], args.debug)\n else:\n parser.print_help()\n" }, { "path": "scripts/wrappers/disable.py", "content": "#!/usr/bin/env python3\n\nimport click\n\nfrom common.utils import (\n ensure_started,\n exit_if_no_permission,\n is_cluster_locked,\n check_help_flag,\n wait_for_ready,\n xable,\n)\n\n\n@click.command(\n context_settings={\n \"ignore_unknown_options\": True,\n \"help_option_names\": [\"-h\", \"--help\"],\n },\n)\n@click.argument(\"addons\", nargs=-1, required=True)\ndef disable(addons):\n \"\"\"Disable one or more MicroK8s addons.\n\n For a list of available addons, run `microk8s status`.\n\n To see help for individual addons, run:\n\n microk8s disable ADDON -- --help\n \"\"\"\n\n if check_help_flag(addons):\n return\n\n is_cluster_locked()\n exit_if_no_permission()\n ensure_started()\n wait_for_ready(timeout=30, with_ready_node=False)\n\n xable(\"disable\", addons)\n\n\nif __name__ == \"__main__\":\n disable(prog_name=\"microk8s disable\")\n" }, { "path": "scripts/wrappers/distributed_op.py", "content": "#!/usr/bin/python3\nimport getopt\nimport subprocess\n\nimport requests\nimport urllib3\nimport os\nimport sys\nimport json\nimport socket\nimport time\n\nfrom common.cluster.utils import (\n get_callback_token,\n get_cluster_agent_port,\n is_node_running_dqlite,\n get_internal_ip_from_get_node,\n is_same_server,\n)\n\n\nurllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\nCLUSTER_API_V1 = \"cluster/api/v1.0\"\nCLUSTER_API_V2 = \"cluster/api/v2.0\"\nsnapdata_path = os.environ.get(\"SNAP_DATA\")\nsnap_path = os.environ.get(\"SNAP\")\ncallback_tokens_file = \"{}/credentials/callback-tokens.txt\".format(snapdata_path)\ncallback_token_file = \"{}/credentials/callback-token.txt\".format(snapdata_path)\n\nKUBECTL = \"{}/microk8s-kubectl.wrapper\".format(snap_path)\nMICROK8S_STATUS = \"{}/microk8s-status.wrapper\".format(snap_path)\n\n\ndef get_cluster_agent_endpoints(include_self=False):\n \"\"\"\n Get a list of all cluster agent endpoints and their callback token.\n\n :param include_self: If true, include the current node in the list.\n\n :return: [(\"node1:25000\", \"token1\"), (\"node2:25000\", \"token2\"), ...]\n \"\"\"\n nodes = []\n if is_node_running_dqlite():\n hostname = socket.gethostname()\n token = get_callback_token()\n\n for attempt in range(10):\n try:\n stdout = subprocess.check_output([KUBECTL, \"get\", \"node\", \"-o\", \"json\"])\n break\n except subprocess.CalledProcessError as e:\n print(\"Failed to list nodes (try {}): {}\".format(attempt + 1, e), file=sys.stderr)\n if attempt == 9:\n raise e\n time.sleep(3)\n\n info = json.loads(stdout)\n for node_info in info[\"items\"]:\n node_ip = get_internal_ip_from_get_node(node_info)\n if not include_self and is_same_server(hostname, node_ip):\n continue\n\n nodes.append((\"{}:25000\".format(node_ip), token.rstrip()))\n else:\n if include_self:\n token = get_callback_token()\n port = get_cluster_agent_port()\n nodes.append((\"127.0.0.1:{}\".format(port), token.rstrip()))\n\n try:\n with open(callback_tokens_file, \"r+\") as fin:\n for line in fin:\n node_ep, token = line.split()\n host = node_ep.split(\":\")[0]\n\n try:\n subprocess.check_call(\n [KUBECTL, \"get\", \"node\", host],\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n nodes.append((node_ep, token.rstrip()))\n except subprocess.CalledProcessError:\n print(\"Node {} not present\".format(host))\n except OSError:\n pass\n return nodes\n\n\ndef do_configure_op(remote_op):\n \"\"\"\n Perform a /configure operation on all remote nodes\n\n :param remote_op: the operation json string\n \"\"\"\n try:\n endpoints = get_cluster_agent_endpoints(include_self=False)\n except subprocess.CalledProcessError as e:\n print(\"Could not query for nodes\")\n raise SystemExit(e)\n\n for node_ep, token in endpoints:\n try:\n remote_op[\"callback\"] = token.rstrip()\n # TODO: handle ssl verification\n res = requests.post(\n \"https://{}/{}/configure\".format(node_ep, CLUSTER_API_V1),\n json=remote_op,\n verify=False,\n )\n if res.status_code != 200:\n print(\n \"Failed to perform a {} on node {} {}\".format(\n remote_op[\"action_str\"], node_ep, res.status_code\n )\n )\n except requests.exceptions.RequestException as e:\n print(\"Failed to reach node.\")\n raise SystemExit(e)\n\n\ndef do_image_import(image_data):\n \"\"\"\n Perform a /image/import operation on all nodes\n\n :param image_data: Raw bytes of the OCI image tar file\n \"\"\"\n\n try:\n endpoints = get_cluster_agent_endpoints(include_self=True)\n except subprocess.CalledProcessError as e:\n print(\"Could not query for nodes\")\n raise SystemExit(e)\n\n for node_ep, token in endpoints:\n try:\n print(\"Pushing OCI images to {}\".format(node_ep))\n res = requests.post(\n \"https://{}/{}/image/import\".format(node_ep, CLUSTER_API_V2),\n data=image_data,\n headers={\n \"x-microk8s-callback-token\": token,\n },\n verify=False,\n )\n\n if res.status_code != 200:\n print(\"Failed to import images on {}: {}\".format(node_ep, res.content.decode()))\n except requests.exceptions.RequestException as e:\n print(\"Failed to reach {}: {}\".format(node_ep, e))\n\n\ndef restart(service):\n \"\"\"\n Restart service on all nodes\n\n :param service: the service name\n \"\"\"\n print(\"Restarting nodes.\")\n remote_op = {\n \"action_str\": \"restart {}\".format(service),\n \"service\": [{\"name\": service, \"restart\": \"yes\"}],\n }\n do_configure_op(remote_op)\n\n\ndef update_argument(service, key, value):\n \"\"\"\n Configure an argument on all nodes\n\n :param service: the service we configure\n :param key: the argument we configure\n :param value: the value we set\n \"\"\"\n print(\"Adding argument {} to nodes.\".format(key))\n remote_op = {\n \"action_str\": \"change of argument {} to {}\".format(key, value),\n \"service\": [{\"name\": service, \"arguments_update\": [{key: value}]}],\n }\n do_configure_op(remote_op)\n\n\ndef remove_argument(service, key):\n \"\"\"\n Drop an argument from all nodes\n\n :param service: the service we configure\n :param key: the argument we configure\n \"\"\"\n print(\"Removing argument {} from nodes.\".format(key))\n remote_op = {\n \"action_str\": \"removal of argument {}\".format(key),\n \"service\": [{\"name\": service, \"arguments_remove\": [key]}],\n }\n do_configure_op(remote_op)\n\n\ndef set_addon(addon, state):\n \"\"\"\n Enable or disable an add-on across all nodes\n\n :param addon: the add-on name\n :param state: 'enable' or 'disable'\n \"\"\"\n if state not in (\"enable\", \"disable\"):\n raise ValueError(\n \"Wrong value '{}' for state. Must be one of 'enable' or 'disable'\".format(state)\n )\n else:\n print(\"Setting add-on {} to {} on nodes.\".format(addon, state))\n remote_op = {\n \"action_str\": \"set of {} to {}\".format(addon, state),\n \"addon\": [{\"name\": addon, state: \"true\"}],\n }\n do_configure_op(remote_op)\n\n\ndef usage():\n print(\"usage: dist_refresh_opt [OPERATION] [SERVICE] (ARGUMENT) (value)\")\n print(\"OPERATION is one of restart, update_argument, remove_argument, set_addon\")\n\n\nif __name__ == \"__main__\":\n if is_node_running_dqlite() and not os.path.isfile(callback_token_file):\n # print(\"Single node cluster.\")\n exit(0)\n\n if not is_node_running_dqlite() and not os.path.isfile(callback_tokens_file):\n print(\"No callback tokens file.\")\n exit(1)\n\n try:\n opts, args = getopt.getopt(sys.argv[1:], \"h\", [\"help\"])\n except getopt.GetoptError as err:\n # print help information and exit:\n print(err) # will print something like \"option -a not recognized\"\n usage()\n sys.exit(2)\n for o, a in opts:\n if o in (\"-h\", \"--help\"):\n usage()\n sys.exit()\n else:\n assert False, \"unhandled option\"\n\n operation = args[0]\n service = args[1]\n if operation == \"restart\":\n restart(service)\n if operation == \"update_argument\":\n update_argument(service, args[2], args[3])\n if operation == \"remove_argument\":\n remove_argument(service, args[2])\n if operation == \"set_addon\":\n set_addon(service, args[2])\n exit(0)\n" }, { "path": "scripts/wrappers/enable.py", "content": "#!/usr/bin/env python3\n\nimport click\n\nfrom common.utils import (\n ensure_started,\n exit_if_no_permission,\n is_cluster_locked,\n check_help_flag,\n wait_for_ready,\n xable,\n)\n\n\n@click.command(\n context_settings={\"ignore_unknown_options\": True, \"help_option_names\": [\"-h\", \"--help\"]},\n)\n@click.argument(\"addons\", nargs=-1, required=True)\ndef enable(addons) -> None:\n \"\"\"\n Enable a MicroK8s addon.\n\n For a list of available addons, run `microk8s status`.\n\n To see help for individual addons, run:\n\n microk8s enable ADDON -- --help\n \"\"\"\n if check_help_flag(addons):\n return\n\n is_cluster_locked()\n exit_if_no_permission()\n ensure_started()\n wait_for_ready(timeout=30, with_ready_node=False)\n\n xable(\"enable\", addons)\n\n\nif __name__ == \"__main__\":\n enable(prog_name=\"microk8s enable\")\n" }, { "path": "scripts/wrappers/images.py", "content": "#!/usr/bin/python3\nimport os\nimport subprocess\nimport sys\nfrom typing import List\n\nimport click\n\nfrom distributed_op import do_image_import\n\nCTR = \"{}/microk8s-ctr.wrapper\".format(os.getenv(\"SNAP\"))\n\nimages = click.Group()\n\n\n@images.command(\"import\", help=\"Import OCI images into the MicroK8s cluster\")\n@click.argument(\"image\", default=\"-\")\ndef import_images(image: str):\n\n if image == \"-\":\n image_data = sys.stdin.buffer.read()\n else:\n try:\n with open(image, \"rb\") as fin:\n image_data = fin.read()\n except OSError as e:\n click.echo(\"Error: failed to read {}: {}\".format(image, e), err=True)\n sys.exit(1)\n\n do_image_import(image_data)\n\n\ndef get_all_ctr_images():\n \"\"\"\n Return list of all OCI images from containerd.\n \"\"\"\n images = subprocess.check_output([CTR, \"image\", \"ls\", \"--quiet\"]).decode().split(\"\\n\")\n\n # drop the sha256-digest aliases\n return [tag for tag in images if tag and not tag.startswith(\"sha256:\")]\n\n\n@images.command(\"export-local\", help=\"Export OCI images from the current MicroK8s node\")\n@click.argument(\"output\", default=\"-\")\n@click.argument(\"images\", nargs=-1)\ndef export_images(output: str, images: List[str]):\n if not images:\n images = get_all_ctr_images()\n\n for image in images:\n click.echo(\"Checking {}\".format(image), err=True)\n try:\n subprocess.check_call([CTR, \"image\", \"export\", \"-\", image], stdout=subprocess.DEVNULL)\n except subprocess.CalledProcessError:\n subprocess.check_call(\n [CTR, \"content\", \"fetch\", \"--all-platforms\", image], stdout=sys.stderr\n )\n\n subprocess.check_call([CTR, \"image\", \"export\", output, *images])\n\n\nif __name__ == \"__main__\":\n images(prog_name=\"microk8s images\")\n" }, { "path": "scripts/wrappers/join.py", "content": "#!/usr/bin/python3\nimport hashlib\nimport http\nimport json\nimport os\nimport random\nimport shutil\nimport socket\nimport ssl\nimport string\nimport subprocess\nimport sys\nimport time\nimport ipaddress\n\nimport click\nimport requests\nimport urllib3\nimport yaml\nfrom common.cluster.utils import (\n ca_one_line,\n enable_token_auth,\n get_cluster_agent_port,\n get_cluster_cidr,\n get_token,\n get_valid_connection_parts,\n is_low_memory_guard_enabled,\n is_node_running_dqlite,\n is_token_auth_enabled,\n mark_no_cert_reissue,\n rebuild_x509_auth_client_configs,\n service,\n set_arg,\n try_initialise_cni_autodetect_for_clustering,\n try_set_file_permissions,\n snap,\n snap_data,\n FINGERPRINT_MIN_LEN,\n InvalidConnectionError,\n)\n\nurllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\nCLUSTER_API = \"cluster/api/v1.0\"\nsnapdata_path = os.environ.get(\"SNAP_DATA\")\nsnap_path = os.environ.get(\"SNAP\")\nca_cert_file_via_env = \"${SNAP_DATA}/certs/ca.remote.crt\"\nca_cert_file = \"{}/certs/ca.remote.crt\".format(snapdata_path)\ncallback_token_file = \"{}/credentials/callback-token.txt\".format(snapdata_path)\ncallback_tokens_file = \"{}/credentials/callback-tokens.txt\".format(snapdata_path)\nserver_cert_file_via_env = \"${SNAP_DATA}/certs/server.remote.crt\"\nserver_cert_file = \"{}/certs/server.remote.crt\".format(snapdata_path)\n\nCLUSTER_API_V2 = \"cluster/api/v2.0\"\ncluster_dir = \"{}/var/kubernetes/backend\".format(snapdata_path)\ncluster_backup_dir = \"{}/var/kubernetes/backend.backup\".format(snapdata_path)\ncluster_cert_file = \"{}/cluster.crt\".format(cluster_dir)\ncluster_key_file = \"{}/cluster.key\".format(cluster_dir)\n\n\ndef get_traefik_port():\n \"\"\"\n Return the port Traefik listens to. Try read the port from the Traefik configuration or return the default value\n \"\"\"\n config_file = \"{}/args/traefik/traefik-template.yaml\".format(snapdata_path)\n with open(config_file) as f:\n data = yaml.load(f, Loader=yaml.FullLoader)\n if (\n \"entryPoints\" in data\n and \"apiserver\" in data[\"entryPoints\"]\n and \"address\" in data[\"entryPoints\"][\"apiserver\"]\n ):\n port = data[\"entryPoints\"][\"apiserver\"][\"address\"]\n port = port.replace(\":\", \"\")\n return port\n else:\n return \"16443\"\n\n\ndef join_request(conn, api_version, req_data, master_ip, verify_peer, fingerprint):\n json_params = json.dumps(req_data)\n headers = {\"Content-type\": \"application/json\", \"Accept\": \"application/json\"}\n\n try:\n if verify_peer and fingerprint:\n if len(fingerprint) < FINGERPRINT_MIN_LEN:\n print(\n \"Joining cluster failed. Fingerprint too short.\"\n \" Use '--skip-verify' to skip server certificate check.\"\n )\n exit(4)\n\n # Do the peer certificate verification\n der_cert_bin = conn.sock.getpeercert(True)\n peer_cert_hash = hashlib.sha256(der_cert_bin).hexdigest()\n if not peer_cert_hash.startswith(fingerprint):\n print(\n \"Joining cluster failed. Could not verify the identity of {}.\"\n \" Use '--skip-verify' to skip server certificate check.\".format(master_ip)\n )\n exit(4)\n\n conn.request(\"POST\", \"/{}/join\".format(api_version), json_params, headers)\n response = conn.getresponse()\n if not response.status == 200:\n message = extract_error(response)\n print(\"{} ({}).\".format(message, response.status))\n exit(6)\n body = response.read()\n return json.loads(body)\n except http.client.HTTPException as e:\n print(\"Please ensure the master node is reachable. {}\".format(e))\n exit(1)\n except ssl.SSLError as e:\n print(\"Peer node verification failed ({}).\".format(e))\n exit(4)\n\n\ndef extract_error(response):\n message = \"Connection failed.\"\n try:\n resp = response.read().decode()\n if resp:\n res_data = json.loads(resp)\n if \"error\" in res_data:\n message = \"{} {}\".format(message, res_data[\"error\"])\n except ValueError:\n pass\n return message\n\n\ndef get_connection_info(\n master_ip,\n master_port,\n token,\n callback_token=None,\n cluster_type=\"etcd\",\n verify_peer=False,\n fingerprint=None,\n worker=False,\n):\n \"\"\"\n Contact the master and get all connection information\n\n :param master_ip: the master IP\n :param master_port: the master port\n :param token: the token to contact the master with\n :param callback_token: callback token for etcd based clusters\n :param cluster_type: the type of cluster we want to join, etcd or dqlite\n :param verify_peer: flag indicating if we should verify peers certificate\n :param fingerprint: the certificate fingerprint we expect from the peer\n :param worker: this is a worker only node\n\n :return: the json response of the master\n \"\"\"\n cluster_agent_port = get_cluster_agent_port()\n try:\n context = ssl._create_unverified_context()\n conn = http.client.HTTPSConnection(\"{}:{}\".format(master_ip, master_port), context=context)\n conn.connect()\n if cluster_type == \"dqlite\":\n req_data = {\n \"token\": token,\n \"hostname\": socket.gethostname().lower(),\n \"port\": cluster_agent_port,\n \"worker\": worker,\n \"can_handle_x509_auth\": True,\n \"can_handle_custom_etcd\": True,\n }\n\n return join_request(conn, CLUSTER_API_V2, req_data, master_ip, verify_peer, fingerprint)\n else:\n req_data = {\n \"token\": token,\n \"hostname\": socket.gethostname().lower(),\n \"port\": cluster_agent_port,\n \"callback\": callback_token,\n \"can_handle_x509_auth\": True,\n }\n return join_request(\n conn, CLUSTER_API, req_data, master_ip, verify_peer=False, fingerprint=None\n )\n except http.client.HTTPException as e:\n print(\"Connecting to cluster failed with {}.\".format(e))\n exit(5)\n except ssl.SSLError as e:\n print(\"Peer node verification failed with {}.\".format(e))\n exit(4)\n\n\ndef get_etcd_client_cert(master_ip, master_port, token):\n \"\"\"\n Get a signed cert to access etcd\n\n :param master_ip: master ip\n :param master_port: master port\n :param token: token to contact the master with\n \"\"\"\n cer_req_file = \"{}/certs/server.remote.csr\".format(snapdata_path)\n cmd_cert = (\n \"{snap}/openssl.wrapper req -new -sha256 -key {snapdata}/certs/server.key -out {csr} \"\n \"-config {snapdata}/certs/csr.conf\".format(\n snap=snap_path, snapdata=snapdata_path, csr=cer_req_file\n )\n )\n subprocess.check_call(cmd_cert.split())\n with open(cer_req_file) as fp:\n csr = fp.read()\n req_data = {\"token\": token, \"request\": csr}\n # TODO: enable ssl verification\n signed = requests.post(\n \"https://{}:{}/{}/sign-cert\".format(master_ip, master_port, CLUSTER_API),\n json=req_data,\n verify=False,\n )\n if signed.status_code != 200:\n print(\"Failed to sign certificate. {}\".format(signed.json()[\"error\"]))\n exit(1)\n info = signed.json()\n with open(server_cert_file, \"w\") as cert_fp:\n cert_fp.write(info[\"certificate\"])\n try_set_file_permissions(server_cert_file)\n\n\ndef get_client_cert(master_ip, master_port, fname: str, token: str, subject: str, with_sans: bool):\n \"\"\"\n Get a signed cert signed by a remote cluster-agent.\n See https://kubernetes.io/docs/reference/access-authn-authz/authentication/#x509-client-certs\n\n :param master_ip: master ip\n :param master_port: master port\n :param fname: file name prefix for the certificate\n :param token: token to contact the master with\n :param subject: the subject of the certificate\n :param with_sans: whether to include hostname and node IPs as subject alternate names\n \"\"\"\n\n cert_crt = (snap_data() / \"certs\" / fname).with_suffix(\".crt\")\n cert_key = (snap_data() / \"certs\" / fname).with_suffix(\".key\")\n # generate csr\n script = \"generate_csr_with_sans\" if with_sans else \"generate_csr\"\n p = subprocess.run(\n [f\"{snap()}/actions/common/utils.sh\", script, subject, cert_key],\n check=True,\n capture_output=True,\n )\n csr = p.stdout.decode()\n\n req_data = {\"token\": token, \"request\": csr}\n # TODO: enable ssl verification\n signed = requests.post(\n \"https://{}:{}/{}/sign-cert\".format(master_ip, master_port, CLUSTER_API),\n json=req_data,\n verify=False,\n )\n if signed.status_code != 200:\n error = \"Failed to sign {} certificate ({}).\".format(fname, signed.status_code)\n try:\n if \"error\" in signed.json():\n error = \"{} {}\".format(error, format(signed.json()[\"error\"]))\n except ValueError:\n print(\"Make sure the cluster you connect to supports joining worker nodes.\")\n print(error)\n exit(1)\n info = signed.json()\n cert_crt.write_text(info[\"certificate\"])\n try_set_file_permissions(cert_crt)\n\n\ndef update_flannel(etcd, master_ip, master_port, token):\n \"\"\"\n Configure flannel\n\n :param etcd: etcd endpoint\n :param master_ip: master ip\n :param master_port: master port\n :param token: token to contact the master with\n \"\"\"\n get_etcd_client_cert(master_ip, master_port, token)\n etcd = etcd.replace(\"0.0.0.0\", master_ip)\n set_arg(\"--etcd-endpoints\", etcd, \"flanneld\")\n set_arg(\"--etcd-cafile\", ca_cert_file_via_env, \"flanneld\")\n set_arg(\"--etcd-certfile\", server_cert_file_via_env, \"flanneld\")\n set_arg(\"--etcd-keyfile\", \"${SNAP_DATA}/certs/server.key\", \"flanneld\")\n service(\"restart\", \"flanneld\")\n\n\ndef create_kubeconfig(token, ca, master_ip, api_port, filename, user):\n \"\"\"\n Create a kubeconfig file. The file in stored under credentials named after the user\n\n :param token: the token to be in the kubeconfig\n :param ca: the ca\n :param master_ip: the master node IP\n :param api_port: the API server port\n :param filename: the name of the config file\n :param user: the user to use al login\n \"\"\"\n snap_path = os.environ.get(\"SNAP\")\n config_template = \"{}/{}\".format(snap_path, \"kubelet.config.template\")\n config = \"{}/credentials/{}\".format(snapdata_path, filename)\n shutil.copyfile(config, \"{}.backup\".format(config))\n try_set_file_permissions(\"{}.backup\".format(config))\n ca_line = ca_one_line(ca)\n with open(config_template, \"r\") as tfp:\n with open(config, \"w+\") as fp:\n config_txt = tfp.read()\n config_txt = config_txt.replace(\"CADATA\", ca_line)\n config_txt = config_txt.replace(\"NAME\", user)\n config_txt = config_txt.replace(\"TOKEN\", token)\n config_txt = config_txt.replace(\"127.0.0.1\", master_ip)\n config_txt = config_txt.replace(\"16443\", api_port)\n fp.write(config_txt)\n try_set_file_permissions(config)\n\n\ndef update_kubeproxy(token, ca, master_ip, api_port):\n \"\"\"\n Configure the kube-proxy\n\n :param token: the token to be in the kubeconfig\n :param ca: the ca\n :param master_ip: the master node IP\n :param api_port: the API server port\n \"\"\"\n create_kubeconfig(token, ca, master_ip, api_port, \"proxy.config\", \"kubeproxy\")\n set_arg(\"--master\", None, \"kube-proxy\")\n set_arg(\"--hostname-override\", None, \"kube-proxy\")\n service(\"restart\", \"proxy\")\n\n\ndef update_cert_auth_kubeproxy(token, master_ip, master_port):\n \"\"\"\n Configure the kube-proxy\n\n :param token: the token to be in the kubeconfig\n :param ca: the ca\n :param master_ip: the master node IP\n :param master_port: the master node port where the cluster agent listens\n \"\"\"\n proxy_token = \"{}-proxy\".format(token)\n get_client_cert(master_ip, master_port, \"proxy\", proxy_token, \"/CN=system:kube-proxy\", False)\n set_arg(\"--master\", None, \"kube-proxy\")\n set_arg(\"--hostname-override\", None, \"kube-proxy\")\n\n\ndef update_kubeproxy_cidr(cidr):\n if cidr is not None:\n set_arg(\"--cluster-cidr\", cidr, \"kube-proxy\")\n service(\"restart\", \"proxy\")\n\n\ndef update_cert_auth_kubelet(token, master_ip, master_port):\n \"\"\"\n Configure the kubelet\n\n :param token: the token to be in the kubeconfig\n :param ca: the ca\n :param master_ip: the master node IP\n :param master_port: the master node port where the cluster agent listens\n \"\"\"\n kubelet_token = \"{}-kubelet\".format(token)\n subject = f\"/CN=system:node:{socket.gethostname().lower()}/O=system:nodes\"\n get_client_cert(master_ip, master_port, \"kubelet\", kubelet_token, subject, True)\n set_arg(\"--client-ca-file\", \"${SNAP_DATA}/certs/ca.remote.crt\", \"kubelet\")\n set_arg(\n \"--node-labels\",\n \"microk8s.io/cluster=true,node.kubernetes.io/microk8s-worker=microk8s-worker\",\n \"kubelet\",\n )\n\n\ndef update_kubelet(token, ca, master_ip, api_port):\n \"\"\"\n Configure the kubelet\n\n :param token: the token to be in the kubeconfig\n :param ca: the ca\n :param master_ip: the master node IP\n :param api_port: the API server port\n \"\"\"\n create_kubeconfig(token, ca, master_ip, api_port, \"kubelet.config\", \"kubelet\")\n set_arg(\"--client-ca-file\", \"${SNAP_DATA}/certs/ca.remote.crt\", \"kubelet\")\n set_arg(\n \"--node-labels\",\n \"microk8s.io/cluster=true,node.kubernetes.io/microk8s-worker=microk8s-worker\",\n \"kubelet\",\n )\n service(\"restart\", \"kubelet\")\n\n\ndef update_apiserver(api_authz_mode, apiserver_port):\n \"\"\"\n Configure the API server\n\n :param api_authz_mode: the authorization mode to be used\n :param apiserver_port: the apiserver port\n \"\"\"\n if api_authz_mode:\n set_arg(\"--authorization-mode\", api_authz_mode, \"kube-apiserver\")\n if apiserver_port:\n set_arg(\"--secure-port\", apiserver_port, \"kube-apiserver\")\n service(\"restart\", \"apiserver\")\n\n\ndef store_remote_ca(ca):\n \"\"\"\n Store the remote ca\n\n :param ca: the CA\n \"\"\"\n with open(ca_cert_file, \"w+\") as fp:\n fp.write(ca)\n try_set_file_permissions(ca_cert_file)\n\n\ndef mark_worker_node():\n \"\"\"\n Mark a node as being part of a cluster not running the control plane\n by creating a var/lock/clustered.lock\n \"\"\"\n locks = [\"clustered.lock\", \"no-k8s-dqlite\"]\n for lock in locks:\n lock_file = \"{}/var/lock/{}\".format(snapdata_path, lock)\n open(lock_file, \"a\").close()\n os.chmod(lock_file, 0o700)\n services = [\"kubelite\", \"etcd\", \"apiserver-kicker\", \"apiserver-proxy\", \"k8s-dqlite\"]\n for s in services:\n service(\"restart\", s)\n\n\ndef generate_callback_token():\n \"\"\"\n Generate a token and store it in the callback token file\n\n :return: the token\n \"\"\"\n token = \"\".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(64))\n with open(callback_token_file, \"w\") as fp:\n fp.write(\"{}\\n\".format(token))\n\n try_set_file_permissions(callback_token_file)\n return token\n\n\ndef store_base_kubelet_args(args_string):\n \"\"\"\n Create a kubelet args file from the set of args provided\n\n :param args_string: the arguments provided\n \"\"\"\n args_file = \"{}/args/kubelet\".format(snapdata_path)\n with open(args_file, \"w\") as fp:\n fp.write(args_string)\n try_set_file_permissions(args_file)\n\n\ndef update_kubelet_node_ip(args_string, hostname_override):\n \"\"\"\n Update the kubelet --node-ip argument if it was set on the node that we join.\n\n :param args_string: the kubelet arguments\n :param hostname_override: the source IP address used by the node when joining\n \"\"\"\n if \"--node-ip\" in args_string:\n set_arg(\"--node-ip\", hostname_override, \"kubelet\")\n\n\ndef update_kubelet_hostname_override(args_string):\n \"\"\"\n Remove the kubelet --hostname-override argument if it was set on the node that we join.\n\n :param args_string: the kubelet arguments\n \"\"\"\n if \"--hostname-override\" in args_string:\n set_arg(\"--hostname-override\", None, \"kubelet\")\n\n\ndef replace_admin_token(token):\n \"\"\"\n Replaces the admin token in the known tokens\n\n :param token: the admin token\n \"\"\"\n file = \"{}/credentials/known_tokens.csv\".format(snapdata_path)\n backup_file = \"{}.backup\".format(file)\n # That is a critical section. We need to protect it.\n with open(backup_file, \"w\") as back_fp:\n with open(file, \"r\") as fp:\n for _, line in enumerate(fp):\n if 'admin,admin,\"system:masters\"' in line:\n continue\n back_fp.write(\"{}\".format(line))\n back_fp.write('{},admin,admin,\"system:masters\"\\n'.format(token))\n\n try_set_file_permissions(backup_file)\n shutil.copyfile(backup_file, file)\n\n\ndef store_cert(filename, payload):\n \"\"\"\n Store a certificate\n\n :param filename: where to store the certificate\n :param payload: certificate payload\n \"\"\"\n file_with_path = \"{}/certs/{}\".format(snapdata_path, filename)\n if os.path.exists(file_with_path):\n backup_file_with_path = \"{}.backup\".format(file_with_path)\n shutil.copyfile(file_with_path, backup_file_with_path)\n try_set_file_permissions(backup_file_with_path)\n with open(file_with_path, \"w+\") as fp:\n fp.write(payload)\n try_set_file_permissions(file_with_path)\n\n\ndef store_cluster_certs(cluster_cert, cluster_key):\n \"\"\"\n Store the dqlite cluster certs\n\n :param cluster_cert: the cluster certificate\n :param cluster_key: the cluster certificate key\n \"\"\"\n with open(cluster_cert_file, \"w+\") as fp:\n fp.write(cluster_cert)\n try_set_file_permissions(cluster_cert_file)\n with open(cluster_key_file, \"w+\") as fp:\n fp.write(cluster_key)\n try_set_file_permissions(cluster_key_file)\n\n\ndef create_admin_kubeconfig(ca, ha_admin_token=None):\n \"\"\"\n Create a kubeconfig file. The file in stored under credentials named after the admin\n\n :param ca: the ca\n :param ha_admin_token: the ha_cluster_token\n \"\"\"\n if not ha_admin_token:\n token = get_token(\"admin\", \"basic_auth.csv\")\n if not token:\n print(\"Error, could not locate admin token. Joining cluster failed.\")\n exit(2)\n else:\n token = ha_admin_token\n assert token is not None\n config_template = \"{}/{}\".format(snap_path, \"client.config.template\")\n config = \"{}/credentials/client.config\".format(snapdata_path)\n shutil.copyfile(config, \"{}.backup\".format(config))\n try_set_file_permissions(\"{}.backup\".format(config))\n ca_line = ca_one_line(ca)\n with open(config_template, \"r\") as tfp:\n with open(config, \"w+\") as fp:\n for _, config_txt in enumerate(tfp):\n if config_txt.strip().startswith(\"username:\"):\n continue\n else:\n config_txt = config_txt.replace(\"CADATA\", ca_line)\n config_txt = config_txt.replace(\"NAME\", \"admin\")\n config_txt = config_txt.replace(\"AUTHTYPE\", \"token\")\n config_txt = config_txt.replace(\"PASSWORD\", token)\n fp.write(config_txt)\n try_set_file_permissions(config)\n\n\ndef store_callback_token(token):\n \"\"\"\n Store the callback token\n\n :param token: the callback token\n \"\"\"\n callback_token_file = \"{}/credentials/callback-token.txt\".format(snapdata_path)\n with open(callback_token_file, \"w\") as fp:\n fp.write(token)\n try_set_file_permissions(callback_token_file)\n\n\ndef update_dqlite(cluster_cert, cluster_key, voters, host):\n \"\"\"\n Configure the dqlite cluster\n\n :param cluster_cert: the dqlite cluster cert\n :param cluster_key: the dqlite cluster key\n :param voters: the dqlite voters\n :param host: the hostname others see of this node\n \"\"\"\n service(\"stop\", \"apiserver\")\n service(\"stop\", \"k8s-dqlite\")\n # will allow for apiservice-kicker to generate new certificates @5s loop rate\n time.sleep(10)\n shutil.rmtree(cluster_backup_dir, ignore_errors=True)\n shutil.move(cluster_dir, cluster_backup_dir)\n os.mkdir(cluster_dir)\n store_cluster_certs(cluster_cert, cluster_key)\n\n # We get the dqlite port from the already existing deployment\n port = 19001\n try:\n with open(\"{}/info.yaml\".format(cluster_backup_dir)) as f:\n data = yaml.safe_load(f)\n if \"Address\" in data:\n port = data[\"Address\"].rsplit(\":\")[-1]\n except OSError:\n pass\n\n # If host is an IPv6 address, wrap it in square brackets\n try:\n if ipaddress.ip_address(host).version == 6:\n host = \"[{}]\".format(host)\n except ValueError:\n pass\n\n init_data = {\"Cluster\": voters, \"Address\": \"{}:{}\".format(host, port)}\n with open(\"{}/init.yaml\".format(cluster_dir), \"w\") as f:\n yaml.dump(init_data, f)\n\n service(\"start\", \"k8s-dqlite\")\n\n waits = 10\n print(\"Waiting for this node to finish joining the cluster.\", end=\" \", flush=True)\n while waits > 0:\n try:\n out = subprocess.check_output(\n \"{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt \"\n \"-k {dbdir}/cluster.key -f json k8s .cluster\".format(\n snappath=snap_path, dbdir=cluster_dir\n ).split(),\n timeout=4,\n stderr=subprocess.STDOUT,\n )\n if host in out.decode():\n break\n else:\n print(\".\", end=\" \", flush=True)\n time.sleep(5)\n waits -= 1\n\n except (subprocess.CalledProcessError, subprocess.TimeoutExpired):\n print(\"..\", end=\" \", flush=True)\n time.sleep(2)\n waits -= 1\n print(\" \")\n\n # start kube-apiserver after dqlite comes up\n service(\"start\", \"apiserver\")\n\n\ndef join_dqlite(connection_parts, verify=False, worker=False):\n \"\"\"\n Configure node to join a dqlite cluster.\n\n :param connection_parts: connection string parts\n \"\"\"\n token = connection_parts[1]\n master_ep = connection_parts[0].rsplit(\":\", 1)\n master_ip = master_ep[0]\n master_port = master_ep[1]\n fingerprint = None\n if len(connection_parts) > 2:\n fingerprint = connection_parts[2]\n else:\n # we do not have a fingerprint, do not attempt to verify the remote cert\n verify = False\n\n print(\"Contacting cluster at {}\".format(master_ip))\n\n info = get_connection_info(\n master_ip,\n master_port,\n token,\n cluster_type=\"dqlite\",\n verify_peer=verify,\n fingerprint=fingerprint,\n worker=worker,\n )\n\n # Get the cluster_cidr from kube-proxy args\n cluster_cidr = get_cluster_cidr()\n\n if \"cluster_cidr\" in info and info[\"cluster_cidr\"] != cluster_cidr:\n print(\n \"WARNING: Joining a cluster that has a different CIDR. \"\n \"The kube-proxy CIDR configuration will be overwritten.\"\n )\n print(\n f\"Cluster CIDR: {info['cluster_cidr']} -- Node CIDR: {cluster_cidr}(will be overwritten)\"\n )\n update_kubeproxy_cidr(info[\"cluster_cidr\"])\n\n if worker:\n join_dqlite_worker_node(info, master_ip, master_port, token)\n else:\n join_dqlite_master_node(info, master_ip)\n\n\ndef update_apiserver_proxy(master_ip, api_port):\n \"\"\"\n Update the apiserver-proxy configuration\n \"\"\"\n lock_path = os.path.expandvars(\"${SNAP_DATA}/var/lock\")\n lock = \"{}/no-apiserver-proxy\".format(lock_path)\n if os.path.exists(lock):\n os.remove(lock)\n\n # add the initial control plane endpoint\n addresses = [{\"address\": \"{}:{}\".format(master_ip, api_port)}]\n\n traefik_providers = os.path.expandvars(\"${SNAP_DATA}/args/traefik/provider-template.yaml\")\n traefik_providers_out = os.path.expandvars(\"${SNAP_DATA}/args/traefik/provider.yaml\")\n with open(traefik_providers) as f:\n p = yaml.safe_load(f)\n p[\"tcp\"][\"services\"][\"kube-apiserver\"][\"loadBalancer\"][\"servers\"] = addresses\n with open(traefik_providers_out, \"w\") as out_file:\n yaml.dump(p, out_file)\n\n try_set_file_permissions(traefik_providers_out)\n service(\"restart\", \"apiserver-proxy\")\n\n\ndef rebuild_token_based_auth_configs(info):\n # We need to make sure token-auth is enabled in this node too.\n if not is_token_auth_enabled():\n enable_token_auth(info[\"admin_token\"])\n else:\n replace_admin_token(info[\"admin_token\"])\n\n subprocess.check_call(\n [f\"{snap_path}/actions/common/utils.sh\", \"create_user_certs_and_configs\"],\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n\n\ndef print_worker_usage():\n \"\"\"\n Print Worker usage\n \"\"\"\n print(\"\")\n print(\"The node has joined the cluster and will appear in the nodes list in a few seconds.\")\n print(\"\")\n print(\"This worker node gets automatically configured with the API server endpoints.\")\n print(\n \"If the API servers are behind a loadbalancer please set the '--refresh-interval' to '0s' in:\"\n )\n print(\" /var/snap/microk8s/current/args/apiserver-proxy\")\n print(\"and replace the API server endpoints with the one provided by the loadbalancer in:\")\n print(\" /var/snap/microk8s/current/args/traefik/provider.yaml\")\n print(\"\")\n\n\ndef join_dqlite_worker_node(info, master_ip, master_port, token):\n \"\"\"\n Join this node as a worker to a cluster running dqlite.\n\n :param info: dictionary with the connection information\n :param master_ip: the IP of the master node we contacted to connect to the cluster\n :param master_port: the port of the mester node we contacted to connect to the cluster\n :param token: the token to pass to the master in order to authenticate with it\n \"\"\"\n hostname_override = info[\"hostname_override\"]\n if info[\"ca_key\"] is not None:\n print(\n \"Joining process failed. Make sure the cluster you connect to supports joining worker nodes.\"\n )\n exit(1)\n\n store_remote_ca(info[\"ca\"])\n\n update_apiserver(info.get(\"api_authz_mode\"), info.get(\"apiport\"))\n store_base_kubelet_args(info[\"kubelet_args\"])\n update_kubelet_node_ip(info[\"kubelet_args\"], hostname_override)\n update_kubelet_hostname_override(info[\"kubelet_args\"])\n update_cert_auth_kubeproxy(token, master_ip, master_port)\n update_cert_auth_kubelet(token, master_ip, master_port)\n subprocess.check_call(\n [f\"{snap()}/actions/common/utils.sh\", \"create_worker_kubeconfigs\"],\n stderr=subprocess.DEVNULL,\n stdout=subprocess.DEVNULL,\n )\n\n store_callback_token(info[\"callback_token\"])\n update_apiserver_proxy(master_ip, info[\"apiport\"])\n mark_worker_node()\n mark_no_cert_reissue()\n print_worker_usage()\n\n\ndef join_dqlite_master_node(info, master_ip):\n \"\"\"\n Join this node to a cluster running dqlite.\n\n :param info: dictionary with the connection information\n :param master_ip: the IP of the master node we contacted to connect to the cluster\n \"\"\"\n\n # The cluster we want to join may be either token-auth based or x509-auth based.\n # The way to identify the cluster type is to look for the \"admin_token\" in the info\n # we got back from the cluster we try to join.\n # In the case of token-auth we need to:\n # - create the known_tokens.csv file (if it does not exist) with the admin token\n # - turn on token-auth on kube-apiserver\n # - create the token based admin kubeconfig\n # - recreate the kubelet, proxy, scheduler, controller kubeconfigs for the new ca\n # - restart kubelite\n # In the case of x509-auth we need to:\n # - recreate the admin/client, kubelet, proxy, scheduler, controller kubeconfigs for the new ca\n # - restart kubelite\n\n hostname_override = info[\"hostname_override\"]\n store_cert(\"ca.crt\", info[\"ca\"])\n store_cert(\"ca.key\", info[\"ca_key\"])\n store_cert(\"serviceaccount.key\", info[\"service_account_key\"])\n\n if \"admin_token\" in info:\n # We try to join a cluster where token-auth is in place.\n rebuild_token_based_auth_configs(info)\n else:\n # We are joining a x509-auth based cluster\n rebuild_x509_auth_client_configs()\n\n update_apiserver(info.get(\"api_authz_mode\"), info.get(\"apiport\"))\n store_base_kubelet_args(info[\"kubelet_args\"])\n update_kubelet_node_ip(info[\"kubelet_args\"], hostname_override)\n update_kubelet_hostname_override(info[\"kubelet_args\"])\n store_callback_token(info[\"callback_token\"])\n\n if \"etcd_servers\" in info:\n set_arg(\"--etcd-servers\", info[\"etcd_servers\"], \"kube-apiserver\")\n if info.get(\"etcd_ca\"):\n store_cert(\"remote-etcd-ca.crt\", info[\"etcd_ca\"])\n set_arg(\"--etcd-cafile\", \"${SNAP_DATA}/certs/remote-etcd-ca.crt\", \"kube-apiserver\")\n if info.get(\"etcd_cert\"):\n store_cert(\"remote-etcd.crt\", info[\"etcd_cert\"])\n set_arg(\"--etcd-certfile\", \"${SNAP_DATA}/certs/remote-etcd.crt\", \"kube-apiserver\")\n if info.get(\"etcd_key\"):\n store_cert(\"remote-etcd.key\", info[\"etcd_key\"])\n set_arg(\"--etcd-keyfile\", \"${SNAP_DATA}/certs/remote-etcd.key\", \"kube-apiserver\")\n\n mark_no_dqlite()\n service(\"restart\", \"k8s-dqlite\")\n service(\"restart\", \"apiserver\")\n else:\n update_dqlite(info[\"cluster_cert\"], info[\"cluster_key\"], info[\"voters\"], hostname_override)\n\n # We want to update the local CNI yaml but we do not want to apply it.\n # The cni is applied already in the cluster we join\n try_initialise_cni_autodetect_for_clustering(master_ip, apply_cni=False)\n mark_no_cert_reissue()\n\n\ndef join_etcd(connection_parts, verify=True):\n \"\"\"\n Configure node to join an etcd cluster.\n\n :param connection_parts: connection string parts\n \"\"\"\n token = connection_parts[1]\n master_ep = connection_parts[0].rsplit(\":\", 1)\n master_ip = master_ep[0]\n master_port = master_ep[1]\n callback_token = generate_callback_token()\n info = get_connection_info(master_ip, master_port, token, callback_token=callback_token)\n\n # check api authn mode from response. default is fallback to Token\n api_authn_mode = info.get(\"api_authn_mode\", \"Token\")\n if api_authn_mode not in [\"Cert\", \"Token\"]:\n print(\"Error: Unknown API auth mode '{api_authn_mode}' received from control plane node.\")\n print(\"Please update this MicroK8s node to the latest version before joining.\")\n exit(7)\n\n # Get the cluster_cidr from kube-proxy args\n cluster_cidr = get_cluster_cidr()\n\n if \"cluster_cidr\" in info and info[\"cluster_cidr\"] != cluster_cidr:\n print(\n \"WARNING: Joining a cluster that has a different CIDR. \"\n \"The kube-proxy CIDR configuration will be overwritten.\"\n )\n print(\n f\"Cluster CIDR: {info['cluster_cidr']} -- Node CIDR: {cluster_cidr}(will be overwritten)\"\n )\n update_kubeproxy_cidr(info[\"cluster_cidr\"])\n\n store_base_kubelet_args(info[\"kubelet_args\"])\n update_kubelet_hostname_override(info[\"kubelet_args\"])\n hostname_override = None\n if \"hostname_override\" in info:\n hostname_override = info[\"hostname_override\"]\n update_kubelet_node_ip(info[\"kubelet_args\"], hostname_override)\n\n store_remote_ca(info[\"ca\"])\n update_flannel(info[\"etcd\"], master_ip, master_port, token)\n\n if api_authn_mode == \"Token\":\n update_kubeproxy(info[\"kubeproxy\"], info[\"ca\"], master_ip, info[\"apiport\"])\n update_kubelet(info[\"kubelet\"], info[\"ca\"], master_ip, info[\"apiport\"])\n elif api_authn_mode == \"Cert\":\n update_cert_auth_kubeproxy(info[\"kubeproxy\"], master_ip, master_port)\n update_cert_auth_kubelet(info[\"kubelet\"], master_ip, master_port)\n subprocess.check_call(\n [\n f\"{snap()}/actions/common/utils.sh\",\n \"create_worker_kubeconfigs\",\n master_ip,\n info[\"apiport\"],\n ],\n stderr=subprocess.DEVNULL,\n stdout=subprocess.DEVNULL,\n )\n else:\n assert False, \"this should never happen\"\n\n mark_worker_node()\n mark_no_cert_reissue()\n\n\ndef mark_join_in_progress():\n \"\"\"\n Mark node as currently being in the process of joining a cluster.\n \"\"\"\n lock_file = \"{}/var/lock/join-in-progress\".format(snapdata_path)\n open(lock_file, \"a\").close()\n os.chmod(lock_file, 0o700)\n\n\ndef unmark_join_in_progress():\n \"\"\"\n Unmark as joining cluster; join operation has finished.\n \"\"\"\n lock_file = \"{}/var/lock/join-in-progress\".format(snapdata_path)\n if os.path.exists(lock_file):\n os.unlink(lock_file)\n\n\ndef mark_no_dqlite():\n \"\"\"\n Mark node to not run k8s-dqlite service.\n \"\"\"\n lock_file = \"{}/var/lock/no-k8s-dqlite\".format(snapdata_path)\n open(lock_file, \"a\").close()\n os.chmod(lock_file, 0o700)\n\n\n@click.command(\n context_settings={\"ignore_unknown_options\": True, \"help_option_names\": [\"-h\", \"--help\"]}\n)\n@click.argument(\"connection\", required=True)\n@click.option(\n \"--worker\", \"worker\", default=False, flag_value=\"as-worker\", help=\"Join as a worker only node.\"\n)\n@click.option(\n \"--controlplane\",\n \"worker\",\n flag_value=\"as-master\",\n help=\"Join running the control plane on HA clusters. (default)\",\n)\n@click.option(\n \"--skip-verify\",\n is_flag=True,\n required=False,\n default=False,\n help=\"Skip the certificate verification of the node we are joining to. (default: false)\",\n)\n@click.option(\n \"--disable-low-memory-guard\",\n is_flag=True,\n required=False,\n default=False,\n help=\"Disable the low memory guard. (default: false)\",\n)\ndef join(connection, worker, skip_verify, disable_low_memory_guard):\n \"\"\"\n Join the node to a cluster\n\n CONNECTION: the cluster connection endpoint in format :/\n \"\"\"\n try:\n connection_parts = get_valid_connection_parts(connection)\n except InvalidConnectionError as err:\n print(\"Invalid connection:\", err)\n sys.exit(1)\n\n verify = not skip_verify\n\n if is_low_memory_guard_enabled() and disable_low_memory_guard:\n os.remove(os.path.expandvars(\"$SNAP_DATA/var/lock/low-memory-guard.lock\"))\n\n if is_low_memory_guard_enabled() and not worker:\n print(\n \"\"\"\nThis node does not have enough RAM to host the Kubernetes control plane services\nand join the database quorum. You may consider joining this node as a worker instead:\n\n microk8s join {connection} --worker\n\nIf you would still like to join the cluster as a control plane node, use:\n\n microk8s join {connection} --disable-low-memory-guard\n\n\"\"\".format(\n connection=connection\n )\n )\n sys.exit(1)\n\n mark_join_in_progress()\n\n if is_node_running_dqlite():\n join_dqlite(connection_parts, verify, worker)\n else:\n join_etcd(connection_parts, verify)\n\n unmark_join_in_progress()\n print(\"Successfully joined the cluster.\")\n sys.exit(0)\n\n\nif __name__ == \"__main__\":\n join(prog_name=\"microk8s join\")\n" }, { "path": "scripts/wrappers/leave.py", "content": "#!/usr/bin/python3\nimport json\nimport os\nimport shutil\nimport socket\nimport subprocess\nimport sys\nimport time\n\nimport click\nimport netifaces\nimport yaml\n\nfrom common.cluster.utils import (\n is_node_running_dqlite,\n service,\n unmark_no_cert_reissue,\n restart_all_services,\n is_node_dqlite_worker,\n rebuild_x509_auth_client_configs,\n)\n\nsnapdata_path = os.environ.get(\"SNAP_DATA\")\nsnap_path = os.environ.get(\"SNAP\")\nca_cert_file = \"{}/certs/ca.remote.crt\".format(snapdata_path)\ncallback_token_file = \"{}/credentials/callback-token.txt\".format(snapdata_path)\nserver_cert_file = \"{}/certs/server.remote.crt\".format(snapdata_path)\n\ncluster_dir = \"{}/var/kubernetes/backend\".format(snapdata_path)\ncluster_backup_dir = \"{}/var/kubernetes/backend.backup\".format(snapdata_path)\n\n\ndef reset_current_dqlite_worker_installation():\n \"\"\"\n Take a node out of a cluster\n \"\"\"\n print(\"Configuring services.\", flush=True)\n disable_apiserver_proxy()\n os.remove(ca_cert_file)\n\n service(\"stop\", \"apiserver\")\n service(\"stop\", \"k8s-dqlite\")\n rebuild_x509_auth_client_configs()\n\n print(\"Generating new cluster certificates.\", flush=True)\n reinit_cluster()\n\n for config_file in [\"kubelet\", \"kube-proxy\"]:\n shutil.copyfile(\n \"{}/default-args/{}\".format(snap_path, config_file),\n \"{}/args/{}\".format(snapdata_path, config_file),\n )\n\n unmark_no_cert_reissue()\n unmark_worker_node()\n restart_all_services()\n apply_cni()\n\n\ndef disable_apiserver_proxy():\n \"\"\"\n Stop apiserver-proxy\n \"\"\"\n lock_path = os.path.expandvars(\"${SNAP_DATA}/var/lock\")\n lock = \"{}/no-apiserver-proxy\".format(lock_path)\n if not os.path.exists(lock):\n open(lock, \"a\").close()\n service(\"stop\", \"apiserver-proxy\")\n\n\ndef unmark_worker_node():\n \"\"\"\n Unmark a node as being part of a cluster not running the control plane\n by deleting a var/lock/clustered.lock\n \"\"\"\n locks = [\"clustered.lock\", \"no-k8s-dqlite\"]\n for lock in locks:\n lock_file = \"{}/var/lock/{}\".format(snapdata_path, lock)\n if not os.path.isfile(lock_file):\n print(\"Not in clustering mode.\")\n exit(2)\n os.remove(lock_file)\n\n\ndef reset_current_etcd_installation():\n \"\"\"\n Take a node out of a cluster\n \"\"\"\n lock_file = \"{}/var/lock/clustered.lock\".format(snapdata_path)\n if not os.path.isfile(lock_file):\n print(\"Not in clustering mode.\")\n exit(2)\n\n os.remove(lock_file)\n os.remove(ca_cert_file)\n os.remove(callback_token_file)\n os.remove(server_cert_file)\n\n for config_file in [\"kubelet\", \"flanneld\", \"kube-proxy\"]:\n shutil.copyfile(\n \"{}/default-args/{}\".format(snap_path, config_file),\n \"{}/args/{}\".format(snapdata_path, config_file),\n )\n\n for user in [\"proxy\", \"kubelet\"]:\n config = \"{}/credentials/{}.config\".format(snapdata_path, user)\n shutil.copyfile(\"{}.backup\".format(config), config)\n\n subprocess.check_call(\"{}/microk8s-stop.wrapper\".format(snap_path).split())\n waits = 10\n while waits > 0:\n try:\n subprocess.check_call(\"{}/microk8s-start.wrapper\".format(snap_path).split())\n break\n except subprocess.CalledProcessError:\n print(\"Services not ready to start. Waiting...\")\n time.sleep(5)\n waits -= 1\n\n unmark_no_cert_reissue()\n\n\ndef reset_current_dqlite_installation():\n \"\"\"\n Take a node out of a dqlite cluster\n \"\"\"\n if is_leader_without_successor():\n print(\n \"This node currently holds the only copy of the Kubernetes \"\n \"database so it cannot leave the cluster.\"\n )\n print(\n \"To remove this node you can either first remove all other \"\n \"nodes with 'microk8s remove-node' or\"\n )\n print(\"form a highly available cluster by adding at least three nodes.\")\n exit(3)\n\n # We need to:\n # 1. Stop the apiserver\n # 2. Send a DELETE request to any member of the dqlite cluster\n # 3. wipe out the existing installation\n my_ep, other_ep = get_dqlite_endpoints()\n\n service(\"stop\", \"apiserver\")\n service(\"stop\", \"k8s-dqlite\")\n time.sleep(10)\n\n delete_dqlite_node(my_ep, other_ep)\n\n print(\"Generating new cluster certificates.\", flush=True)\n reinit_cluster()\n rebuild_x509_auth_client_configs()\n\n service(\"start\", \"k8s-dqlite\")\n service(\"start\", \"apiserver\")\n\n apply_cni()\n unmark_no_cert_reissue()\n restart_all_services()\n\n\ndef apply_cni():\n waits = 10 # type: int\n print(\"Waiting for node to start.\", end=\" \", flush=True)\n time.sleep(10)\n while waits > 0:\n try:\n subprocess.check_call(\n \"{}/microk8s-kubectl.wrapper get service/kubernetes\".format(snap_path).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n subprocess.check_call(\n \"{}/microk8s-kubectl.wrapper apply -f {}/args/cni-network/cni.yaml\".format(\n snap_path, snapdata_path\n ).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n break\n except subprocess.CalledProcessError:\n print(\".\", end=\" \", flush=True)\n time.sleep(5)\n waits -= 1\n print(\" \")\n\n\ndef reinit_cluster():\n shutil.rmtree(cluster_dir, ignore_errors=True)\n os.mkdir(cluster_dir)\n if os.path.isfile(\"{}/cluster.crt\".format(cluster_backup_dir)):\n # reuse the certificates we had before the cluster formation\n shutil.copy(\n \"{}/cluster.crt\".format(cluster_backup_dir), \"{}/cluster.crt\".format(cluster_dir)\n )\n shutil.copy(\n \"{}/cluster.key\".format(cluster_backup_dir), \"{}/cluster.key\".format(cluster_dir)\n )\n else:\n # This node never joined a cluster. A cluster was formed around it.\n hostname = socket.gethostname() # type: str\n ip = \"127.0.0.1\" # type: str\n shutil.copy(\n \"{}/certs/csr-dqlite.conf.template\".format(snap_path),\n \"{}/var/tmp/csr-dqlite.conf\".format(snapdata_path),\n )\n subprocess.check_call(\n \"{}/bin/sed -i s/HOSTNAME/{}/g {}/var/tmp/csr-dqlite.conf\".format(\n snap_path, hostname, snapdata_path\n ).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n subprocess.check_call(\n \"{}/bin/sed -i s/HOSTIP/{}/g {}/var/tmp/csr-dqlite.conf\".format(\n snap_path, ip, snapdata_path\n ).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n subprocess.check_call(\n \"{0}/openssl.wrapper req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \"\n \"-keyout {1}/var/kubernetes/backend/cluster.key \"\n \"-out {1}/var/kubernetes/backend/cluster.crt \"\n \"-subj /CN=k8s -config {1}/var/tmp/csr-dqlite.conf -extensions v3_ext\".format(\n snap_path, snapdata_path\n ).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n # We reset to the default port and address\n init_data = {\"Address\": \"127.0.0.1:19001\"}\n with open(\"{}/init.yaml\".format(cluster_dir), \"w\") as f:\n yaml.dump(init_data, f)\n\n\ndef is_leader_without_successor():\n \"\"\"Checks if the current node is safe to be removed.\n\n Check if this node acts as a leader to a cluster with more than one nodes where there\n is no other node to take over the leadership.\n\n :return: True if this node is the leader without a successor.\n \"\"\"\n out = subprocess.check_output(\n \"{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt \"\n \"-k {dbdir}/cluster.key -f json k8s .cluster\".format(\n snappath=snap_path, dbdir=cluster_dir\n ).split()\n )\n voters = 0\n data = json.loads(out.decode())\n ep_addresses = []\n for ep in data:\n ep_addresses.append((ep[\"Address\"], ep[\"Role\"]))\n # Role == 0 means we are voters\n if ep[\"Role\"] == 0:\n voters += 1\n\n local_ips = []\n for interface in netifaces.interfaces():\n if netifaces.AF_INET not in netifaces.ifaddresses(interface):\n continue\n for link in netifaces.ifaddresses(interface)[netifaces.AF_INET]:\n local_ips.append(link[\"addr\"])\n\n is_voter = False\n for ep in ep_addresses:\n for ip in local_ips:\n if \"{}:\".format(ip) in ep[0]:\n # ep[1] == ep[Role] == 0 means we are voters\n if ep[1] == 0:\n is_voter = True\n\n if voters == 1 and is_voter and len(ep_addresses) > 1:\n # We have one voter in the cluster and the current node is the only voter\n # and there are other nodes that depend on this node.\n return True\n else:\n return False\n\n\ndef get_dqlite_endpoints():\n \"\"\"\n Return the endpoints the current node has on dqlite and the endpoints of the rest of the nodes.\n\n :return: two lists with the endpoints\n \"\"\"\n out = subprocess.check_output(\n \"{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt \"\n \"-k {dbdir}/cluster.key -f json k8s .cluster\".format(\n snappath=snap_path, dbdir=cluster_dir\n ).split()\n )\n data = json.loads(out.decode())\n ep_addresses = []\n for ep in data:\n ep_addresses.append(ep[\"Address\"])\n local_ips = []\n for interface in netifaces.interfaces():\n if netifaces.AF_INET not in netifaces.ifaddresses(interface):\n continue\n for link in netifaces.ifaddresses(interface)[netifaces.AF_INET]:\n local_ips.append(link[\"addr\"])\n my_ep = []\n other_ep = []\n for ep in ep_addresses:\n found = False\n for ip in local_ips:\n if \"{}:\".format(ip) in ep:\n my_ep.append(ep)\n found = True\n if not found:\n other_ep.append(ep)\n\n return my_ep, other_ep\n\n\ndef delete_dqlite_node(delete_node, dqlite_ep):\n if len(delete_node) > 0 and \"127.0.0.1\" not in delete_node[0]:\n for ep in dqlite_ep:\n try:\n cmd = (\n \"{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt \"\n \"-k {dbdir}/cluster.key -f json k8s\".format(\n snappath=snap_path, dbdir=cluster_dir\n ).split()\n )\n cmd.append(\".remove {}\".format(delete_node[0]))\n subprocess.check_output(cmd)\n break\n except Exception as err:\n print(\"Contacting node {} failed. Error:\".format(ep))\n print(repr(err))\n exit(2)\n\n\n@click.command(context_settings={\"help_option_names\": [\"-h\", \"--help\"]})\ndef leave():\n \"\"\"\n The node will depart from the cluster it is in.\n \"\"\"\n if is_node_running_dqlite():\n if is_node_dqlite_worker():\n reset_current_dqlite_worker_installation()\n else:\n reset_current_dqlite_installation()\n else:\n reset_current_etcd_installation()\n sys.exit(0)\n\n\nif __name__ == \"__main__\":\n leave(prog_name=\"microk8s leave\")\n" }, { "path": "scripts/wrappers/refresh_certs.py", "content": "#!/usr/bin/python3\nimport click\nimport os\nimport subprocess\nfrom dateutil.parser import parse\nimport datetime\n\nfrom common.utils import (\n exit_if_no_root,\n)\n\nfrom common.cluster.utils import (\n is_token_auth_enabled,\n rebuild_x509_auth_client_configs,\n)\n\nsnapdata_path = os.environ.get(\"SNAP_DATA\")\nsnap_path = os.environ.get(\"SNAP\")\nbackup_dir = \"{}/certs-backup/\".format(snapdata_path)\n\ncerts = {\n \"ca.crt\": \"CA\",\n \"server.crt\": \"server\",\n \"front-proxy-client.crt\": \"front proxy client\",\n}\n\n\ndef check_certificate():\n \"\"\"\n Print the days until the current certificate expires\n \"\"\"\n try:\n for file in certs.keys():\n cmd = \"{}/openssl.wrapper x509 -enddate -noout -in {}/certs/{}\".format(\n snap_path, snapdata_path, file\n )\n cert_expire = subprocess.check_output(cmd.split())\n cert_expire_date = cert_expire.decode().split(\"=\")\n date = parse(cert_expire_date[1])\n diff = date - datetime.datetime.now(datetime.timezone.utc)\n click.echo(\"The {} certificate will expire in {} days.\".format(certs[file], diff.days))\n except subprocess.CalledProcessError as e:\n click.echo(\"Failed to get certificate info. {}\".format(e))\n exit(4)\n\n\ndef undo_refresh():\n \"\"\"\n Revert last certificate operation\n \"\"\"\n if not os.path.exists(backup_dir):\n click.echo(\"No previous backup found\")\n exit(1)\n\n try:\n subprocess.check_call(\"cp -r {}/certs {}/\".format(backup_dir, snapdata_path).split())\n subprocess.check_call(\"cp -r {}/credentials {}\".format(backup_dir, snapdata_path).split())\n except subprocess.CalledProcessError:\n click.echo(\"Failed to recover certificates\")\n exit(4)\n\n restart()\n\n\ndef restart(service=\"all\"):\n \"\"\"Restart microk8s services\"\"\"\n if service == \"all\":\n click.echo(\"Restarting, please wait.\")\n try:\n subprocess.check_call(\"{}/microk8s-stop.wrapper\".format(snap_path).split())\n except subprocess.CalledProcessError:\n pass\n\n try:\n subprocess.check_call(\"{}/microk8s-start.wrapper\".format(snap_path).split())\n except subprocess.CalledProcessError:\n click.echo(\"Failed to start MicroK8s after reverting the certificates\")\n exit(4)\n else:\n click.echo(\"Restarting service {}.\".format(service))\n try:\n subprocess.check_call(\"snapctl restart microk8s.daemon-{}\".format(service).split())\n except subprocess.CalledProcessError:\n click.echo(\"Failed to restart service microk8s.daemon-{}.\".format(service))\n exit(4)\n\n\ndef update_configs():\n \"\"\"\n Update all kubeconfig files used by the client and the services\n \"\"\"\n if is_token_auth_enabled():\n p = subprocess.Popen(\n [\"bash\", \"-c\", \". {}/actions/common/utils.sh; update_configs\".format(snap_path)]\n )\n p.communicate()\n else:\n rebuild_x509_auth_client_configs()\n restart(\"kubelite\")\n restart(\"cluster-agent\")\n\n\ndef take_backup():\n \"\"\"\n Backup the current certificates and credentials\n \"\"\"\n try:\n subprocess.check_call(\"mkdir -p {}\".format(backup_dir).split())\n subprocess.check_call(\"cp -r {}/certs {}\".format(snapdata_path, backup_dir).split())\n subprocess.check_call(\"cp -r {}/credentials {}\".format(snapdata_path, backup_dir).split())\n except subprocess.CalledProcessError as e:\n click.echo(\"Failed to backup the current CA. {}\".format(e))\n exit(10)\n\n\ndef reproduce_all_root_ca_certs():\n \"\"\"\n Produce the CA and the rest of the needed certificates (eg service, front-proxy)\n \"\"\"\n subprocess.check_call(\"rm -rf {}/certs/ca.crt\".format(snapdata_path).split())\n subprocess.check_call(\"rm -rf {}/certs/csr.conf\".format(snapdata_path).split())\n p = subprocess.Popen(\n [\"bash\", \"-c\", \". {}/actions/common/utils.sh; produce_certs\".format(snap_path)]\n )\n p.communicate()\n subprocess.check_call(\"rm -rf .slr\".split())\n click.echo(\"Creating new kubeconfig file\")\n update_configs()\n msg = \"\"\"\nThe CA certificates have been replaced. Kubernetes will restart the pods of your workloads.\nAny worker nodes you may have in your cluster need to be removed and \\\nre-joined to become aware of the new CA.\n\"\"\"\n click.echo(msg)\n\n\ndef reproduce_front_proxy_client_cert():\n \"\"\"\n Produce the front proxy client certificate\n \"\"\"\n subprocess.check_call(\"rm -rf {}/certs/front-proxy-client.crt\".format(snapdata_path).split())\n subprocess.check_call(\n [\n \"bash\",\n \"-c\",\n \". {}/actions/common/utils.sh; refresh_csr_conf; gen_proxy_client_cert\".format(\n snap_path\n ),\n ]\n )\n subprocess.check_call(\"rm -rf .slr\".split())\n restart(\"kubelite\")\n\n\ndef reproduce_server_cert():\n \"\"\"\n Produce the server certificate\n \"\"\"\n subprocess.check_call(\"rm -rf {}/certs/server.crt\".format(snapdata_path).split())\n subprocess.check_call(\n [\n \"bash\",\n \"-c\",\n \". {}/actions/common/utils.sh; refresh_csr_conf; gen_server_cert\".format(snap_path),\n ]\n )\n subprocess.check_call(\"rm -rf .slr\".split())\n restart(\"kubelite\")\n restart(\"cluster-agent\")\n\n\ndef refresh_cert(cert):\n \"\"\"\n Refresh the selected certificate with an autogenerated CA\n \"\"\"\n click.echo(\"Taking a backup of the current certificates under {}\".format(backup_dir))\n take_backup()\n try:\n click.echo(\"Creating new certificates\")\n if cert == \"ca.crt\":\n reproduce_all_root_ca_certs()\n elif cert == \"server.crt\":\n reproduce_server_cert()\n elif cert == \"front-proxy-client.crt\":\n reproduce_front_proxy_client_cert()\n else:\n # this should never happen\n click.echo(\"Unknown certificate to refresh\")\n except subprocess.CalledProcessError:\n click.echo(\"Failed to produce new certificates. Reverting.\")\n undo_refresh()\n exit(20)\n\n\ndef install_certs(ca_dir):\n \"\"\"\n Recreate service certificate and front proxy using a user provided CA\n :param ca_dir: path to the ca.crt and ca.key\n \"\"\"\n subprocess.check_call(\"cp {}/ca.crt {}/certs/\".format(ca_dir, snapdata_path).split())\n subprocess.check_call(\"cp {}/ca.key {}/certs/\".format(ca_dir, snapdata_path).split())\n p = subprocess.Popen(\n [\"bash\", \"-c\", \". {}/actions/common/utils.sh; gen_server_cert\".format(snap_path)]\n )\n p.communicate()\n\n\ndef validate_certificates(ca_dir):\n \"\"\"\n Perform some basic testing of the user provided CA\n :param ca_dir: path to the ca.crt and ca.key\n \"\"\"\n if not os.path.isfile(\"{}/ca.crt\".format(ca_dir)) or not os.path.isfile(\n \"{}/ca.key\".format(ca_dir)\n ):\n click.echo(\"Could not find ca.crt and ca.key files in {}\".format(ca_dir))\n exit(30)\n\n try:\n cmd = \"{}/openssl.wrapper rsa -in {}/ca.key -check -noout -out /dev/null\".format(\n snap_path, ca_dir\n )\n subprocess.check_call(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n except subprocess.CalledProcessError as e:\n click.echo(\"CA private key is invalid. {}\".format(e))\n exit(31)\n\n try:\n cmd = \"{}/openssl.wrapper x509 -in {}/ca.crt -text -noout -out /dev/null\".format(\n snap_path, ca_dir\n )\n subprocess.check_call(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n except subprocess.CalledProcessError as e:\n click.echo(\"CA certificate is invalid. {}\".format(e))\n exit(32)\n\n\ndef install_ca(ca_dir):\n \"\"\"\n Install the user provided CA\n :param ca_dir: path to the user provided CA files\n \"\"\"\n click.echo(\"Validating provided certificates\")\n validate_certificates(ca_dir)\n click.echo(\"Taking a backup of the current certificates under {}\".format(backup_dir))\n take_backup()\n click.echo(\"Installing provided certificates\")\n try:\n install_certs(ca_dir)\n except subprocess.CalledProcessError:\n click.echo(\"Failed to produce new certificates. Reverting.\")\n undo_refresh()\n exit(20)\n click.echo(\"Creating new kubeconfig file\")\n update_configs()\n msg = \"\"\"\n The CA certificates have been replaced. Kubernetes will restart the pods of your workloads.\n Any worker nodes you may have in your cluster need to be removed and re-joined to become\n aware of the new CA.\n \"\"\"\n click.echo(msg)\n\n\n@click.command(\n name=\"refresh-certs\",\n help=\"Autogenerate a certificate or replace the root CA with the one found in CA_DIR.\\n\"\n \"Omit the CA_DIR and use the --cert flag to autogenerate a new certificate.\",\n)\n@click.argument(\"ca_dir\", required=False, default=None, type=click.Path(exists=True))\n@click.option(\"-u\", \"--undo\", is_flag=True, default=False, help=\"Revert the last refresh performed\")\n@click.option(\n \"-c\",\n \"--check\",\n is_flag=True,\n default=False,\n help=\"Check the expiration time of the installed CA\",\n)\n@click.option(\n \"-e\",\n \"--cert\",\n type=click.Choice(certs.keys()),\n help=\"The certificate to be autogenerated\",\n)\n@click.option(\n \"-h\",\n \"--help\",\n is_flag=True,\n default=False,\n)\ndef refresh_certs(ca_dir, undo, check, cert, help):\n if help:\n show_help()\n exit(0)\n\n if ca_dir is not None and (undo or check or cert):\n click.echo(\"Please do not set any options in combination with the CA_DIR.\")\n exit(1)\n\n operations_selected = 0\n for op in [undo, check, cert]:\n if op:\n operations_selected += 1\n if operations_selected > 1:\n click.echo(\"Please select only one of the options -c, -u or -e.\")\n exit(2)\n\n # Operations here will need root privileges as some of the credentials\n # and certificates are used by system services.\n exit_if_no_root()\n if check:\n check_certificate()\n exit(0)\n\n if undo:\n undo_refresh()\n exit(0)\n\n if not ca_dir:\n if not cert:\n click.echo(\"Please use the '--cert' flag to select the certificate you need refreshed.\")\n click.echo(\"\")\n click.echo(\"Available certificate options:\")\n click.echo(\"'server.crt': refreshes the server certificate\")\n click.echo(\"'front-proxy-client.crt': refreshes the front proxy client certificate\")\n click.echo(\"'ca.crt': refreshes the root CA and all certificates created from it.\")\n click.echo(\n \" Warning: refreshing the root CA requires nodes to leave and re-join the cluster\"\n )\n exit(3)\n else:\n refresh_cert(cert)\n else:\n install_ca(ca_dir)\n\n\ndef show_help():\n msg = \"\"\"Usage: microk8s refresh-certs [OPTIONS] [CA_DIR]\n\n Replace the CA certificates with the ca.crt and ca.key found in CA_DIR.\n Omit the CA_DIR argument and use the '--cert' flag to auto-generate a new CA\n or any other certificate.\n\nOptions:\n -c, --check Check the expiration time of the installed certificates\n -e, --cert The certificate to be autogenerated, must be one of {}\n -u, --undo Revert the last refresh performed\n -h, --help Show this message and exit.\"\"\"\n click.echo(msg.format(list(certs.keys())))\n\n\nif __name__ == \"__main__\":\n refresh_certs()\n" }, { "path": "scripts/wrappers/remove_node.py", "content": "#!/usr/bin/python3\nimport json\nimport os\nimport shutil\nimport subprocess\nimport sys\n\nimport click\nimport netifaces\n\nfrom ipaddress import ip_address\n\nfrom common.cluster.utils import (\n try_set_file_permissions,\n is_node_running_dqlite,\n is_token_auth_enabled,\n)\n\nsnap_path = os.environ.get(\"SNAP\")\nsnapdata_path = os.environ.get(\"SNAP_DATA\")\ncallback_tokens_file = \"{}/credentials/callback-tokens.txt\".format(snapdata_path)\n\ncluster_dir = \"{}/var/kubernetes/backend\".format(snapdata_path)\n\n\ndef remove_dqlite_node(node, force=False):\n try:\n # If node is an IP address, find the node name.\n is_node_ip = True\n try:\n ip_address(node)\n except ValueError:\n is_node_ip = False\n\n if is_node_ip:\n node_info = subprocess.check_output(\n \"{}/microk8s-kubectl.wrapper get no -o json\".format(snap_path).split()\n )\n info = json.loads(node_info.decode())\n found = False\n for n in info[\"items\"]:\n if found:\n break\n for a in n[\"status\"][\"addresses\"]:\n if a[\"type\"] == \"InternalIP\" and a[\"address\"] == node:\n node = n[\"metadata\"][\"name\"]\n found = True\n break\n\n # Make sure this node exists\n node_info = subprocess.check_output(\n \"{}/microk8s-kubectl.wrapper get no {} -o json\".format(snap_path, node).split()\n )\n info = json.loads(node_info.decode())\n node_address = None\n for a in info[\"status\"][\"addresses\"]:\n if a[\"type\"] == \"InternalIP\":\n node_address = a[\"address\"]\n break\n\n if not node_address:\n print(\"Node {} is not part of the cluster.\".format(node))\n exit(1)\n\n node_ep = None\n my_ep, other_ep = get_dqlite_endpoints()\n for ep in other_ep:\n if ep.startswith(\"{}:\".format(node_address)):\n node_ep = ep\n\n if node_ep and force:\n delete_dqlite_node([node_ep], my_ep)\n elif node_ep and not force:\n print(\n \"Removal failed. Node {} is registered with dqlite. \"\n \"Please, run first 'microk8s leave' on the departing node. \\n\"\n \"If the node is not available anymore and will never attempt to join the cluster \"\n \"in the future use the '--force' flag \\n\"\n \"to unregister the node while removing it.\".format(node)\n )\n exit(1)\n\n except subprocess.CalledProcessError:\n print(\"Node {} does not exist in Kubernetes.\".format(node))\n if force:\n print(\"Attempting to remove {} from dqlite.\".format(node))\n # Make sure we do not have the node in dqlite.\n # We assume the IP is provided to denote the\n my_ep, other_ep = get_dqlite_endpoints()\n for ep in other_ep:\n if ep.startswith(\"{}:\".format(node)):\n print(\"Removing node entry found in dqlite.\")\n delete_dqlite_node([ep], my_ep)\n exit(1)\n\n remove_node(node)\n\n\ndef remove_node(node):\n try:\n # Make sure this node exists\n subprocess.check_call(\n \"{}/microk8s-kubectl.wrapper get no {}\".format(snap_path, node).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n except subprocess.CalledProcessError:\n print(\"Node {} does not exist.\".format(node))\n exit(1)\n\n if is_token_auth_enabled():\n remove_kubelet_token(node)\n remove_callback_token(node)\n subprocess.check_call(\n \"{}/microk8s-kubectl.wrapper delete no {}\".format(snap_path, node).split(),\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n\n\ndef remove_kubelet_token(node):\n \"\"\"\n Remove a token for a node in the known tokens\n\n :param node: the name of the node\n \"\"\"\n file = \"{}/credentials/known_tokens.csv\".format(snapdata_path)\n backup_file = \"{}.backup\".format(file)\n token = \"system:node:{}\".format(node)\n # That is a critical section. We need to protect it.\n with open(backup_file, \"w\") as back_fp:\n with open(file, \"r\") as fp:\n for _, line in enumerate(fp):\n if token in line:\n continue\n back_fp.write(\"{}\".format(line))\n\n try_set_file_permissions(backup_file)\n shutil.copyfile(backup_file, file)\n\n\ndef get_dqlite_endpoints():\n \"\"\"\n Return the endpoints the current node has on dqlite and the endpoints of the rest of the nodes.\n\n :return: two lists with the endpoints\n \"\"\"\n out = subprocess.check_output(\n \"{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt \"\n \"-k {dbdir}/cluster.key -f json k8s .cluster\".format(\n snappath=snap_path, dbdir=cluster_dir\n ).split()\n )\n data = json.loads(out.decode())\n ep_addresses = []\n for ep in data:\n ep_addresses.append(ep[\"Address\"])\n local_ips = []\n for interface in netifaces.interfaces():\n if netifaces.AF_INET not in netifaces.ifaddresses(interface):\n continue\n for link in netifaces.ifaddresses(interface)[netifaces.AF_INET]:\n local_ips.append(link[\"addr\"])\n my_ep = []\n other_ep = []\n for ep in ep_addresses:\n found = False\n for ip in local_ips:\n if \"{}:\".format(ip) in ep:\n my_ep.append(ep)\n found = True\n if not found:\n other_ep.append(ep)\n\n return my_ep, other_ep\n\n\ndef delete_dqlite_node(delete_node, dqlite_ep):\n if len(delete_node) > 0 and \"127.0.0.1\" not in delete_node[0]:\n for ep in dqlite_ep:\n try:\n cmd = (\n \"{snappath}/bin/dqlite -s file://{dbdir}/cluster.yaml -c {dbdir}/cluster.crt \"\n \"-k {dbdir}/cluster.key -f json k8s\".format(\n snappath=snap_path, dbdir=cluster_dir\n ).split()\n )\n cmd.append(\".remove {}\".format(delete_node[0]))\n subprocess.check_output(cmd)\n break\n except Exception as err:\n print(\"Contacting node {} failed. Error:\".format(ep))\n print(repr(err))\n exit(2)\n\n\ndef remove_callback_token(node):\n \"\"\"\n Remove a callback token\n\n :param node: the node\n \"\"\"\n tmp_file = \"{}.tmp\".format(callback_tokens_file)\n if not os.path.isfile(callback_tokens_file):\n open(callback_tokens_file, \"a+\")\n os.chmod(callback_tokens_file, 0o600)\n with open(tmp_file, \"w\") as backup_fp:\n os.chmod(tmp_file, 0o600)\n with open(callback_tokens_file, \"r+\") as callback_fp:\n # Entries are of the format: 'node_hostname:agent_port token'\n # We need to get the node_hostname part\n for line in callback_fp:\n parts = line.split(\":\")\n if parts[0] == node:\n continue\n else:\n backup_fp.write(line)\n\n try_set_file_permissions(tmp_file)\n shutil.move(tmp_file, callback_tokens_file)\n\n\n@click.command()\n@click.argument(\"node\", required=True)\n@click.option(\n \"--force\",\n is_flag=True,\n required=False,\n default=False,\n help=\"Force the node removal operation. (default: false)\",\n)\ndef reset(node, force):\n \"\"\"\n Remove a node from the cluster\n \"\"\"\n if is_node_running_dqlite():\n remove_dqlite_node(node, force)\n else:\n remove_node(node)\n sys.exit(0)\n\n\nif __name__ == \"__main__\":\n reset(prog_name=\"microk8s remove-node\")\n" }, { "path": "scripts/wrappers/reset.py", "content": "#!/usr/bin/python3\nimport click\nimport os\nimport subprocess\nimport shutil\nimport sys\nimport time\n\nfrom common.utils import (\n get_available_addons,\n get_current_arch,\n get_status,\n snap_data,\n wait_for_ready,\n snap_common,\n is_cluster_locked,\n exit_if_no_permission,\n ensure_started,\n exit_if_no_root,\n)\n\n\nKUBECTL = os.path.expandvars(\"$SNAP/microk8s-kubectl.wrapper\")\n\n\ndef exit_if_multinode():\n \"\"\"\n Exit if we cannot get the list of nodes or if we are in a multinode cluster\n \"\"\"\n cmd = [KUBECTL, \"get\", \"no\", \"-o\", \"name\"]\n res = run_silently(cmd)\n if not res:\n print(\"Failed to query the cluster nodes.\")\n sys.exit(1)\n nodes = res.split()\n if len(nodes) > 1:\n print(\n \"This is a multi-node MicroK8s deployment. Reset is applicable for single node clusters.\"\n )\n print(\"Please remove all joined nodes before calling reset.\")\n sys.exit(0)\n\n\ndef disable_addon(repo, addon, args=None):\n \"\"\"\n Try to disable an addon. Ignore any errors and/or silence any output.\n \"\"\"\n if args is None:\n args = []\n wait_for_ready(timeout=30)\n script = snap_common() / \"addons\" / repo / \"addons\" / addon / \"disable\"\n if not script.exists():\n return\n\n try:\n subprocess.run([script, *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n except (FileNotFoundError, PermissionError, subprocess.CalledProcessError) as e:\n print(f\"Ignoring error: {e}\", file=sys.stderr)\n\n wait_for_ready(timeout=30)\n\n\ndef disable_addons(destroy_storage):\n \"\"\"\n Iterate over all addons and disable the enabled ones.\n \"\"\"\n print(\"Disabling all addons\")\n\n available_addons_info = get_available_addons(get_current_arch())\n enabled, disabled = get_status(available_addons_info, True)\n for addon in available_addons_info:\n # Do not disable HA\n if addon[\"name\"] == \"ha-cluster\":\n continue\n\n # Do not disable the community repository\n if addon[\"name\"] == \"community\":\n continue\n\n print(f\"Disabling addon : {addon['repository']}/{addon['name']}\")\n # Do not disable disabled addons\n if addon in disabled:\n continue\n\n if (addon[\"name\"] == \"hostpath-storage\" or addon[\"name\"] == \"storage\") and destroy_storage:\n disable_addon(addon[\"repository\"], f\"{addon['name']}\", [\"destroy-storage\"])\n else:\n disable_addon(addon[\"repository\"], addon[\"name\"])\n print(\"All addons are disabled.\")\n\n\ndef cni(operation=\"apply\"):\n \"\"\"\n Apply of delete the CNI manifest of our cluster if it exists. Silence any output.\n \"\"\"\n cni_yaml = f\"{snap_data()}/args/cni-network/cni.yaml\"\n if os.path.exists(cni_yaml):\n if operation == \"apply\":\n print(\"Setting up the CNI\")\n else:\n print(\"Deleting the CNI\")\n subprocess.run(\n [KUBECTL, operation, \"-f\", cni_yaml],\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n )\n\n\ndef clean_cluster():\n \"\"\"\n Clean up the cluster by:\n 1. Delete any resources left\n 2. Restart so the cluster resets\n 3. Delete any locks and addon binaries.\n \"\"\"\n cmd = [KUBECTL, \"get\", \"ns\", \"-o=name\"]\n res = run_silently(cmd)\n nss = []\n if res:\n nss = res.split()\n resources = [\"replicationcontrollers\", \"daemonsets\", \"deployments\", \"statefulsets\"]\n for ns in nss:\n ns_name = ns.split(\"/\")[-1]\n print(f\"Cleaning resources in namespace {ns_name}\")\n for rs in resources:\n # we remove first resources that are automatically recreated so we do not risk race conditions\n # during which a deployment for example is recreated while any tokens are missing\n cmd = [KUBECTL, \"delete\", \"--all\", rs, \"-n\", ns_name, \"--timeout=60s\"]\n subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n remove_extra_resources(ns_name)\n\n remove_crds()\n remove_priority_classes()\n remove_storage_classes()\n remove_non_namespaced_resources()\n\n for ns in nss:\n ns_name = ns.split(\"/\")[-1]\n if ns_name in [\"default\", \"kube-public\", \"kube-system\", \"kube-node-lease\"]:\n continue\n print(f\"Removing {ns}\")\n cmd = [KUBECTL, \"delete\", ns, \"--timeout=60s\"]\n subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n\n restart_cluster()\n remove_binaries()\n reset_cert_reissue()\n\n\ndef remove_storage_classes():\n \"\"\"\n Remove storage classes. Silence any output.\n \"\"\"\n print(\"Removing StorageClasses\")\n cmd = [KUBECTL, \"get\", \"storageclasses\", \"-o=name\"]\n res = run_silently(cmd)\n classes = []\n if res:\n classes = res.split()\n for cs in classes:\n if \"microk8s-hostpath\" in cs:\n continue\n cmd = [KUBECTL, \"delete\", cs, \"--timeout=60s\"]\n subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n\n\ndef remove_crds():\n print(\"Removing CRDs\")\n cmd = [\n KUBECTL,\n \"delete\",\n \"--all\",\n \"customresourcedefinitions.apiextensions.k8s.io\",\n \"--timeout=60s\",\n ]\n subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n\n\ndef remove_non_namespaced_resources():\n resources = [\n \"MutatingWebhookConfigurations\",\n \"ValidatingAdmissionPolicies\",\n \"ValidatingAdmissionPolicyBindings\",\n \"ValidatingWebhookConfigurations\",\n \"CertificateSigningRequests\",\n ]\n\n for r in resources:\n print(f\"Removing {r}\")\n cmd = [KUBECTL, \"delete\", \"--all\", r, \"--timeout=60s\"]\n subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n\n\ndef remove_priority_classes():\n \"\"\"\n Remove priority classes. Silence any output.\n \"\"\"\n print(\"Removing PriorityClasses\")\n cmd = [KUBECTL, \"get\", \"priorityclasses\", \"-o=name\"]\n res = run_silently(cmd)\n classes = []\n if res:\n classes = res.split()\n for cs in classes:\n if \"system-cluster-critical\" in cs or \"system-node-critical\" in cs:\n continue\n cmd = [KUBECTL, \"delete\", cs, \"--timeout=60s\"]\n subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n\n\ndef reset_cert_reissue():\n \"\"\"\n Remove the certificate no refresh lock.\n \"\"\"\n lock = snap_data() / \"var\" / \"lock\" / \"no-cert-reissue\"\n if lock.exists():\n cmd = f\"rm -rf {lock}\"\n subprocess.run(cmd.split())\n\n\ndef remove_binaries():\n \"\"\"\n Remove binaries pulled in by addons.\n \"\"\"\n bins_dir = snap_data() / \"bin\"\n if bins_dir.exists():\n shutil.rmtree(bins_dir)\n\n\ndef restart_cluster():\n \"\"\"\n Restart a cluster by calling the stop and start wrappers.\n \"\"\"\n print(\"Restarting cluster\")\n cmd = [f\"{os.environ['SNAP']}/microk8s-stop.wrapper\"]\n subprocess.run(cmd)\n time.sleep(5)\n cmd = [f\"{os.environ['SNAP']}/microk8s-start.wrapper\"]\n subprocess.run(cmd)\n wait_for_ready(timeout=30)\n ensure_started()\n\n\ndef remove_extra_resources(ns_name):\n # Remove all resource types except the standard k8s apiservices themselves\n cmd = [KUBECTL, \"api-resources\", \"-o\", \"name\", \"--verbs=delete\", \"--namespaced=true\"]\n res = run_silently(cmd)\n if not res:\n return\n extra_resources = res.split()\n for rs in extra_resources:\n if rs.startswith(\"apiservices\"):\n continue\n cmd = [KUBECTL, \"delete\", \"--all\", rs, \"-n\", ns_name, \"--timeout=3s\"]\n subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n\n\ndef run_silently(cmd):\n result = subprocess.run(\n cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE\n )\n\n try:\n result.check_returncode()\n except subprocess.CalledProcessError:\n return None\n\n return result.stdout.decode(\"utf-8\")\n\n\ndef preflight_check():\n \"\"\"\n Preliminary checks to see if we can proceed with cluster reset\n \"\"\"\n exit_if_no_root()\n exit_if_no_permission()\n is_cluster_locked()\n ensure_started()\n wait_for_ready(timeout=30)\n exit_if_multinode()\n\n\n@click.command(context_settings={\"help_option_names\": [\"-h\", \"--help\"]})\n@click.option(\n \"--destroy-storage\",\n is_flag=True,\n required=False,\n default=True,\n help=\"Also destroy storage. (default: true)\",\n)\ndef reset(destroy_storage):\n \"\"\"\n Return the MicroK8s node to the default initial state.\n\n This process may take some time. All addons will be disabled and\n the configuration will be reinitialized.\n \"\"\"\n preflight_check()\n disable_addons(destroy_storage)\n cni(\"delete\")\n clean_cluster()\n cni(\"apply\")\n\n\nif __name__ == \"__main__\":\n reset(prog_name=\"microk8s reset\")\n" }, { "path": "scripts/wrappers/status.py", "content": "#!/usr/bin/python3\nimport argparse\nimport sys\n\nfrom common.utils import (\n exit_if_no_permission,\n exit_if_stopped,\n is_cluster_locked,\n is_ha_enabled,\n get_dqlite_info,\n wait_for_ready,\n is_cluster_ready,\n get_available_addons,\n get_current_arch,\n get_addon_by_name,\n get_status,\n get_etcd_info,\n is_external_etcd,\n)\n\n\ndef print_short(isReady, enabled_addons, disabled_addons):\n if isReady:\n print(\"microk8s is running\")\n print(\"addons:\")\n if enabled_addons and len(enabled_addons) > 0:\n for enabled in enabled_addons:\n print(\"{}/{}: enabled\".format(enabled[\"repository\"], enabled[\"name\"]))\n if disabled_addons and len(disabled_addons) > 0:\n for disabled in disabled_addons:\n print(\"{}/{}: disabled\".format(disabled[\"repository\"], disabled[\"name\"]))\n else:\n print(\"microk8s is not running. Use microk8s inspect for a deeper inspection.\")\n\n\ndef print_pretty(isReady, enabled_addons, disabled_addons):\n console_formatter = \"{:>3} {:<20} # ({}) {}\"\n if isReady:\n print(\"microk8s is running\")\n etcd_endpoints = get_etcd_info()\n if not is_ha_enabled():\n print(\"high-availability: no\")\n if etcd_endpoints:\n print(\"{:>2}{}\".format(\"\", \"datastore endpoints:\"))\n for endpoint in etcd_endpoints:\n print(\"{:>4}{}\".format(\"\", endpoint))\n elif not is_external_etcd():\n info = get_dqlite_info()\n if ha_cluster_formed(info):\n print(\"high-availability: yes\")\n else:\n print(\"high-availability: no\")\n\n masters = \"none\"\n standby = \"none\"\n for node in info:\n if node[1] == \"voter\":\n if masters == \"none\":\n masters = \"{}\".format(node[0])\n else:\n masters = \"{} {}\".format(masters, node[0])\n if node[1] == \"standby\":\n if standby == \"none\":\n standby = \"{}\".format(node[0])\n else:\n standby = \"{} {}\".format(standby, node[0])\n\n print(\"{:>2}{} {}\".format(\"\", \"datastore master nodes:\", masters))\n print(\"{:>2}{} {}\".format(\"\", \"datastore standby nodes:\", standby))\n elif etcd_endpoints:\n print(\"{:>2}{}\".format(\"\", \"datastore endpoints:\"))\n for endpoint in etcd_endpoints:\n print(\"{:>4}{}\".format(\"\", endpoint))\n\n print(\"addons:\")\n if enabled_addons and len(enabled_addons) > 0:\n print(\"{:>2}{}\".format(\"\", \"enabled:\"))\n for enabled in enabled_addons:\n print(\n console_formatter.format(\n \"\", enabled[\"name\"], enabled[\"repository\"], enabled[\"description\"]\n )\n )\n if disabled_addons and len(disabled_addons) > 0:\n print(\"{:>2}{}\".format(\"\", \"disabled:\"))\n for disabled in disabled_addons:\n print(\n console_formatter.format(\n \"\", disabled[\"name\"], disabled[\"repository\"], disabled[\"description\"]\n )\n )\n else:\n print(\"microk8s is not running. Use microk8s inspect for a deeper inspection.\")\n\n\ndef print_short_yaml(isReady, enabled_addons, disabled_addons):\n print(\"microk8s:\")\n print(\"{:>2}{} {}\".format(\"\", \"running:\", isReady))\n\n if isReady:\n print(\"addons:\")\n for enabled in enabled_addons:\n print(\" {}/{}: enabled\".format(enabled[\"repository\"], enabled[\"name\"]))\n\n for disabled in disabled_addons:\n print(\" {}/{}: disabled\".format(disabled[\"repository\"], disabled[\"name\"]))\n else:\n print(\n \"{:>2}{} {}\".format(\n \"\",\n \"message:\",\n \"microk8s is not running. Use microk8s inspect for a deeper inspection.\",\n )\n )\n\n\ndef print_yaml(isReady, enabled_addons, disabled_addons):\n print(\"microk8s:\")\n print(\"{:>2}{} {}\".format(\"\", \"running:\", isReady))\n\n print(\"{:>2}\".format(\"high-availability:\"))\n ha_enabled = is_ha_enabled()\n print(\"{:>2}{} {}\".format(\"\", \"enabled:\", ha_enabled))\n if ha_enabled:\n info = get_dqlite_info()\n print(\"{:>2}{}\".format(\"\", \"nodes:\"))\n for node in info:\n print(\"{:>6}address: {:<1}\".format(\"- \", node[0]))\n print(\"{:>6}role: {:<1}\".format(\"\", node[1]))\n\n if isReady:\n print(\"{:>2}\".format(\"addons:\"))\n for enabled in enabled_addons:\n print(\"{:>4}name: {:<1}\".format(\"- \", enabled[\"name\"]))\n print(\"{:>4}repository: {:<1}\".format(\"\", enabled[\"repository\"]))\n print(\"{:>4}description: {:<1}\".format(\"\", enabled[\"description\"]))\n print(\"{:>4}version: {:<1}\".format(\"\", enabled[\"version\"]))\n print(\"{:>4}status: enabled\".format(\"\"))\n\n for disabled in disabled_addons:\n print(\"{:>4}name: {:<1}\".format(\"- \", disabled[\"name\"]))\n print(\"{:>4}repository: {:<1}\".format(\"\", disabled[\"repository\"]))\n print(\"{:>4}description: {:<1}\".format(\"\", disabled[\"description\"]))\n print(\"{:>4}version: {:<1}\".format(\"\", disabled[\"version\"]))\n print(\"{:>4}status: disabled\".format(\"\"))\n else:\n print(\n \"{:>2}{} {}\".format(\n \"\",\n \"message:\",\n \"microk8s is not running. Use microk8s inspect for a deeper inspection.\",\n )\n )\n\n\ndef print_addon_status(enabled):\n if len(enabled) > 0:\n print(\"enabled\")\n else:\n print(\"disabled\")\n\n\ndef ha_cluster_formed(info):\n voters = 0\n for node in info:\n if node[1] == \"voter\":\n voters += 1\n ha_formed = False\n if voters > 2:\n ha_formed = True\n return ha_formed\n\n\nif __name__ == \"__main__\":\n exit_if_no_permission()\n exit_if_stopped()\n is_cluster_locked()\n\n # initiate the parser with a description\n parser = argparse.ArgumentParser(\n description=\"Microk8s cluster status check.\", prog=\"microk8s status\"\n )\n parser.add_argument(\n \"--format\",\n help=\"print cluster and addon status, output can be in yaml, pretty or short\",\n default=\"pretty\",\n choices={\"pretty\", \"yaml\", \"short\"},\n )\n parser.add_argument(\n \"-w\", \"--wait-ready\", action=\"store_true\", help=\"wait until the cluster is in ready state\"\n )\n parser.add_argument(\n \"-t\",\n \"--timeout\",\n help=\"specify a timeout in seconds when waiting for the cluster to be ready.\",\n type=int,\n default=0,\n )\n parser.add_argument(\"-a\", \"--addon\", help=\"check the status of an addon.\", default=\"all\")\n parser.add_argument(\n \"--yaml\", action=\"store_true\", help=\"DEPRECATED, use '--format yaml' instead\"\n )\n\n # read arguments from the command line\n args = parser.parse_args()\n\n wait_ready = args.wait_ready\n timeout = args.timeout\n yaml_short = args.yaml\n\n if wait_ready:\n is_ready = wait_for_ready(timeout)\n else:\n is_ready = is_cluster_ready()\n\n available_addons = get_available_addons(get_current_arch())\n\n if args.addon != \"all\":\n available_addons = get_addon_by_name(available_addons, args.addon)\n\n enabled, disabled = get_status(available_addons, is_ready)\n\n if args.addon != \"all\":\n print_addon_status(enabled)\n else:\n if args.format == \"yaml\":\n print_yaml(is_ready, enabled, disabled)\n elif args.format == \"short\":\n print_short(is_ready, enabled, disabled)\n else:\n if yaml_short:\n print_short_yaml(is_ready, enabled, disabled)\n else:\n print_pretty(is_ready, enabled, disabled)\n\n if not is_ready:\n sys.exit(1)\n" }, { "path": "scripts/wrappers/upgrade.py", "content": "#!/usr/bin/python3\nimport os\nimport argparse\nimport subprocess\n\nimport requests\nimport urllib3\nfrom common.utils import exit_if_no_permission, is_cluster_locked\n\nurllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\nCLUSTER_API = \"cluster/api/v1.0\"\nsnapdata_path = os.environ.get(\"SNAP_DATA\")\nsnap_path = os.environ.get(\"SNAP\")\n\n\ndef upgrade_master(upgrade, phase):\n \"\"\"\n Upgrade the master node\n :param upgrade: which upgrade to call\n :param phase: prepare, commit or rollback\n :return:\n \"\"\"\n try:\n upgrade_script = \"{}/upgrade-scripts/{}/{}-master.sh\".format(snap_path, upgrade, phase)\n if os.path.isfile(upgrade_script):\n print(\"Running {}-upgrade script\".format(phase))\n out = subprocess.check_output(upgrade_script)\n print(out)\n except subprocess.CalledProcessError as e:\n print(\"{}-upgrade step failed\".format(phase))\n raise e\n\n\ndef node_upgrade(upgrade, phase, node_ep, token):\n \"\"\"\n Upgrade a node\n :param upgrade: which upgrade to call\n :param phase: prepare, commit or rollback\n :param node_ep: the node endpoint the nodes cluster agent listens at\n :param token: the callback token to access the node\n :return:\n \"\"\"\n try:\n upgrade_script = \"{}/upgrade-scripts/{}/{}-node.sh\".format(snap_path, upgrade, phase)\n if os.path.isfile(upgrade_script):\n remote_op = {\"callback\": token, \"phase\": phase, \"upgrade\": upgrade}\n # TODO: handle ssl verification\n res = requests.post(\n \"https://{}/{}/upgrade\".format(node_ep, CLUSTER_API), json=remote_op, verify=False\n )\n if res.status_code != 200:\n print(\"Failed to perform a {} on node {}\".format(remote_op[\"upgrade\"], node_ep))\n raise Exception(\"Failed to {} on {}\".format(phase, node_ep))\n except subprocess.CalledProcessError as e:\n print(\"{} upgrade step failed on {}\".format(phase, node_ep))\n raise e\n\n\ndef rollback(upgrade):\n \"\"\"\n The rollback method that oversees the rollback of the cluster\n :param upgrade: which upgrade to call\n \"\"\"\n # We should get the nodes without checking their existence from the API server\n node_info = get_nodes_info(safe=False)\n\n upgrade_log_file = \"{}/var/log/upgrades/{}.log\".format(snapdata_path, upgrade)\n with open(upgrade_log_file, \"r\") as log:\n for line in log:\n parts = line.split(\" \")\n node_type = parts[0]\n phase = parts[1]\n if node_type == \"node\":\n node_ep = parts[2].rstrip()\n else:\n node_ep = \"localhost\"\n if phase == \"commit\":\n print(\"Rolling back {} on {}\".format(phase, node_ep))\n if node_type == \"node\":\n tokens = [t for ep, t in node_info if node_ep.startswith(ep)]\n if len(tokens) != 0:\n token = tokens[0]\n node_upgrade(upgrade, \"rollback\", node_ep, token)\n else:\n upgrade_master(upgrade, \"rollback\")\n\n\ndef run_upgrade(upgrade):\n \"\"\"\n The upgrade method that oversees the upgrade of the cluster\n :param upgrade: which upgrade to call\n \"\"\"\n node_info = get_nodes_info()\n\n log_dir = \"{}/var/log/upgrades\".format(snapdata_path)\n upgrade_log_file = \"{}/{}.log\".format(log_dir, upgrade)\n try:\n os.makedirs(log_dir, exist_ok=True)\n with open(upgrade_log_file, \"w\") as log:\n log.writelines([\"master prepare\"])\n upgrade_master(upgrade, \"prepare\")\n log.flush()\n for node_ep, token in node_info:\n log.writelines([\"\\nnode prepare {}\".format(node_ep)])\n node_upgrade(upgrade, \"prepare\", node_ep, token)\n log.flush()\n\n for node_ep, token in node_info:\n log.writelines([\"\\nnode commit {}\".format(node_ep)])\n node_upgrade(upgrade, \"commit\", node_ep, token)\n log.flush()\n\n log.writelines([\"\\nmaster commit\"])\n upgrade_master(upgrade, \"commit\")\n log.flush()\n\n except Exception as e:\n print(\"Error in upgrading. Error: {}\".format(e))\n log.close()\n rollback(upgrade)\n exit(2)\n\n\ndef get_nodes_info(safe=True):\n \"\"\"\n Get the list of node endpoints and tokens in the cluster\n :return:\n \"\"\"\n callback_tokens_file = \"{}/credentials/callback-tokens.txt\".format(snapdata_path)\n node_info = []\n if safe:\n try:\n nodes = subprocess.check_output(\n \"{}/microk8s-kubectl.wrapper get no\".format(snap_path).split()\n )\n if os.path.isfile(callback_tokens_file):\n with open(callback_tokens_file, \"r+\") as fp:\n for _, line in enumerate(fp):\n parts = line.split()\n node_ep = parts[0]\n host = node_ep.split(\":\")[0]\n if host not in nodes.decode():\n print(\"Node {} not present\".format(host))\n continue\n node_info.append((parts[0], parts[1]))\n except subprocess.CalledProcessError:\n print(\"Error in gathering cluster node information. Upgrade aborted.\")\n exit(1)\n else:\n if os.path.isfile(callback_tokens_file):\n with open(callback_tokens_file, \"r+\") as fp:\n for _, line in enumerate(fp):\n parts = line.split()\n node_info.append((parts[0], parts[1]))\n\n return node_info\n\n\ndef list_upgrades():\n \"\"\"\n List all available upgrades\n \"\"\"\n upgrades_dir = \"{}/upgrade-scripts/\".format(snap_path)\n upgrades = [\n dI for dI in os.listdir(upgrades_dir) if os.path.isdir(os.path.join(upgrades_dir, dI))\n ]\n for u in upgrades:\n print(u)\n\n\nif __name__ == \"__main__\":\n exit_if_no_permission()\n is_cluster_locked()\n\n # initiate the parser with a description\n parser = argparse.ArgumentParser(description=\"MicroK8s supervised upgrades.\", prog=\"upgrade\")\n parser.add_argument(\n \"-l\", \"--list\", help=\"list available upgrades\", nargs=\"?\", const=True, type=bool\n )\n parser.add_argument(\n \"-r\", \"--run\", help=\"run a specific upgrade script\", nargs=\"?\", type=str, default=None\n )\n parser.add_argument(\n \"-u\", \"--undo\", help=\"rollback a specific upgrade\", nargs=\"?\", type=str, default=None\n )\n args = parser.parse_args()\n\n run = args.run\n ls = args.list\n undo = args.undo\n\n if ls:\n list_upgrades()\n elif run:\n run_upgrade(run)\n elif undo:\n rollback(undo)\n else:\n print(\"Unknown option\")\n exit(1)\n\n exit(0)\n" }, { "path": "scripts/wrappers/version.py", "content": "#!/usr/bin/env python3\n\nfrom os import getenv\n\n\ndef get_snap_version() -> str:\n return getenv(\"SNAP_VERSION\", \"(unknown)\")\n\n\ndef get_snap_revision() -> str:\n return getenv(\"SNAP_REVISION\", \"(unknown)\")\n\n\ndef print_versions() -> None:\n version = get_snap_version()\n revision = get_snap_revision()\n print(f\"MicroK8s {version} revision {revision}\")\n\n\nif __name__ == \"__main__\":\n print_versions()\n" }, { "path": "snap/hooks/configure", "content": "#!/usr/bin/env bash\n\nset -eux\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\n# Make sure either the install hook has run or we are refreshing an already existing snap as indicated\n# by the existence of certificates.\nif [ ! -f \"${SNAP_DATA}/var/lock/installed.lock\" ] && [ ! -f ${SNAP_DATA}/certs/csr.conf.template ]\nthen\n exit 0\nfi\n\nif is_strict\nthen\n echo \"Checking snap interfaces...\"\n\n check_snap_interfaces # Check for interfaces but do not start until this script has run.\nelse\n # In classic we do not make use of the status flag. Here we clean any \"blocked\" message that may have\n # come from a failed strict deployment\n snapctl set-health okay\nfi\n\nneed_api_restart=false\nneed_cluster_agent_restart=false\nneed_proxy_restart=false\nneed_kubelet_restart=false\nneed_controller_restart=false\nneed_scheduler_restart=false\n\n# Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box.\nif ! [ -e /var/lib/kubelet ] && ln -s $SNAP_COMMON/var/lib/kubelet /var/lib/kubelet; then\n echo \"/var/lib/kubelet linked to $SNAP_COMMON\"\nfi\n\nSNAP_DATA_CURRENT=`echo \"${SNAP_DATA}\" | sed -e \"s,${SNAP_REVISION},current,\"`\n\n# Try to symlink /var/lib/calico so that the Calico CNI plugin picks up the mtu configuration.\nif ! [ -e /var/lib/calico ]; then\n if ln -s $SNAP_DATA_CURRENT/var/lib/calico /var/lib/calico; then\n echo \"/var/lib/calico linked to $SNAP_DATA_CURRENT/var/lib/calico\"\n fi\nfi\n\n# Try to symlink standard CNI Kubernetes directories.\nif ! [ -e /etc/cni/net.d ]; then\n if mkdir -p /etc/cni && ln -s $SNAP_DATA_CURRENT/args/cni-network /etc/cni/net.d; then\n echo \"/etc/cni/net.d linked to $SNAP_DATA_CURRENT/args/cni-network\"\n fi\nfi\nif ! [ -e /opt/cni/bin ]; then\n if mkdir -p /opt/cni && ln -s $SNAP_DATA_CURRENT/opt/cni/bin /opt/cni/bin; then\n echo \"/opt/cni/bin linked to $SNAP_DATA_CURRENT/opt/cni/bin\"\n fi\nfi\n\n# If the configurations directory is missing from SNAP_COMMON, we are upgrading from an older MicroK8s version.\nif [ ! -d \"${SNAP_COMMON}/etc/launcher\" ]\nthen\n mkdir -p \"${SNAP_COMMON}/etc/launcher\"\nfi\n\n# snap set microk8s config=\"$(cat config.yaml)\"\nconfig=\"$(snapctl get config || true)\"\nif [ ! -z \"${config}\" ]\nthen\n # Only write config file if not already applied.\n applied_config=\"$(cat ${SNAP_COMMON}/etc/launcher/snap-set.yaml.applied || true)\"\n if [ -z \"${applied_config}\" ] || [ \"${config}\" != \"${applied_config}\" ]\n then\n echo \"${config}\" > \"${SNAP_COMMON}/etc/launcher/snap-set.yaml\"\n fi\nfi\n\n# If the addons directory is missing from SNAP_COMMON, then we are upgrading from an older MicroK8s version.\nif [ ! -d \"${SNAP_COMMON}/addons\" ]\nthen\n mkdir -p ${SNAP_COMMON}/addons\n snap_current=`echo \"${SNAP}\" | sed -e \"s,${SNAP_REVISION},current,\"`\n for addon in $(cat \"${SNAP}/addons/.auto-add\"); do\n \"${SNAP}/git.wrapper\" clone \"${snap_current}/addons/${addon}\" \"${SNAP_COMMON}/addons/${addon}\"\n done\nfi\n\nif [ ! -d \"${SNAP_COMMON}/plugins\" ]\nthen\n mkdir -p ${SNAP_COMMON}/plugins\nfi\n\n#Allow the ability to add external IPs to the csr, by moving the csr.conf.template to SNAP_DATA\n# TODO(neoaggelos): investigate if this is needed\nif [ ! -f ${SNAP_DATA}/certs/csr.conf.template ]\nthen\n cp ${SNAP}/certs/csr.conf.template ${SNAP_DATA}/certs/csr.conf.template\nfi\n\n# Enable the aggregation layer\nif ! grep \"requestheader-client-ca-file\" ${SNAP_DATA}/args/kube-apiserver\nthen\n echo \"Patching requestheader-client-ca-file argument\"\n # Add a new line at the end\n echo \"\" >> ${SNAP_DATA}/args/kube-apiserver\n echo \"--requestheader-client-ca-file=\\${SNAP_DATA}/certs/front-proxy-ca.crt\" >> ${SNAP_DATA}/args/kube-apiserver\n need_api_restart=true\nfi\n\n# Enable the aggregation layer (continue)\nif ! grep -E -- '--(requestheader-allowed-names|requestheader-extra-headers-prefix|requestheader-group-headers|requestheader-username-headers|proxy-client-cert-file|proxy-client-key-file)=' ${SNAP_DATA}/args/kube-apiserver\nthen\n echo \"Enabling Enable the aggregation layer\"\n echo \"\" >> ${SNAP_DATA}/args/kube-apiserver\n echo '--requestheader-allowed-names=front-proxy-client' >> ${SNAP_DATA}/args/kube-apiserver\n echo '--requestheader-extra-headers-prefix=X-Remote-Extra-' >> ${SNAP_DATA}/args/kube-apiserver\n echo '--requestheader-group-headers=X-Remote-Group' >> ${SNAP_DATA}/args/kube-apiserver\n echo '--requestheader-username-headers=X-Remote-User' >> ${SNAP_DATA}/args/kube-apiserver\n echo '--proxy-client-cert-file=${SNAP_DATA}/certs/front-proxy-client.crt' >> ${SNAP_DATA}/args/kube-apiserver\n echo '--proxy-client-key-file=${SNAP_DATA}/certs/front-proxy-client.key' >> ${SNAP_DATA}/args/kube-apiserver\n need_api_restart=true\nfi\n\n# Patch for issue: https://github.com/canonical/microk8s/issues/121\nif grep -e \"requestheader-client-ca-file=/var/snap/microk8s/.../certs/ca.crt\" ${SNAP_DATA}/args/kube-apiserver\nthen\n \"$SNAP/bin/sed\" -i 's@requestheader-client-ca-file=/var/snap/microk8s/.../certs/ca.crt@requestheader-client-ca-file=\\${SNAP_DATA}/certs/ca.crt@g' ${SNAP_DATA}/args/kube-apiserver\nfi\n\n# Patch for issue: https://github.com/canonical/microk8s/issues/721\nif grep -F 'requestheader-client-ca-file=${SNAP_DATA}/certs/ca.crt' ${SNAP_DATA}/args/kube-apiserver\nthen\n \"$SNAP/bin/sed\" -i 's@requestheader-client-ca-file=${SNAP_DATA}/certs/ca.crt@requestheader-client-ca-file=${SNAP_DATA}/certs/front-proxy-ca.crt@g' ${SNAP_DATA}/args/kube-apiserver\nfi\n\n# Create the locks directory\nmkdir -p ${SNAP_DATA}/var/lock/\n\n# Create tmp directory\nmkdir -p ${SNAP_DATA}/tmp/\n\n# This will allow us to refresh the snap to the more secure version.\n# We need to make sure the client certificate used in microk8s kubectl is available under $SNAP_DATA\n# TODO(neoaggelos): investigate whether this is needed\nif [ ! -f ${SNAP_DATA}/credentials/client.config ]\nthen\n echo \"Patching client config location\"\n mkdir -p ${SNAP_DATA}/credentials/\n cp ${SNAP}/client.config ${SNAP_DATA}/credentials/\nfi\n\n# copy kubectl-env\nif [ ! -e ${SNAP_DATA}/args/kubectl-env ] && grep -e \"\\-\\-kubeconfig=\\${SNAP_DATA}/credentials/client.config\" ${SNAP_DATA}/args/kubectl\nthen\n echo \"Making sure we have kubectl environment file\"\n cp ${SNAP}/default-args/kubectl-env ${SNAP_DATA}/args/kubectl-env\n skip_opt_in_config kubeconfig kubectl\nfi\n\n# copy kubectl\nif [ ! -e ${SNAP_DATA}/args/kubectl ]\nthen\n echo \"Making sure we have kubectl arguments file\"\n cp ${SNAP}/default-args/kubectl ${SNAP_DATA}/args/kubectl\nfi\n\n# copy traefik\nif [ ! -e ${SNAP_DATA}/args/traefik ]\nthen\n echo \"Making sure we have traefik configuration\"\n cp -r ${SNAP}/default-args/traefik ${SNAP_DATA}/args/\nfi\n\n# copy apiserver-proxy\nif [ ! -e ${SNAP_DATA}/args/apiserver-proxy ]\nthen\n echo \"Making sure we have apiserver-proxy configuration\"\n cp -r ${SNAP}/default-args/apiserver-proxy ${SNAP_DATA}/args/\nfi\n\n# (1.24 -> 1.25) migrate from traefik to apiserver-proxy\nif [ -e ${SNAP_DATA}/var/lock/no-traefik ]\nthen\n touch ${SNAP_DATA}/var/lock/no-apiserver-proxy\nfi\n\n# Upgrading to containerd\nif [ ! -e ${SNAP_DATA}/args/containerd ] ||\n grep -e \"\\-\\-docker unix://\\${SNAP_DATA}/docker.sock\" ${SNAP_DATA}/args/kubelet\nthen\n echo \"Making sure we have containerd file\"\n cp ${SNAP_DATA}/args/containerd ${SNAP_DATA}/args/containerd.backup || true\n cp ${SNAP}/default-args/containerd ${SNAP_DATA}/args/containerd\n cp ${SNAP_DATA}/args/containerd-template.toml ${SNAP_DATA}/args/containerd-template.toml.backup || true\n cp ${SNAP}/default-args/containerd-template.toml ${SNAP_DATA}/args/containerd-template.toml\n cp ${SNAP_DATA}/args/containerd-env ${SNAP_DATA}/args/containerd-env.backup || true\n cp ${SNAP}/default-args/containerd-env ${SNAP_DATA}/args/containerd-env\n\n cp -r ${SNAP}/default-args/cni-network ${SNAP_DATA}/args/\n\n cp ${SNAP}/default-args/ctr ${SNAP_DATA}/args/ctr\n\n refresh_opt_in_config container-runtime remote kubelet\n refresh_opt_in_config container-runtime-endpoint \\${SNAP_COMMON}/run/containerd.sock kubelet\n\n skip_opt_in_config docker-root kubelet\n skip_opt_in_config docker kubelet\n skip_opt_in_config docker-endpoint kubelet\n\n snapctl restart ${SNAP_NAME}.daemon-containerd\n need_kubelet_restart=true\n\n if [ -e ${SNAP_DATA}/args/dockerd ] && grep -e \"default-runtime=nvidia\" ${SNAP_DATA}/args/dockerd\n then\n # Deployment used to run docker with nvidia enabled we need to enable nvidia on containerd\n # Allow for kubelet and containerd to restart\n sleep 10\n ${SNAP}/microk8s-enable.wrapper gpu\n fi\n\nfi\n\n# Install default-hooks.\n#\n# Updated hooks may contain fixes, so we'll always replace existing files.\n# Custom hooks are expected to use separate files so that they won't get\n# overwritten during upgrades.\nmkdir -p ${SNAP_COMMON}/hooks\ncp -r --preserve=mode ${SNAP}/default-hooks/* ${SNAP_COMMON}/hooks/\n\n# Make sure the server certificate includes the IP we are using\nif [ \"$(produce_certs)\" == \"1\" ]\nthen\n rm -rf .srl\n need_api_restart=true\n need_proxy_restart=true\n need_cluster_agent_restart=true\nfi\n\n# Make containerd stream server listen to localhost\nif [ -e ${SNAP_DATA}/args/containerd-template.toml ] && grep -e \"stream_server_address = \\\"\\\"\" ${SNAP_DATA}/args/containerd-template.toml\nthen\n \"$SNAP/bin/sed\" -i 's@stream_server_address = \"\"@stream_server_address = \"127.0.0.1\"@g' ${SNAP_DATA}/args/containerd-template.toml\n if grep -e \"stream_server_port = \\\"10010\\\"\" ${SNAP_DATA}/args/containerd-template.toml\n then\n \"$SNAP/bin/sed\" -i 's@stream_server_port = \"10010\"@stream_server_port = \"0\"@g' ${SNAP_DATA}/args/containerd-template.toml\n fi\n snapctl restart ${SNAP_NAME}.daemon-containerd\n need_kubelet_restart=true\nfi\n\n# With v1.15 allow-privileged is removed from kubelet\nif grep -e \"\\-\\-allow-privileged\" ${SNAP_DATA}/args/kubelet\nthen\n echo \"Patching 1.15 allow-privileged\"\n \"${SNAP}/bin/sed\" -i '/allow-privileged/d' ${SNAP_DATA}/args/kubelet\n need_kubelet_restart=true\nfi\n\n# Add option to support kata containers\nif [ -e \"${SNAP_DATA}/args/containerd-env\" ] &&\n ! grep -e \"KATA_PATH\" ${SNAP_DATA}/args/containerd-env\nthen\n echo \"\" >> \"${SNAP_DATA}/args/containerd-env\"\n echo \"# You can set the of the kata containers runtime here.\" >> \"${SNAP_DATA}/args/containerd-env\"\n echo \"#\" >> \"${SNAP_DATA}/args/containerd-env\"\n echo \"# KATA_PATH=\" >> \"${SNAP_DATA}/args/containerd-env\"\n echo \"#\" >> \"${SNAP_DATA}/args/containerd-env\"\n echo \"PATH=\\$PATH:\\$KATA_PATH\" >> \"${SNAP_DATA}/args/containerd-env\"\nfi\n\n# Add option to support kata containers\nif [ -e \"${SNAP_DATA}/args/containerd-template.toml\" ] &&\n ! grep -e \"io.containerd.kata.v2\" ${SNAP_DATA}/args/containerd-template.toml\nthen\n KATA_HANDLER_BEFORE='\\[plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.nvidia-container-runtime\\]'\n KATA_HANDLER_AFTER=' [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.kata]\n runtime_type = \"io.containerd.kata.v2\"\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.kata.options]\n BinaryName = \"kata-runtime\"\n '\n CD_TOML=\"${SNAP_DATA}/args/containerd-template.toml\"\n CD_TOML_TMP=\"${SNAP_DATA}/args/containerd-template.toml.tmp\"\n\n \"$SNAP/usr/bin/gawk\" -v kata=\"${KATA_HANDLER_AFTER}\" '/'${KATA_HANDLER_BEFORE}'/{print kata }1' ${CD_TOML} > ${CD_TOML_TMP}\n mv ${CD_TOML_TMP} ${CD_TOML}\nfi\n\nfor dir in ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks\ndo\n chmod -R ug+rwX ${dir}\n chmod -R o-rwX ${dir}\ndone\n\ngroup=$(get_microk8s_or_cis_group)\n\n# Try to create the snap_microk8s group. Do not fail the installation if something goes wrong\nif ! getent group ${group} >/dev/null 2>&1\nthen\n groupadd ${group} || true\nfi\n\nif getent group ${group} >/dev/null 2>&1 && ! [ -e \"${SNAP_DATA}/var/lock/cis-hardening\" ]\nthen\n chgrp ${group} -R ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true\nfi\n\nif [ -e \"${SNAP_DATA}/var/lock/cis-hardening\" ]\nthen\n chmod -R g-wr ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true\n chmod -R o-wr ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true\n\n if ! is_strict && [ -e /etc/systemd/system/snap.microk8s.daemon-kubelite.service ]\n then\n chmod -R g-wr /etc/systemd/system/snap.microk8s.daemon-kubelite.service\n chmod -R o-wr /etc/systemd/system/snap.microk8s.daemon-kubelite.service\n fi\nfi\n\nif ! is_strict\nthen\n try_copy_users_to_snap_microk8s\nfi\n\n# as only one cni bin dir can be used we will use the one in SNAP_DATA but have links to\n# the real CNI plugins we distribute in SNAP\nmkdir -p \"${SNAP_DATA}/opt/cni/bin/\"\n(\n cd \"${SNAP}/opt/cni/bin/\"\n MY_SNAP_DIR=$(dirname \"${SNAP}\")\n for i in *; do ln -sf \"${MY_SNAP_DIR}/current/opt/cni/bin/$i\" \"${SNAP_DATA}/opt/cni/bin/${i}\"; done\n)\n\nif ! [ -e \"${SNAP_DATA}/opt/cni/bin/flanneld\" ]\nthen\n # cover situation where cilium was installed prior to this update\n if [ -f \"${SNAP_DATA}/opt/cni/bin/loopback\" ] && [ -f \"${SNAP}/opt/cni/bin/loopback\" ]; then\n rm -f \"${SNAP_DATA}/opt/cni/bin/loopback\"\n fi\nfi\n\nif ! [ -f \"${SNAP_DATA}/args/flanneld\" ]\nthen\n mkdir -p ${SNAP_DATA}/args/cni-network/\n cp -r ${SNAP}/default-args/cni-network/flannel.conflist ${SNAP_DATA}/args/cni-network/\n cp ${SNAP}/default-args/flanneld ${SNAP_DATA}/args/\n cp ${SNAP}/default-args/flannel-template.conflist ${SNAP_DATA}/args/\n cp ${SNAP}/default-args/flannel-network-mgr-config ${SNAP_DATA}/args/\n snapctl restart ${SNAP_NAME}.daemon-etcd\n snapctl restart ${SNAP_NAME}.daemon-containerd\n snapctl restart ${SNAP_NAME}.daemon-flanneld\nfi\n\nif grep -e \"etcd.socket:2379\" ${SNAP_DATA}/args/etcd\nthen\n echo \"Using a port for etcd\"\n # TODO: Do something smart in selecting a port\n refresh_opt_in_config advertise-client-urls https://\\${DEFAULT_INTERFACE_IP_ADDR}:12379 etcd\n refresh_opt_in_config listen-client-urls https://0.0.0.0:12379 etcd\n refresh_opt_in_config client-cert-auth true etcd\n refresh_opt_in_config trusted-ca-file \\${SNAP_DATA}/certs/ca.crt etcd\n refresh_opt_in_config cert-file \\${SNAP_DATA}/certs/server.crt etcd\n refresh_opt_in_config key-file \\${SNAP_DATA}/certs/server.key etcd\n snapctl restart ${SNAP_NAME}.daemon-etcd\n\n refresh_opt_in_config etcd-servers https://127.0.0.1:12379 kube-apiserver\n refresh_opt_in_config etcd-cafile \\${SNAP_DATA}/certs/ca.crt kube-apiserver\n refresh_opt_in_config etcd-certfile \\${SNAP_DATA}/certs/server.crt kube-apiserver\n refresh_opt_in_config etcd-keyfile \\${SNAP_DATA}/certs/server.key kube-apiserver\n need_api_restart=true\nfi\n\nif ! grep -e \"service-account-issuer\" ${SNAP_DATA}/args/kube-apiserver\nthen\n echo \"--service-account-issuer='https://kubernetes.default.svc'\" >> ${SNAP_DATA}/args/kube-apiserver\n need_api_restart=true\nfi\n\nif ! grep -e \"service-account-signing-key-file\" ${SNAP_DATA}/args/kube-apiserver\nthen\n echo '--service-account-signing-key-file=${SNAP_DATA}/certs/serviceaccount.key' >> ${SNAP_DATA}/args/kube-apiserver\n need_api_restart=true\nfi\n\n# RemoveSelfLink feature flag is removed after 1.24\nif grep -e \"feature-gates=RemoveSelfLink\" ${SNAP_DATA}/args/kube-apiserver\nthen\n \"${SNAP}/bin/sed\" -i '/feature-gates=RemoveSelfLink/d' \"$SNAP_DATA/args/kube-apiserver\"\n need_api_restart=true\nfi\n\nif remove_docker_specific_args\nthen\n need_kubelet_restart=true\nfi\n\n# scheduler --address flag is removed after 1.24\nif grep -e \"--address=\" ${SNAP_DATA}/args/kube-scheduler\nthen\n \"${SNAP}/bin/sed\" -i '/--address=/d' \"$SNAP_DATA/args/kube-scheduler\"\nfi\n\n# controller-manager --address flag is removed after 1.24\nif grep -e \"--address=\" ${SNAP_DATA}/args/kube-controller-manager\nthen\n \"${SNAP}/bin/sed\" -i '/--address=/d' \"$SNAP_DATA/args/kube-controller-manager\"\n need_api_restart=true\nfi\n\n# etcd --enable-v2 flag is removed after 3.6\nif grep -e \"--enable-v2=\" ${SNAP_DATA}/args/etcd\nthen\n \"${SNAP}/bin/sed\" -i '/--enable-v2=/d' \"$SNAP_DATA/args/etcd\"\n snapctl restart ${SNAP_NAME}.daemon-etcd\nfi\n\nif [ -e ${SNAP_DATA}/var/lock/clustered.lock ]\nthen\n if grep -e \"\\-\\-etcd-cafile /var/snap/microk8s/.*/ca.remote.crt\" ${SNAP_DATA}/args/flanneld\n then\n skip_opt_in_config etcd-cafile flanneld\n refresh_opt_in_config etcd-cafile \\${SNAP_DATA}/certs/ca.remote.crt flanneld\n fi\n\n if grep -e \"\\-\\-etcd-certfile /var/snap/microk8s/.*/server.remote.crt\" ${SNAP_DATA}/args/flanneld\n then\n skip_opt_in_config etcd-certfile flanneld\n refresh_opt_in_config etcd-certfile \\${SNAP_DATA}/certs/server.remote.crt flanneld\n fi\nfi\n\n# This patches flanneld conf template by adding cniversion if it does not exist.\nif [ -e ${SNAP_DATA}/args/flannel-template.conflist ] && ! grep -e \"cniVersion\" ${SNAP_DATA}/args/flannel-template.conflist\nthen\n \"$SNAP/bin/sed\" -i 's@\"name\": \"microk8s-flannel-network\",@\"name\": \"microk8s-flannel-network\",\\n \"cniVersion\": \"0.3.1\",@g' ${SNAP_DATA}/args/flannel-template.conflist\n snapctl restart ${SNAP_NAME}.daemon-flanneld\n snapctl restart ${SNAP_NAME}.daemon-containerd\nfi\n\nif [ ! -f ${SNAP_DATA}/args/cluster-agent ]\nthen\n cp ${SNAP}/default-args/cluster-agent ${SNAP_DATA}/args/cluster-agent\nfi\n\nif ! grep -e \"\\-\\-timeout\" ${SNAP_DATA}/args/cluster-agent\nthen\n refresh_opt_in_config timeout 240 cluster-agent\n snapctl restart ${SNAP_NAME}.daemon-containerd\nfi\n\nif ! grep -e \"\\-\\-ip-masq\" ${SNAP_DATA}/args/flanneld\nthen\n refresh_opt_in_config ip-masq true flanneld\n snapctl restart ${SNAP_NAME}.daemon-flanneld\nfi\n\nif grep -e \"\\-\\-cluster-cidr=10.152.183.0/24\" ${SNAP_DATA}/args/kube-proxy\nthen\n refresh_opt_in_config cluster-cidr 10.1.0.0/16 kube-proxy\n need_proxy_restart=true\nfi\n\nif [ -e ${SNAP_DATA}/var/lock/stopped.lock ]\nthen\n snapctl stop ${SNAP_NAME}.daemon-kubelite\nfi\n\n# Enable kubelite\nif ! [ -e ${SNAP_DATA}/var/lock/lite.lock ]\nthen\n touch \"${SNAP_DATA}/var/lock/lite.lock\"\n if ! [ -e ${SNAP_DATA}/args/kubelite ]\n then\n cp ${SNAP}/default-args/kubelite ${SNAP_DATA}/args/kubelite\n fi\n date\n if [ -e ${SNAP_DATA}/var/lock/stopped.lock ]\n then\n snapctl stop ${SNAP_NAME}.daemon-kubelite\n else\n snapctl start ${SNAP_NAME}.daemon-kubelite\n fi\nfi\n\nif ! [ -e ${SNAP_DATA}/args/kubelite ]\nthen\n cp ${SNAP}/default-args/kubelite ${SNAP_DATA}/args/kubelite\n need_api_restart=true\nfi\n\n# Removed --insecure-port argument\nif grep -e \"\\-\\-insecure\\-port\" ${SNAP_DATA}/args/kube-apiserver\nthen\n $SNAP/bin/sed -i '/\\-\\-insecure\\-port/d' ${SNAP_DATA}/args/kube-apiserver\n need_api_restart=true\nfi\n\n# Are we using etcd or some other non-dqlite datastore?\nif ! [ -e ${SNAP_DATA}/args/k8s-dqlite ] &&\n ! grep -e \"\\-\\-storage-backend=dqlite\" ${SNAP_DATA}/args/kube-apiserver\nthen\n set_service_not_expected_to_start k8s-dqlite\n snapctl stop ${SNAP_NAME}.daemon-k8s-dqlite\nfi\n\n# Configure the API sever to talk to the external dqlite\nif [ -e ${SNAP}/default-args/k8s-dqlite ] &&\n ! [ -e ${SNAP_DATA}/args/k8s-dqlite ] &&\n grep -e \"\\-\\-storage-backend=dqlite\" ${SNAP_DATA}/args/kube-apiserver\nthen\n echo \"Reconfiguring the API server for dqlite\"\n cp ${SNAP}/default-args/k8s-dqlite ${SNAP_DATA}/args/k8s-dqlite\n cp ${SNAP}/default-args/k8s-dqlite-env ${SNAP_DATA}/args/k8s-dqlite-env\n need_api_restart=true\n snapctl stop ${SNAP_NAME}.daemon-kubelite\n\n refresh_opt_in_local_config etcd-servers unix://\\${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379 kube-apiserver\n $SNAP/bin/sed -i '/\\-\\-storage\\-backend=dqlite/d' ${SNAP_DATA}/args/kube-apiserver\n\n storage_dir=\"$(get_opt_in_config '--storage-dir' 'kube-apiserver')\"\n if ! [ -z $storage_dir ]\n then\n refresh_opt_in_local_config storage-dir \"$storage_dir\" k8s-dqlite\n fi\n $SNAP/bin/sed -i '/\\-\\-storage\\-dir/d' ${SNAP_DATA}/args/kube-apiserver\n\n snapctl restart ${SNAP_NAME}.daemon-k8s-dqlite\nfi\n\n# Disable the features misbehaving with dqlite\nif grep -e '^\\-\\-etcd\\-servers=.*kine.sock:12379' ${SNAP_DATA}/args/kube-apiserver\nthen\n if grep -e '^\\-\\-feature\\-gates' ${SNAP_DATA}/args/kube-apiserver\n then\n for feature in ListFromCacheSnapshot SizeBasedListCostEstimate DetectCacheInconsistency\n do\n if ! grep -e \"$feature\" \"$SNAP_DATA/args/kube-apiserver\"\n then\n \"${SNAP}/bin/sed\" -i '/^--feature-gates=/ s/$/,'$feature'=false/' \"$SNAP_DATA/args/kube-apiserver\"\n fi\n done\n else\n echo '--feature-gates=ListFromCacheSnapshot=false,SizeBasedListCostEstimate=false,DetectCacheInconsistency=false' >> ${SNAP_DATA}/args/kube-apiserver\n fi\n need_api_restart=true\nfi\n\n# Fix hard-coded snap revision numbers in worker node services of existing clusters\n# https://github.com/canonical/microk8s/pull/3554\nfor svc in kubelet proxy; do\n cfg=\"${SNAP_DATA}/credentials/${svc}.config\"\n if [ -e ${cfg} ]; then\n sed -i 's,/var/snap/microk8s/[x0-9]*/,/var/snap/microk8s/current/,' \"${cfg}\" || true\n fi\ndone\n\n# (1.26) Removed --log-dir argument from kubelet\nif grep -e \"\\-\\-log\\-dir\" \"${SNAP_DATA}/args/kubelet\"\nthen\n \"${SNAP}/bin/sed\" -i '/\\-\\-log\\-dir/d' \"${SNAP_DATA}/args/kubelet\"\n need_api_restart=true\nfi\n\n# Refresh calico if needed\nrefresh_calico_if_needed\n\n# Refresh apiserver proxy\nsnapctl restart \"${SNAP_NAME}.daemon-apiserver-proxy\"\n\nif is_strict\nthen\n enable_snap\nfi\n\n# if we are refreshing in a no-flanneld we need to restart the CNI pods because they mount parts of $SNAP_DATA\nif [ -e \"${SNAP_DATA}/var/lock/no-flanneld\" ]\nthen\n touch \"${SNAP_DATA}/var/lock/snapdata-mounts-need-reload\"\nfi\n\n# Restart reconfigured services\nif ${need_api_restart} ||\n ${need_proxy_restart} ||\n ${need_controller_restart} ||\n ${need_kubelet_restart}\nthen\n if [ -e ${SNAP_DATA}/var/lock/lite.lock ]\n then\n snapctl restart ${SNAP_NAME}.daemon-kubelite\n else\n echo \"Unable to restart service\"\n exit 1\n fi\nfi\nif ${need_cluster_agent_restart}\nthen\n snapctl restart ${SNAP_NAME}.daemon-cluster-agent\nfi\n" }, { "path": "snap/hooks/connect-plug-network-control", "content": "#!/usr/bin/env bash\nset -eu\n\nfor link in cni0 cilium_vxlan\ndo\n if $SNAP/sbin/ip link show ${link}\n then\n $SNAP/sbin/ip link delete ${link}\n fi\ndone\n\n${SNAP}/meta/hooks/configure\n" }, { "path": "snap/hooks/disconnect-plug-network-control", "content": "#!/usr/bin/env bash\nset -eu\n\nfor link in cni0 cilium_vxlan\ndo\n if $SNAP/sbin/ip link show ${link}\n then\n $SNAP/sbin/ip link delete ${link}\n fi\ndone\n" }, { "path": "snap/hooks/install", "content": "#!/usr/bin/env bash\n\nset -eux\n\nsource $SNAP/actions/common/utils.sh\n\nif [ -f \"${SNAP_DATA}/var/lock/installed.lock\" ]\nthen\n exit 0\nfi\n\nuse_snap_env\n\n# LXD Specific Checks\nif ! is_strict && cat /proc/1/environ | grep \"container=lxc\" &> /dev/null\n then\n\n # make sure the /dev/kmsg is available, indicating a potential missing profile\n if [ ! -c \"/dev/kmsg\" ] # kmsg is a character device\n then\n printf -- '\\e[1;31mERROR: \\033[0m the lxc profile for MicroK8s might be missing. \\n'\n printf -- '\\t Refer to this help document to get MicroK8s working in with LXD: \\n'\n printf -- '\\t https://microk8s.io/docs/lxd \\n'\n exit 1\n fi\nfi\n\ncp -r --preserve=mode ${SNAP}/default-args ${SNAP_DATA}/args\nmv ${SNAP_DATA}/args/certs.d/localhost__32000 ${SNAP_DATA}/args/certs.d/localhost:32000\n\nSNAP_DATA_CURRENT=`echo \"${SNAP_DATA}\" | sed -e \"s,${SNAP_REVISION},current,\"`\n\n# Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box.\nif ! [ -e /var/lib/kubelet ] && ln -s $SNAP_COMMON/var/lib/kubelet /var/lib/kubelet; then\n echo \"/var/lib/kubelet linked to $SNAP_COMMON\"\nfi\n\n# Try to symlink /var/lib/calico so that the Calico CNI plugin picks up the mtu configuration.\nif ! [ -e /var/lib/calico ]; then\n if ln -s $SNAP_DATA_CURRENT/var/lib/calico /var/lib/calico; then\n echo \"/var/lib/calico linked to $SNAP_DATA_CURRENT/var/lib/calico\"\n fi\nfi\n\n# Try to symlink standard CNI Kubernetes directories.\nif ! [ -e /etc/cni/net.d ]; then\n if mkdir -p /etc/cni && ln -s $SNAP_DATA_CURRENT/args/cni-network /etc/cni/net.d; then\n echo \"/etc/cni/net.d linked to $SNAP_DATA_CURRENT/args/cni-network\"\n fi\nfi\nif ! [ -e /opt/cni/bin ]; then\n if mkdir -p /opt/cni && ln -s $SNAP_DATA_CURRENT/opt/cni/bin /opt/cni/bin; then\n echo \"/opt/cni/bin linked to $SNAP_DATA_CURRENT/opt/cni/bin\"\n fi\nfi\n\n# Create the credentials directory\nmkdir -p ${SNAP_DATA}/credentials\n\n# Create the certificates\nmkdir ${SNAP_DATA}/certs\n# Allow the ability to add external IPs to the csr, by moving the csr.conf.template to SNAP_DATA\ncp ${SNAP}/certs/csr.conf.template ${SNAP_DATA}/certs/csr.conf.template\n\n# Copy initial configuration from well-known paths.\n# This is done prior to any other initialization.\n#\n# The following paths are checked in order for a config file.\n# Only the first file found will be used, the rest will be ignored.\n#\n# - /etc/microk8s.yaml <-- user-defined config file\n# - $SNAP_USER_COMMON/.microk8s.yaml <-- user-defined config file, typically in /root/snap/microk8s/common/.microk8s.yaml\n# - $SNAP_COMMON/.microk8s.yaml <-- user-defined config file, typically in /var/snap/microk8s/common/.microk8s.yaml\n# - $SNAP/microk8s.default.yaml <-- default config file bundled with the snap\n#\n# If the pre-init step fails, the snap installation will fail, as it is considered to be a user error.\nmkdir -p \"${SNAP_COMMON}/etc/launcher\"\nfor config_file in \"/etc/microk8s.yaml\" \"$SNAP_USER_COMMON/.microk8s.yaml\" \"$SNAP_COMMON/.microk8s.yaml\" \"$SNAP/microk8s.default.yaml\"; do\n if [ -f \"${config_file}\" ]; then\n echo \"Found config file ${config_file}, will use to initialize cluster.\"\n\n if cp \"${config_file}\" \"${SNAP_COMMON}/etc/launcher/install.yaml\"; then\n \"${SNAP}/bin/cluster-agent\" init --pre-init --config-file \"${SNAP_COMMON}/etc/launcher/install.yaml\"\n break\n fi\n fi\ndone\n\n# Side-load images from well-known paths.\n# This is done prior to any other initialization.\n#\n# Any *.tar files that are found in these directories will be loaded into containerd after it starts.\n#\n# - $SNAP_USER_COMMON/sideload/*.tar <-- typically /root/snap/microk8s/common/sideload/images.tar\n# - $SNAP_COMMON/sideload/*.tar <-- typically /var/snap/microk8s/common/sideload/images.tar\n# - $SNAP/sideload/*.tar <-- typically empty, reserved for future use-cases\nmkdir -p \"${SNAP_COMMON}/etc/sideload\"\nfor source_dir in \"${SNAP_USER_COMMON}\" \"${SNAP_COMMON}\" \"${SNAP}\"; do\n cp \"${source_dir}/sideload/\"*.tar \"${SNAP_COMMON}/etc/sideload/\" || true\ndone\n\n# Produce cluster certificates\nproduce_certs\nrm -rf .srl\n\ncreate_user_certs_and_configs\n\n# Install default-hooks\ncp -r --preserve=mode ${SNAP}/default-hooks ${SNAP_COMMON}/hooks\n\nfor dir in ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_COMMON}/hooks\ndo\n chmod -R ug+rwX ${dir}\n chmod -R o-rwX ${dir}\ndone\n\nif is_strict && snapctl is-connected k8s-kubelet\nthen\n snapctl restart microk8s.daemon-containerd\nfi\n\ninit_cluster\n\nmkdir -p \"${SNAP_DATA}/var/lock\"\nset_service_not_expected_to_start etcd\nset_service_not_expected_to_start flanneld\nset_service_not_expected_to_start apiserver-proxy\n\ntouch \"${SNAP_DATA}/var/lock/ha-cluster\"\ntouch \"${SNAP_DATA}/var/lock/lite.lock\"\n\nRESOURCES=\"${SNAP}/upgrade-scripts/000-switch-to-calico/resources\"\nBACKUP_DIR=\"${SNAP_DATA}/var/tmp/upgrades/000-switch-to-calico\"\n\nmkdir -p \"${BACKUP_DIR}\"\n\nmkdir -p \"${BACKUP_DIR}/args/cni-network/\"\ncp \"${SNAP_DATA}\"/args/cni-network/* \"${BACKUP_DIR}/args/cni-network/\" 2>/dev/null || true\nrm -rf \"${SNAP_DATA}\"/args/cni-network/*\n${SNAP}/scripts/generate-cni.sh\nmkdir -p \"$SNAP_DATA/opt/cni/bin/\"\ncp -R \"$SNAP\"/opt/cni/bin/* \"$SNAP_DATA\"/opt/cni/bin/\n\n# Low memory guard. Enable by default when system RAM is less than 512MB.\nMEMORY=`cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'`\nif [ $MEMORY -le 524288 ]\nthen\n touch ${SNAP_DATA}/var/lock/low-memory-guard.lock\nfi\n\n# copy git config\nmkdir -p ${SNAP_DATA}/args/git\ncp ${SNAP}/default-args/git/.gitconfig ${SNAP_DATA}/args/git/.gitconfig\n\ntouch \"${SNAP_DATA}/var/lock/installed.lock\"\n" }, { "path": "snap/hooks/post-refresh", "content": "#!/usr/bin/env bash\n\n. $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\n# Run post-refresh hooks\n$SNAP/usr/bin/python3 $SNAP/scripts/run-lifecycle-hooks.py post-refresh || true\n" }, { "path": "snap/hooks/remove", "content": "#!/usr/bin/env bash\nset -eu\n\nsource $SNAP/actions/common/utils.sh\n\nuse_snap_env\n\nsnapctl stop ${SNAP_NAME}.daemon-kubelite 2>&1 || true\n\n# Temporarily start containerd so we can stop and kill all the microk8s containers.\nsnapctl start ${SNAP_NAME}.daemon-containerd 2>&1 || true\n# wait for containerd to start.\nsleep 5\n\n# Remove any lingering containers and shims.\nremove_all_containers\nkill_all_container_shims\n\n# Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box.\nif test -L /var/lib/kubelet; then\n unlink /var/lib/kubelet || true\nfi\n\n# Try to symlink /var/lib/calico so that the Calico CNI plugin picks up the mtu configuration.\nif test -L /var/lib/calico; then\n unlink /var/lib/calico || true\nfi\n\n# Try to symlink standard CNI Kubernetes directories.\nif test -L /etc/cni/net.d; then\n unlink /etc/cni/net.d || true\nfi\nif test -L /opt/cni/bin; then\n unlink /opt/cni/bin || true\nfi\n\npod_cidr=\"$(cat $SNAP_DATA/args/kube-proxy | grep \"cluster-cidr\" | tr \"=\" \" \"| gawk '{print $2}')\"\nif [ -z \"$pod_cidr\" ]\nthen\n pod_cidr=\"$(cat $SNAP_DATA/args/kubelet | grep \"pod-cidr\" | tr \"=\" \" \"| gawk '{print $2}')\"\n if [ -z \"$pod_cidr\" ]\n then\n pod_cidr=\"$(jq .Network $SNAP_DATA/args/flannel-network-mgr-config | tr -d '\\\"')\"\n fi\nfi\nif ! [ -z \"$pod_cidr\" ]\nthen\n iptables -D FORWARD -s \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT || true\n iptables -D FORWARD -d \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT || true\n iptables-nft -D FORWARD -s \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT || true\n iptables-nft -D FORWARD -d \"$pod_cidr\" -m comment --comment \"generated for MicroK8s pods\" -j ACCEPT || true\n iptables-legacy -t nat -F CNI-HOSTPORT-DNAT || true\nfi\n\nsnapctl stop ${SNAP_NAME}.daemon-containerd 2>&1 || true\n# wait for containerd to stop its processes or we will be getting a umount error\n# because the mount points are busy\nsleep 10\n\nif ! is_strict\nthen\n # remove custom sysctl parameters\n rm -f /etc/sysctl.d/10-microk8s.conf\n sysctl --system\n\n rm -rf /etc/systemd/system/snap.microk8s.daemon-kubelite.service.d || true\nfi\n\n# Clean the container location so we do not snapshot it.\nrm -rf ${SNAP_COMMON}/var/lib/containerd/* || true\nrm -rf ${SNAP_COMMON}/run/containerd/* || true\n\n(cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/kubelet/pods | cut -d ' ' -f 2 | xargs umount -l) || true\n# in case this is a pre root-dir fix deployment\n(cat /proc/mounts | grep ${SNAP_COMMON}/pods | cut -d ' ' -f 2 | xargs umount -l) || true\n(cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/containerd | cut -d ' ' -f 2 | xargs umount -l) || true\n(cat /proc/mounts | grep ${SNAP_COMMON}/run/containerd | cut -d ' ' -f 2 | xargs umount) || true\n(cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/docker | cut -d ' ' -f 2 | xargs umount -l) || true\n(cat /proc/mounts | grep ${SNAP_COMMON}/var/run/docker | cut -d ' ' -f 2 | xargs umount) || true\n(cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/kubelet | cut -d ' ' -f 2 | xargs umount) || true\n\n# Run remove hooks\n$SNAP/usr/bin/python3 $SNAP/scripts/run-lifecycle-hooks.py remove || true\n" }, { "path": "snap/snapcraft.yaml", "content": "name: microk8s\nadopt-info: kubernetes-version\nsummary: Kubernetes for workstations and appliances\ndescription: |-\n MicroK8s is a small, fast, secure, single node Kubernetes that installs on\n just about any Linux box. Use it for offline development, prototyping,\n testing, or use it on a VM as a small, cheap, reliable k8s for CI/CD. It's\n also a great k8s for appliances - develop your IoT apps for k8s and deploy\n them to MicroK8s on your boxes.\nlicense: Apache-2.0\ngrade: stable\nconfinement: classic\nbase: core22\nassumes: [snapd2.52]\nenvironment:\n REAL_PATH: $PATH\n REAL_LD_LIBRARY_PATH: $LD_LIBRARY_PATH\n REAL_PYTHONPATH: $PYTHONPATH\n SNAPCRAFT_ARCH_TRIPLET: $CRAFT_ARCH_TRIPLET\n\nparts:\n build-deps:\n plugin: nil\n build-snaps:\n - go/1.24/stable\n build-attributes: [enable-patchelf]\n build-packages:\n - sudo\n - autoconf\n - automake\n - autopoint\n - autotools-dev\n - bison\n - btrfs-progs\n - libbtrfs-dev\n - build-essential\n - curl\n - flex\n - git\n - libjansson-dev\n - liblz4-dev\n - libnetfilter-conntrack-dev\n - libnetfilter-conntrack3\n - libnfnetlink-dev\n - libseccomp-dev\n - libtool\n - libuv1-dev\n - pkg-config\n - rsync\n - tcl\n\n k8s-dqlite:\n after: [build-deps]\n source: build-scripts/components/k8s-dqlite\n build-attributes: [enable-patchelf]\n plugin: nil\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh k8s-dqlite\n\n etcd:\n after: [build-deps]\n plugin: nil\n source: build-scripts/components/etcd\n build-attributes: [enable-patchelf]\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh etcd\n\n cni:\n after: [build-deps]\n plugin: nil\n source: build-scripts/components/cni\n build-attributes: [enable-patchelf]\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh cni\n\n flannel-cni-plugin:\n after: [build-deps]\n plugin: nil\n source: build-scripts/components/flannel-cni-plugin\n build-attributes: [enable-patchelf]\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh flannel-cni-plugin\n\n flanneld:\n after: [build-deps]\n plugin: nil\n source: build-scripts/components/flanneld\n build-attributes: [enable-patchelf]\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh flanneld\n\n kubernetes:\n after: [build-deps]\n plugin: nil\n source: build-scripts/components/kubernetes\n build-attributes: [enable-patchelf]\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh kubernetes\n\n kubernetes-version:\n plugin: nil\n source: build-scripts/components/kubernetes\n override-build: snapcraftctl set-version \"$(./version.sh)\"\n\n helm:\n after: [build-deps]\n plugin: nil\n source: build-scripts/components/helm\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh helm\n\n libmnl:\n after: [build-deps]\n plugin: autotools\n source: https://www.netfilter.org/pub/libmnl/libmnl-1.0.5.tar.bz2\n build-attributes: [enable-patchelf]\n prime:\n - -usr/local/include\n\n libnftnl:\n after: [libmnl]\n plugin: autotools\n source: https://www.netfilter.org/projects/libnftnl/files/libnftnl-1.1.8.tar.bz2\n build-attributes: [enable-patchelf]\n build-environment:\n - LIBMNL_LIBS: $CRAFT_STAGE/usr/lib\n prime:\n - -usr/local/include\n\n iptables:\n after: [libnftnl]\n source: https://www.netfilter.org/projects/iptables/files/iptables-1.8.6.tar.bz2\n build-attributes: [enable-patchelf]\n plugin: autotools\n build-environment:\n - LIBMNL_LIBS: $CRAFT_STAGE/usr/lib\n - LIBNFTNL_LIBS: $CRAFT_STAGE/usr/lib\n autotools-configure-parameters:\n - \"--prefix=/usr\"\n - \"--exec-prefix=/\"\n - \"--disable-shared\"\n - \"--enable-static\"\n stage:\n - -usr\n - -lib/pkgconfig\n - -bin/iptables-xml\n\n containerd:\n after: [runc]\n plugin: nil\n source: build-scripts/components/containerd\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh containerd\n build-attributes: [no-patchelf]\n\n runc:\n after: [iptables, build-deps]\n source: build-scripts/components/runc\n build-attributes: [no-patchelf]\n plugin: nil\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh runc\n\n bash-utils:\n plugin: nil\n build-attributes: [enable-patchelf]\n stage-packages:\n - conntrack\n - coreutils\n - curl\n - diffutils\n - ethtool\n - gawk\n - git\n - grep\n - hostname\n - iproute2\n - jq\n - kmod\n - libatm1\n - libnss-resolve\n - libnss-myhostname\n - libnss-mymachines\n - members\n - nano\n - net-tools\n - openssl\n - procps\n - python3-rados\n - python3-rbd\n - sed\n - tar\n - ufw\n - util-linux\n - zfsutils-linux\n - erofs-utils\n stage:\n - -etc/bash_completion.d\n - -etc/cron.d\n - -etc/depmod.d\n - -etc/ldap\n - -etc/logrotate.d\n - -etc/init.d\n - -etc/perl\n - -etc/rsyslog.d\n - -etc/sudoers.d\n - -lib/systemd/system\n - -usr/bin/perl*\n - -usr/include\n - -usr/lib/*/*perl*\n - -usr/share/bash-completion\n - -usr/share/doc\n - -usr/share/doc-base\n - -usr/share/info\n - -usr/share/initramfs-tools\n - -usr/share/lintian\n - -usr/share/man\n - -usr/share/nano\n - -usr/share/perl\n - -usr/share/perl5\n - -usr/share/zsh\n\n cluster-agent:\n after: [build-deps]\n plugin: nil\n source: build-scripts/components/cluster-agent\n build-attributes: [enable-patchelf]\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh cluster-agent\n\n microk8s-addons:\n plugin: nil\n source: build-scripts/addons\n override-build: ./repositories.sh \"${CRAFT_PART_INSTALL}\"\n\n microk8s-scripts:\n plugin: nil\n source: scripts/\n override-build: |\n if [ -d \"${CRAFT_PART_INSTALL}\" ]; then\n rm -rf \"${CRAFT_PART_INSTALL}/*\"\n fi\n\n cp -r . \"${CRAFT_PART_INSTALL}/scripts\"\n cp inspect.sh \"${CRAFT_PART_INSTALL}/inspect.sh\"\n\n microk8s-upgrade-scripts:\n plugin: nil\n source: upgrade-scripts/\n override-build: |\n if [ -d \"${CRAFT_PART_INSTALL}\" ]; then\n rm -rf \"${CRAFT_PART_INSTALL}/*\"\n fi\n\n cp -r . \"${CRAFT_PART_INSTALL}/upgrade-scripts\"\n\n microk8s:\n plugin: nil\n source: microk8s-resources/\n override-build: |\n if [ -d \"${CRAFT_PART_INSTALL}\" ]; then\n rm -rf \"${CRAFT_PART_INSTALL}/*\"\n fi\n\n cp microk8s.default.yaml \"${CRAFT_PART_INSTALL}/microk8s.default.yaml\"\n\n cp -r default-args \"${CRAFT_PART_INSTALL}/default-args\"\n cp -r default-hooks \"${CRAFT_PART_INSTALL}/default-hooks\"\n cp -r certs \"${CRAFT_PART_INSTALL}/certs\"\n\n cp containerd-profile \"${CRAFT_PART_INSTALL}/containerd-profile\"\n\n cp client.config \"${CRAFT_PART_INSTALL}/client.config\"\n cp client.config.template \"${CRAFT_PART_INSTALL}/client.config.template\"\n cp kubelet.config.template \"${CRAFT_PART_INSTALL}/kubelet.config.template\"\n cp client-x509.config.template \"${CRAFT_PART_INSTALL}/client-x509.config.template\"\n\n cp -r wrappers/* \"${CRAFT_PART_INSTALL}/\"\n cp -r actions/ \"${CRAFT_PART_INSTALL}/actions\"\n\n microk8s-completion:\n after: [build-deps]\n plugin: nil\n source: build-scripts/components/microk8s-completion\n override-build: $CRAFT_PROJECT_DIR/build-scripts/build-component.sh microk8s-completion\n\n python-runtime:\n after: [build-deps]\n build-attributes: [enable-patchelf]\n plugin: nil\n source: build-scripts/components/python\n override-build: |\n pip3 install -r requirements.txt\n build-packages:\n - python3-dev\n build-environment:\n - C_INCLUDE_PATH: /usr/include/python3.10\n stage-packages:\n - libpython3-stdlib\n - libpython3.10-stdlib\n - libpython3.10-minimal\n - python3-pip\n - python3-setuptools\n - python3-wheel\n - python3-venv\n - python3-minimal\n - python3-distutils\n - python3-pkg-resources\n - python3.10-minimal\n - python3-openssl\n - python3-requests\n - python3-click\n - python3-dateutil\n stage:\n - -usr/lib/python3.11\n - -usr/share/doc\n - -usr/share/lintian\n - -usr/share/man\n - -usr/share/python-wheels\n\n bom:\n after:\n - cluster-agent\n - cni\n - containerd\n - etcd\n - flannel-cni-plugin\n - flanneld\n - helm\n - k8s-dqlite\n - kubernetes\n - microk8s-addons\n - python-runtime\n - runc\n plugin: nil\n source: .\n build-packages:\n - python3-yaml\n override-build: |\n ./build-scripts/generate-bom.py > \"${CRAFT_PART_INSTALL}/bom.json\"\n\napps:\n microk8s:\n command: microk8s.wrapper\n completer: microk8s.bash\n daemon-etcd:\n command: run-etcd-with-args\n daemon: simple\n daemon-flanneld:\n command: run-flanneld-with-args\n daemon: simple\n daemon-containerd:\n command: run-containerd-with-args\n daemon: notify\n # when stopped send only sigterm\n # https://forum.snapcraft.io/t/process-lifecycle-on-snap-refresh/140/37\n stop-mode: sigterm\n restart-condition: always\n start-timeout: 5m\n daemon-kubelite:\n command: run-kubelite-with-args\n daemon: simple\n after: [daemon-containerd]\n daemon-apiserver-kicker:\n command: apiservice-kicker\n daemon: simple\n daemon-apiserver-proxy:\n command: run-apiserver-proxy-with-args\n daemon: simple\n daemon-cluster-agent:\n command: run-cluster-agent-with-args\n daemon: simple\n daemon-k8s-dqlite:\n command: run-k8s-dqlite-with-args\n daemon: simple\n dashboard-proxy:\n command: microk8s-dashboard-proxy.wrapper\n kubectl:\n command: microk8s-kubectl.wrapper\n completer: kubectl.bash\n add-node:\n command: microk8s-add-node.wrapper\n addons:\n command: microk8s-addons.wrapper\n refresh-certs:\n command: microk8s-refresh-certs.wrapper\n images:\n command: microk8s-images.wrapper\n join:\n command: microk8s-join.wrapper\n remove-node:\n command: microk8s-remove-node.wrapper\n leave:\n command: microk8s-leave.wrapper\n ctr:\n command: microk8s-ctr.wrapper\n inspect:\n command: microk8s.wrapper inspect\n enable:\n command: microk8s-enable.wrapper\n disable:\n command: microk8s-disable.wrapper\n start:\n command: microk8s-start.wrapper\n stop:\n command: microk8s-stop.wrapper\n status:\n command: microk8s-status.wrapper\n config:\n command: microk8s-config.wrapper\n reset:\n command: microk8s-reset.wrapper\n istioctl:\n command: microk8s-istioctl.wrapper\n linkerd:\n command: microk8s-linkerd.wrapper\n helm:\n command: microk8s-helm.wrapper\n completer: helm.bash\n helm3:\n command: microk8s-helm3.wrapper\n completer: helm3.bash\n dbctl:\n command: microk8s-dbctl.wrapper\n version:\n command: microk8s-version.wrapper\n" }, { "path": "tests/libs/addons-upgrade.sh", "content": "#!/usr/bin/env bash\n\nfunction setup_addons_upgrade_tests() {\n local NAME=$1\n local DISTRO=$2\n local PROXY=$3\n create_machine \"$NAME\" \"$DISTRO\" \"$PROXY\"\n}\n\nfunction run_addons_upgrade_tests() {\n local NAME=$1\n local FROM_CHANNEL=$2\n local TO_CHANNEL=$3\n # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774\n lxc exec \"$NAME\" -- script -e -c \"UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=${TO_CHANNEL} pytest -s /root/tests/test-upgrade.py\"\n}\n\nfunction post_addons_upgrade_tests() {\n local NAME=$1\n lxc delete \"$NAME\" --force\n}\n\nTEMP=$(getopt -o \"lh\" \\\n --long lib-mode,help,node-name:,distro:,from-channel:,to-channel:,proxy: \\\n -n \"$(basename \"$0\")\" -- \"$@\")\n\nif [ $? != 0 ] ; then echo \"Terminating...\" >&2 ; exit 1 ; fi\n\neval set -- \"$TEMP\"\n\nNAME=\"${NAME-\"machine-$RANDOM\"}\"\nDISTRO=\"${DISTRO-}\"\nFROM_CHANNEL=\"${FROM_CHANNEL-}\"\nTO_CHANNEL=\"${TO_CHANNEL-}\"\nPROXY=\"${PROXY-}\"\nLIBRARY_MODE=false\n\nwhile true; do\n case \"$1\" in\n -l | --lib-mode ) LIBRARY_MODE=true; shift ;;\n --node-name ) NAME=\"$2\"; shift 2 ;;\n --distro ) DISTRO=\"$2\"; shift 2 ;;\n --from-channel ) FROM_CHANNEL=\"$2\"; shift 2 ;;\n --to-channel ) TO_CHANNEL=\"$2\"; shift 2 ;;\n --proxy ) PROXY=\"$2\"; shift 2 ;;\n -h | --help )\n prog=$(basename -s.wrapper \"$0\")\n echo \"Usage: $prog [options...]\"\n echo \" --node-name Name to be used for LXD containers\"\n echo \" Can also be set by using NAME environment variable\"\n echo \" --distro Distro image to be used for LXD containers Eg. ubuntu:18.04\"\n echo \" Can also be set by using DISTRO environment variable\"\n echo \" --from-channel Channel to upgrade from to the channel under testing Eg. latest/beta\"\n echo \" Can also be set by using FROM_CHANNEL environment variable\"\n echo \" --to-channel Channel to be tested Eg. latest/edge\"\n echo \" Can also be set by using TO_CHANNEL environment variable\"\n echo \" --proxy Proxy url to be used by the nodes\"\n echo \" Can also be set by using PROXY environment variable\"\n echo \" -l, --lib-mode Make the script act like a library Eg. true / false\"\n echo\n exit ;;\n -- ) shift; break ;;\n * ) break ;;\n esac\ndone\n\nif [ \"$LIBRARY_MODE\" == \"false\" ];\nthen\n setup_addons_upgrade_tests \"$NAME\" \"$DISTRO\" \"$PROXY\"\n run_addons_upgrade_tests \"$NAME\" \"$FROM_CHANNEL\" \"$TO_CHANNEL\"\n post_addons_upgrade_tests \"$NAME\"\nfi\n" }, { "path": "tests/libs/addons.sh", "content": "#!/usr/bin/env bash\n\nfunction setup_addons_tests() {\n local NAME=$1\n local DISTRO=$2\n local PROXY=$3\n local TO_CHANNEL=$4\n\n create_machine \"$NAME\" \"$DISTRO\" \"$PROXY\"\n if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]]\n then\n lxc file push \"${TO_CHANNEL}\" \"$NAME\"/tmp/microk8s_latest_amd64.snap\n lxc exec \"$NAME\" -- snap install /tmp/microk8s_latest_amd64.snap --dangerous --classic\n else\n lxc exec \"$NAME\" -- snap install microk8s --channel=\"${TO_CHANNEL}\" --classic\n fi\n}\n\nfunction run_smoke_test() {\n local NAME=$1\n lxc exec \"$NAME\" -- /root/tests/smoke-test.sh\n lxc exec \"$NAME\" -- script -e -c \"pytest -s /root/tests/test-cluster-agent.py\"\n}\n\nfunction run_core_addons_tests() {\n local NAME=$1\n # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774\n lxc exec \"$NAME\" -- script -e -c \"pytest -s /var/snap/microk8s/common/addons/core/tests/test-addons.py\"\n}\n\nfunction run_community_addons_tests() {\n local NAME=$1\n lxc exec \"$NAME\" -- microk8s enable community\n lxc exec \"$NAME\" -- script -e -c \"pytest -s /var/snap/microk8s/common/addons/community/tests/\"\n}\n\nfunction run_gpu_addon_test() {\n if [ -f \"/var/snap/microk8s/common/addons/core/tests/test-addons.py\" ] &&\n grep test_gpu /var/snap/microk8s/common/addons/core/tests/test-addons.py -q\n then\n timeout 3600 pytest -s /var/snap/microk8s/common/addons/core/tests/test-addons.py -k test_gpu\n fi\n}\n\nfunction post_addons_tests() {\n local NAME=$1\n lxc exec \"$NAME\" -- microk8s reset\n lxc delete \"$NAME\" --force\n}\n\nTEMP=$(getopt -o \"l,h\" \\\n --long help,lib-mode,node-name:,distro:,channel:,proxy: \\\n -n \"$(basename \"$0\")\" -- \"$@\")\n\nif [ $? != 0 ] ; then echo \"Terminating...\" >&2 ; exit 1 ; fi\n\neval set -- \"$TEMP\"\n\nNAME=\"${NAME-\"machine-$RANDOM\"}\"\nDISTRO=\"${DISTRO-}\"\nTO_CHANNEL=\"${TO_CHANNEL-}\"\nPROXY=\"${PROXY-}\"\nLIBRARY_MODE=false\n\nwhile true; do\n case \"$1\" in\n -l | --lib-mode ) LIBRARY_MODE=true; shift ;;\n --node-name ) NAME=\"$2\"; shift 2 ;;\n --distro ) DISTRO=\"$2\"; shift 2 ;;\n --channel ) TO_CHANNEL=\"$2\"; shift 2 ;;\n --proxy ) PROXY=\"$2\"; shift 2 ;;\n -h | --help )\n prog=$(basename -s.wrapper \"$0\")\n echo \"Usage: $prog [options...]\"\n echo \" --node-name Name to be used for LXD containers\"\n echo \" Can also be set by using NAME environment variable\"\n echo \" --distro Distro image to be used for LXD containers Eg. ubuntu:18.04\"\n echo \" Can also be set by using DISTRO environment variable\"\n echo \" --channel Channel to be tested Eg. latest/edge\"\n echo \" Can also be set by using TO_CHANNEL environment variable\"\n echo \" --proxy Proxy url to be used by the nodes\"\n echo \" Can also be set by using PROXY environment variable\"\n echo \" -l, --lib-mode Make the script act like a library Eg. true / false\"\n echo\n exit ;;\n -- ) shift; break ;;\n * ) break ;;\n esac\ndone\n\nif [ \"$LIBRARY_MODE\" == \"false\" ];\nthen\n setup_addons_tests \"$NAME\" \"$DISTRO\" \"$PROXY\" \"$TO_CHANNEL\"\n run_smoke_test \"$NAME\"\n run_core_addons_tests \"$NAME\"\n DISABLE_COMMUNITY_TESTS=\"${DISABLE_COMMUNITY_TESTS:-0}\"\n if [ \"x${DISABLE_COMMUNITY_TESTS}\" != \"x1\" ]; then\n run_community_addons_tests \"$NAME\"\n fi\n run_gpu_addon_test\n post_addons_tests \"$NAME\"\nfi\n" }, { "path": "tests/libs/airgap.sh", "content": "#!/usr/bin/env bash\n\nsource tests/libs/utils.sh\n\nfunction airgap_wait_for_pods() {\n container=\"$1\"\n\n lxc exec \"$container\" -- bash -c \"\n while ! microk8s kubectl wait -n kube-system ds/calico-node --for=jsonpath='{.status.numberReady}'=1; do\n echo waiting for calico\n sleep 3\n done\n\n while ! microk8s kubectl wait -n kube-system deploy/hostpath-provisioner --for=jsonpath='{.status.readyReplicas}'=1; do\n echo waiting for hostpath provisioner\n sleep 3\n done\n\n while ! microk8s kubectl wait -n kube-system deploy/coredns --for=jsonpath='{.status.readyReplicas}'=1; do\n echo waiting for coredns\n sleep 3\n done\n \"\n}\n\nfunction setup_airgap_registry_mirror() {\n local NAME=$1\n local DISTRO=$2\n local PROXY=$3\n local TO_CHANNEL=$4\n\n create_machine \"$NAME\" \"$DISTRO\" \"$PROXY\"\n\n lxc exec \"$NAME\" -- bash -c \"\n mkdir -p /root/snap/microk8s/common\n echo '\n---\nversion: 0.1.0\n# pre-configure DNS args to save time from unnecessary kubelet restarts\nextraKubeletArgs:\n --cluster-dns: 10.152.183.10\n --cluster-domain: cluster.local\naddons:\n - name: dns\n - name: storage\n - name: registry\n' > /root/snap/microk8s/common/.microk8s.yaml\n \"\n\n if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]]\n then\n lxc file push \"${TO_CHANNEL}\" \"$NAME\"/var/tmp/microk8s_latest_amd64.snap\n while ! lxc exec \"$NAME\" -- bash -c \"snap install snapd\"; do\n echo retry install snapd\n sleep 1\n done\n while ! lxc exec \"$NAME\" -- bash -c \"snap install core22\"; do\n echo retry install core22\n sleep 1\n done\n while ! lxc exec \"$NAME\" -- bash -c \"snap install /var/tmp/microk8s_latest_amd64.snap --dangerous --classic\"; do\n echo retry snap install\n sleep 1\n done\n else\n lxc exec \"$NAME\" -- snap install microk8s --channel=\"${TO_CHANNEL}\" --classic\n fi\n}\n\nfunction wait_airgap_registry() {\n local NAME=$1\n airgap_wait_for_pods \"$NAME\"\n lxc exec \"$NAME\" -- bash -c '\n while ! curl --silent 127.0.0.1:32000/v2/_catalog; do\n echo waiting for registry\n sleep 2\n done\n '\n}\n\nfunction push_images_to_registry() {\n local NAME=$1\n lxc exec \"$NAME\" -- bash -c '\n for image in $(microk8s ctr image ls -q | grep -v \"sha256:\"); do\n mirror=$(echo $image | sed '\"'s,\\(docker.io\\|k8s.gcr.io\\|registry.k8s.io\\|quay.io\\|public.ecr.aws\\),${NAME}:32000,g'\"')\n sudo microk8s ctr image convert ${image} ${mirror}\n sudo microk8s ctr image push --plain-http ${mirror}\n done\n '\n}\n\nfunction setup_airgapped_microk8s() {\n local NAME=$1\n local DISTRO=$2\n local PROXY=$3\n local TO_CHANNEL=$4\n\n create_machine \"$NAME\" \"$DISTRO\" \"$PROXY\"\n if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]]\n then\n lxc file push \"${TO_CHANNEL}\" \"$NAME\"/var/tmp/microk8s.snap\n else\n lxc exec \"$NAME\" -- snap download microk8s --channel=\"${TO_CHANNEL}\" --target-directory /var/tmp --basename microk8s\n fi\n while ! lxc exec \"$NAME\" -- bash -c \"snap install snapd\"; do\n echo retry install snapd\n sleep 1\n done\n while ! lxc exec \"$NAME\" -- bash -c \"snap install core22\"; do\n echo retry install core22\n sleep 1\n done\n\n lxc exec \"$NAME\" -- bash -c \"\n echo '\n network:\n version: 2\n ethernets:\n eth0:\n dhcp4-overrides: { use-routes: false }\n routes: [{ to: 0.0.0.0/0, scope: link }]\n ' > /etc/netplan/70-airgap.yaml\n netplan apply\n \"\n if lxc exec \"$NAME\" -- bash -c \"ping -c1 1.1.1.1\"; then\n echo \"machine for airgap test has internet access when it should not\"\n exit 1\n fi\n lxc exec \"$NAME\" -- bash -c '\n mkdir -p /root/snap/microk8s/common\n echo \"\n---\nversion: 0.1.0\n# pre-configure DNS args to save time from unnecessary kubelet restarts\nextraKubeletArgs:\n --cluster-dns: 10.152.183.10\n --cluster-domain: cluster.local\ncontainerdRegistryConfigs:\n docker.io: |\n [host.\\\"http://'\"${REGISTRY_NAME}\"':32000\\\"]\n capabilities = [\\\"pull\\\", \\\"resolve\\\"]\n registry.k8s.io: |\n [host.\\\"http://'\"${REGISTRY_NAME}\"':32000\\\"]\n capabilities = [\\\"pull\\\", \\\"resolve\\\"]\n quay.io: |\n [host.\\\"http://'\"${REGISTRY_NAME}\"':32000\\\"]\n capabilities = [\\\"pull\\\", \\\"resolve\\\"]\n k8s.gcr.io: |\n [host.\\\"http://'\"${REGISTRY_NAME}\"':32000\\\"]\n capabilities = [\\\"pull\\\", \\\"resolve\\\"]\n public.ecr.aws: |\n [host.\\\"http://'\"${REGISTRY_NAME}\"':32000\\\"]\n capabilities = [\\\"pull\\\", \\\"resolve\\\"]\naddons:\n - name: dns\n - name: storage\n - name: registry\n \" > /root/snap/microk8s/common/.microk8s.yaml\n\n while ! snap install /var/tmp/microk8s.snap --dangerous --classic; do\n sleep 1\n done\n '\n}\n\nfunction test_airgapped_microk8s() {\n local NAME=$1\n lxc exec \"$NAME\" -- bash -c 'sudo microk8s enable hostpath-storage dns'\n airgap_wait_for_pods \"$NAME\"\n}\n\nfunction post_airgap_tests() {\n local REGISTRY_NAME=$1\n local AIRGAPPED_NAME=$2\n lxc rm \"$REGISTRY_NAME\" --force\n lxc rm \"$AIRGAPPED_NAME\" --force\n}\n\nTEMP=$(getopt -o \"lh\" \\\n --long help,lib-mode,registry-name:,node-name:,distro:,channel:,proxy: \\\n -n \"$(basename \"$0\")\" -- \"$@\")\n\nif [ $? != 0 ] ; then echo \"Terminating...\" >&2 ; exit 1 ; fi\n\neval set -- \"$TEMP\"\n\nREGISTRY_NAME=\"${REGISTRY_NAME:-\"registry-$RANDOM\"}\"\nAIRGAPPED_NAME=\"${AIRGAPPED_NAME:-\"machine-$RANDOM\"}\"\nDISTRO=\"${DISTRO:-}\"\nTO_CHANNEL=\"${TO_CHANNEL:-}\"\nPROXY=\"${PROXY:-}\"\nLIBRARY_MODE=false\n\nwhile true; do\n case \"$1\" in\n -l | --lib-mode ) LIBRARY_MODE=true; shift ;;\n --registry-name ) REGISTRY_NAME=\"$2\"; shift 2 ;;\n --node-name ) AIRGAPPED_NAME=\"$2\"; shift 2 ;;\n --distro ) DISTRO=\"$2\"; shift 2 ;;\n --channel ) TO_CHANNEL=\"$2\"; shift 2 ;;\n --proxy ) PROXY=\"$2\"; shift 2 ;;\n -h | --help )\n prog=$(basename -s.wrapper \"$0\")\n echo \"Usage: $prog [options...]\"\n echo \" --registry-name Name to be used for registry LXD containers\"\n echo \" Can also be set by using REGISTRY_NAME environment variable\"\n echo \" --node-name Name to be used for LXD containers\"\n echo \" Can also be set by using AIRGAPPED_NAME environment variable\"\n echo \" --distro Distro image to be used for LXD containers Eg. ubuntu:18.04\"\n echo \" Can also be set by using DISTRO environment variable\"\n echo \" --channel Channel to be tested Eg. latest/edge\"\n echo \" Can also be set by using TO_CHANNEL environment variable\"\n echo \" --proxy Proxy url to be used by the nodes\"\n echo \" Can also be set by using PROXY environment variable\"\n echo \" -l, --lib-mode Make the script act like a library Eg. true / false\"\n echo\n exit ;;\n -- ) shift; break ;;\n * ) break ;;\n esac\ndone\n\nif [ \"$LIBRARY_MODE\" == \"false\" ];\nthen\n echo \"1/5 -- Install registry mirror\"\n setup_airgap_registry_mirror \"$REGISTRY_NAME\" \"$DISTRO\" \"$PROXY\" \"$TO_CHANNEL\"\n echo \"2/5 -- Wait for MicroK8s instance with registry to come up\"\n wait_airgap_registry \"$REGISTRY_NAME\"\n echo \"3/5 -- Push images to registry mirror\"\n push_images_to_registry \"$REGISTRY_NAME\"\n echo \"4/5 -- Install MicroK8s on an airgap environment (using registry mirror)\"\n setup_airgapped_microk8s \"$AIRGAPPED_NAME\" \"$DISTRO\" \"$PROXY\" \"$TO_CHANNEL\"\n echo \"5/5 -- Wait for airgapped MicroK8s to come up\"\n airgap_wait_for_pods \"$AIRGAPPED_NAME\"\n echo \"Cleaning up\"\n post_airgap_tests \"$REGISTRY_NAME\" \"$AIRGAPPED_NAME\"\nfi\n" }, { "path": "tests/libs/clustering.sh", "content": "#!/usr/bin/env bash\n\nfunction run_clustering_tests() {\n # Test clustering. This test will create lxc containers or multipass VMs\n # therefore we do not need to run it inside a VM/container\n TRY_ATTEMPT=0\n while ! (timeout 3600 pytest -s tests/test-cluster.py) &&\n ! [ ${TRY_ATTEMPT} -eq 3 ]\n do\n TRY_ATTEMPT=$((TRY_ATTEMPT+1))\n sleep 1\n done\n if [ ${TRY_ATTEMPT} -eq 3 ]\n then\n echo \"Test clusterring took longer than expected\"\n exit 1\n fi\n}\n\nTEMP=$(getopt -o \"lh\" \\\n --long help,lib-mode,channel:,backend:,lxd-profile: \\\n -n \"$(basename \"$0\")\" -- \"$@\")\n\nif [ $? != 0 ] ; then echo \"Terminating...\" >&2 ; exit 1 ; fi\n\neval set -- \"$TEMP\"\n\nBACKEND=\"${BACKEND-\"lxc\"}\"\nCHANNEL_TO_TEST=\"${CHANNEL_TO_TEST-}\"\nLXC_PROFILE=\"${LXC_PROFILE-}\"\nLIBRARY_MODE=false\n\nwhile true; do\n case \"$1\" in\n -l | --lib-mode ) LIBRARY_MODE=true; shift ;;\n --backend ) BACKEND=\"$2\"; shift 2 ;;\n --lxd-profile ) LXC_PROFILE=\"$2\"; shift 2 ;;\n --channel ) CHANNEL_TO_TEST=\"$2\"; shift 2 ;;\n -h | --help )\n prog=$(basename -s.wrapper \"$0\")\n echo \"Usage: $prog [options...]\"\n echo \" --backend Backend to be used for clustering tests Eg. lxc\"\n echo \" Can also be set by using BACKEND environment variable\"\n echo \" --lxd-profile Profile to be used for lxc backend Eg. tests/lxc/microk8s.profile\"\n echo \" Can also be set by using LXC_PROFILE environment variable\"\n echo \" --channel Channel to be tested Eg. latest/edge\"\n echo \" Can also be set by using CHANNEL_TO_TEST environment variable\"\n echo \" --proxy Proxy url to be used by the nodes\"\n echo \" Can also be set by using PROXY environment variable\"\n echo \" -l, --lib-mode Make the script act like a library Eg. true / false\"\n echo\n exit ;;\n -- ) shift; break ;;\n * ) break ;;\n esac\ndone\n\nif [ \"$LIBRARY_MODE\" == \"false\" ];\nthen\n run_clustering_tests\nfi\n" }, { "path": "tests/libs/spread.sh", "content": "#!/usr/bin/env bash\n\nset -ex\n\nsource tests/libs/utils.sh\n\nfunction run_spread_tests() {\n local NAME=$1\n local DISTRO=$2\n local PROXY=$3\n local TO_CHANNEL=$4\n\n create_machine \"$NAME\" \"$DISTRO\" \"$PROXY\"\n\n if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]]\n then\n lxc file push \"${TO_CHANNEL}\" \"$NAME\"/tmp/microk8s_latest_amd64.snap\n for i in {1..5}; do lxc exec \"$NAME\" -- snap install /tmp/microk8s_latest_amd64.snap --dangerous --classic && break || sleep 5; done\n else\n lxc exec \"$NAME\" -- snap install microk8s --channel=\"${TO_CHANNEL}\" --classic\n fi\n\n lxc exec \"$NAME\" -- /snap/bin/microk8s stop\n lxc exec \"$NAME\" -- sed -i '/\\[plugins.\"io.containerd.grpc.v1.cri\"\\]/a \\ \\ disable_apparmor=true' /var/snap/microk8s/current/args/containerd-template.toml\n lxc exec \"$NAME\" -- /snap/bin/microk8s start\n lxc exec \"$NAME\" -- /snap/bin/microk8s status --wait-ready --timeout 300\n sleep 45\n lxc exec \"$NAME\" -- /snap/bin/microk8s kubectl wait pod --all --for=condition=Ready -A --timeout=300s\n lxc exec \"$NAME\" -- script -e -c \"pytest -s /root/tests/test-simple.py\"\n}\n\nTEMP=$(getopt -o \"l,h\" \\\n --long help,lib-mode,node-name:,distro:,channel:,proxy: \\\n -n \"$(basename \"$0\")\" -- \"$@\")\n\nif [ $? != 0 ] ; then echo \"Terminating...\" >&2 ; exit 1 ; fi\n\neval set -- \"$TEMP\"\n\nNAME=\"${NAME-\"machine-$RANDOM\"}\"\nDISTRO=\"${DISTRO-}\"\nTO_CHANNEL=\"${TO_CHANNEL-}\"\nPROXY=\"${PROXY-}\"\nLIBRARY_MODE=false\n\nwhile true; do\n case \"$1\" in\n -l | --lib-mode ) LIBRARY_MODE=true; shift ;;\n --node-name ) NAME=\"$2\"; shift 2 ;;\n --distro ) DISTRO=\"$2\"; shift 2 ;;\n --channel ) TO_CHANNEL=\"$2\"; shift 2 ;;\n --proxy ) PROXY=\"$2\"; shift 2 ;;\n -h | --help )\n prog=$(basename -s.wrapper \"$0\")\n echo \"Usage: $prog [options...]\"\n echo \" --node-name Name to be used for LXD containers\"\n echo \" Can also be set by using NAME environment variable\"\n echo \" --distro Distro image to be used for LXD containers Eg. ubuntu:18.04\"\n echo \" Can also be set by using DISTRO environment variable\"\n echo \" --channel Channel to be tested Eg. latest/edge\"\n echo \" Can also be set by using TO_CHANNEL environment variable\"\n echo \" --proxy Proxy url to be used by the nodes\"\n echo \" Can also be set by using PROXY environment variable\"\n echo \" -l, --lib-mode Make the script act like a library Eg. true / false\"\n echo\n exit ;;\n -- ) shift; break ;;\n * ) break ;;\n esac\ndone\n\nif [ \"$LIBRARY_MODE\" == \"false\" ];\nthen\n run_spread_tests \"$NAME\" \"$DISTRO\" \"$PROXY\" \"$TO_CHANNEL\"\nfi\n" }, { "path": "tests/libs/upgrade-path.sh", "content": "#!/usr/bin/env bash\n\nfunction setup_upgrade_path_tests() {\n # Test upgrade-path\n local NAME=$1\n local DISTRO=$2\n local PROXY=$3\n create_machine \"$NAME\" \"$DISTRO\" \"$PROXY\"\n}\n\nfunction run_upgrade_path_tests() {\n local NAME=$1\n local FROM_CHANNEL=$2\n local TO_CHANNEL=$3\n # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774\n if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]]\n then\n lxc file push \"${TO_CHANNEL}\" \"$NAME\"/tmp/microk8s_latest_amd64.snap\n lxc exec \"$NAME\" -- script -e -c \"UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=/tmp/microk8s_latest_amd64.snap pytest -s /root/tests/test-upgrade-path.py\"\n else\n lxc exec \"$NAME\" -- script -e -c \"UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=${TO_CHANNEL} pytest -s /root/tests/test-upgrade-path.py\"\n fi\n}\n\nfunction post_upgrade_path_test() {\n local NAME=$1\n lxc delete \"$NAME\" --force\n}\n\nTEMP=$(getopt -o \"lh\" \\\n --long help,lib-mode,node-name:,distro:,from-channel:,to-channel:,proxy: \\\n -n \"$(basename \"$0\")\" -- \"$@\")\n\nif [ $? != 0 ] ; then echo \"Terminating...\" >&2 ; exit 1 ; fi\n\neval set -- \"$TEMP\"\n\nNAME=\"${NAME-\"machine-$RANDOM\"}\"\nDISTRO=\"${DISTRO-}\"\nFROM_CHANNEL=\"${FROM_CHANNEL-}\"\nTO_CHANNEL=\"${TO_CHANNEL-}\"\nPROXY=\"${PROXY-}\"\nLIBRARY_MODE=false\n\nwhile true; do\n case \"$1\" in\n -l | --lib-mode ) LIBRARY_MODE=true; shift ;;\n --node-name ) NAME=\"$2\"; shift 2 ;;\n --distro ) DISTRO=\"$2\"; shift 2 ;;\n --from-channel ) FROM_CHANNEL=\"$2\"; shift 2 ;;\n --to-channel ) TO_CHANNEL=\"$2\"; shift 2 ;;\n --proxy ) PROXY=\"$2\"; shift 2 ;;\n -h | --help )\n prog=$(basename -s.wrapper \"$0\")\n echo \"Usage: $prog [options...]\"\n echo \" --node-name Name to be used for LXD containers\"\n echo \" Can also be set by using NAME environment variable\"\n echo \" --distro Distro image to be used for LXD containers Eg. ubuntu:18.04\"\n echo \" Can also be set by using DISTRO environment variable\"\n echo \" --from-channel Channel to upgrade from to the channel under testing Eg. latest/beta\"\n echo \" Can also be set by using FROM_CHANNEL environment variable\"\n echo \" --to-channel Channel to be tested Eg. latest/edge\"\n echo \" Can also be set by using TO_CHANNEL environment variable\"\n echo \" --proxy Proxy url to be used by the nodes\"\n echo \" Can also be set by using PROXY environment variable\"\n echo \" -l, --lib-mode Make the script act like a library Eg. true / false\"\n echo\n exit ;;\n -- ) shift; break ;;\n * ) break ;;\n esac\ndone\n\nif [ \"$LIBRARY_MODE\" == \"false\" ];\nthen\n setup_upgrade_path_tests \"$NAME\" \"$DISTRO\" \"$PROXY\"\n run_upgrade_path_tests \"$NAME\" \"$FROM_CHANNEL\" \"$TO_CHANNEL\"\n post_upgrade_path_test \"$NAME\"\nfi\n" }, { "path": "tests/libs/utils.sh", "content": "#!/usr/bin/env bash\n\nfunction create_machine() {\n local NAME=$1\n local DISTRO=$2\n local PROXY=$3\n if ! lxc profile show microk8s\n then\n lxc profile copy default microk8s\n fi\n lxc profile edit microk8s < tests/lxc/microk8s.profile\n\n lxc launch -p default -p microk8s \"$DISTRO\" \"$NAME\"\n\n # Allow for the machine to boot and get an IP\n sleep 20\n # CentOS 8,9 variants(rocky, alma) don't ship with tar, such a dirty hack...\n lxc exec \"$NAME\" -- /bin/bash -c \"yum install tar -y || true\"\n tar cf - ./tests | lxc exec \"$NAME\" -- tar xvf - -C /root\n DISTRO_DEPS_TMP=\"${DISTRO//:/_}\"\n DISTRO_DEPS=\"${DISTRO_DEPS_TMP////-}\"\n lxc exec \"$NAME\" -- /bin/bash \"/root/tests/lxc/install-deps/$DISTRO_DEPS\"\n lxc exec \"$NAME\" -- reboot\n sleep 20\n\n trap 'lxc delete '\"${NAME}\"' --force || true' EXIT\n if [ ! -z \"${PROXY}\" ]\n then\n lxc exec \"$NAME\" -- /bin/bash -c \"echo HTTPS_PROXY=$PROXY >> /etc/environment\"\n lxc exec \"$NAME\" -- /bin/bash -c \"echo https_proxy=$PROXY >> /etc/environment\"\n lxc exec \"$NAME\" -- reboot\n sleep 20\n fi\n}\n\nfunction setup_tests() {\n DISTRO=\"${1-$DISTRO}\"\n FROM_CHANNEL=\"${2-$FROM_CHANNEL}\"\n TO_CHANNEL=\"${3-$TO_CHANNEL}\"\n PROXY=\"${4-$PROXY}\"\n\n export DEBIAN_FRONTEND=noninteractive\n apt-get install python3-pip -y\n pip3 install -U pytest requests pyyaml sh psutil\n apt-get install jq -y\n snap install kubectl --classic\n export ARCH=$(uname -m)\n export LXC_PROFILE=\"tests/lxc/microk8s.profile\"\n export BACKEND=\"lxc\"\n export CHANNEL_TO_TEST=${TO_CHANNEL}\n}\n" }, { "path": "tests/lxc/install-deps/images_almalinux-8", "content": "#!/usr/bin/env bash\n\ndnf install epel-release -y\ndnf upgrade -y\nyum install sudo -y\nyum install fuse squashfuse -y\nyum install snapd -y\nsystemctl enable --now snapd.socket\nln -s /var/lib/snapd/snap /snap\nyum install python3-pip -y\nyum install docker -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_archlinux", "content": "#!/usr/bin/env bash\n\npacman -S --noconfirm --needed base-devel\npacman -S --noconfirm --needed git\nuseradd user\necho \"user ALL=(ALL) NOPASSWD: ALL\" >> /etc/sudoers\nmkdir /home/user\nchown user:user /home/user\ncd /tmp\nsu user -c \"git clone https://aur.archlinux.org/snapd.git\"\nsu user -c \"cd snapd; makepkg -si --noconfirm\"\nsudo systemctl enable --now snapd.socket\nsudo ln -s /var/lib/snapd/snap /snap\nsu user -c \"git clone https://aur.archlinux.org/squashfuse-git.git\"\nsu user -c \"cd squashfuse-git; makepkg -si --noconfirm\"\npacman -S --noconfirm --needed fuse3\necho \"PATH=$PATH:/snap/bin\" >> /etc/profile\npacman -S --noconfirm python-pip\npacman -S --noconfirm python\npacman -S --noconfirm docker\nsudo systemctl enable --now docker.service\necho \"127.0.0.1 localhost\" | sudo tee -a /etc/hosts\npip3 install pytest==8.3.4 requests pyyaml\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_centos-7", "content": "#!/usr/bin/env bash\n\nyum install epel-release -y\nyum install sudo -y\nyum install snapd -y\nsystemctl enable --now snapd.socket\nln -s /var/lib/snapd/snap /snap\nyum install python3-pip -y\nyum install docker -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_centos-8-Stream", "content": "#!/usr/bin/env bash\n\ndnf install epel-release -y\ndnf upgrade -y\nyum install sudo -y\nyum install fuse squashfuse -y\nyum install snapd -y\nsystemctl enable --now snapd.socket\nln -s /var/lib/snapd/snap /snap\nyum install python3-pip -y\nyum install docker -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_debian-10", "content": "#!/usr/bin/env bash\n\nexport $(grep -v '^#' /etc/environment | xargs)\nexport DEBIAN_FRONTEND=noninteractive\n\napt-get update\napt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698\n# if core is to be installed by microk8s it fails\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_debian-11", "content": "#!/usr/bin/env bash\n\nexport $(grep -v '^#' /etc/environment | xargs)\nexport DEBIAN_FRONTEND=noninteractive\n\napt-get update\napt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698\n# if core is to be installed by microk8s it fails\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n\n\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_debian-12", "content": "#!/usr/bin/env bash\n\nexport $(grep -v '^#' /etc/environment | xargs)\nexport DEBIAN_FRONTEND=noninteractive\n\napt-get update\napt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil --break-system-packages\n# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698\n# if core is to be installed by microk8s it fails\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_fedora-37", "content": "#!/usr/bin/env bash\n\nyum install epel-release -y\nyum install sudo -y\nyum install fuse squashfuse -y\nyum install snapd -y\nsystemctl enable --now snapd.socket\nln -s /var/lib/snapd/snap /snap\nyum install python3-pip -y\nyum install docker -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_fedora-38", "content": "#!/usr/bin/env bash\n\nyum install epel-release -y\nyum install sudo -y\nyum install fuse squashfuse -y\nyum install snapd -y\nsystemctl enable --now snapd.socket\nln -s /var/lib/snapd/snap /snap\nyum install python3-pip -y\nyum install docker -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/images_rockylinux-8", "content": "#!/usr/bin/env bash\n\ndnf install epel-release -y\ndnf upgrade -y\nyum install sudo -y\nyum install fuse squashfuse -y\nyum install snapd -y\nsystemctl enable --now snapd.socket\nln -s /var/lib/snapd/snap /snap\nyum install python3-pip -y\nyum install docker -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n\n# wait for the snapd seeding to take place!\nn=0\nuntil [ $n -ge 7 ]\ndo\n sudo snap install core22 && break # substitute your command here\n n=$[$n+1]\n sleep 10\ndone\n" }, { "path": "tests/lxc/install-deps/ubuntu_16.04", "content": "#!/usr/bin/env bash\n\nexport $(grep -v '^#' /etc/environment | xargs)\nexport DEBIAN_FRONTEND=noninteractive\n\napt-get update\napt-get install python3-pip docker.io -y\npip3 install importlib-metadata==2.1.0 pytest requests pyyaml sh\n# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698\n# if core is to be installed by microk8s it fails\nsnap install core22 | true\n" }, { "path": "tests/lxc/install-deps/ubuntu_18.04", "content": "#!/usr/bin/env bash\n\nexport $(grep -v '^#' /etc/environment | xargs)\nexport DEBIAN_FRONTEND=noninteractive\n\napt-get update\napt-get install python3-pip docker.io -y\n# In Ubuntu 18.04 for arm64 on LXC, \"pip3 install -U pyyaml\" breaks netplan\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698\n# if core is to be installed by microk8s it fails\nsnap install core22 | true\n" }, { "path": "tests/lxc/install-deps/ubuntu_20.04", "content": "#!/usr/bin/env bash\n\nexport $(grep -v '^#' /etc/environment | xargs)\nexport DEBIAN_FRONTEND=noninteractive\n\napt-get update\napt-get install xdelta3 -y\napt-get install python3-pip docker.io -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698\n# if core is to be installed by microk8s it fails\nsnap install core22 | true\n" }, { "path": "tests/lxc/install-deps/ubuntu_22.04", "content": "#!/usr/bin/env bash\n\nexport $(grep -v '^#' /etc/environment | xargs)\nexport DEBIAN_FRONTEND=noninteractive\n\napt-get update\napt-get install python3-pip docker.io -y\npip3 install pytest==8.3.4 requests pyyaml sh psutil\n# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698\n# if core is to be installed by microk8s it fails\nsnap install core22 | true\n" }, { "path": "tests/lxc/microk8s-zfs.profile", "content": "name: microk8s\nconfig:\n boot.autostart: \"true\"\n linux.kernel_modules: ip_vs,ip_vs_rr,ip_vs_wrr,ip_vs_sh,ip_tables,ip6_tables,netlink_diag,nf_nat,overlay,br_netfilter\n raw.lxc: |\n lxc.apparmor.profile=unconfined\n lxc.mount.auto=proc:rw sys:rw cgroup:rw\n lxc.cgroup.devices.allow=a\n lxc.cap.drop=\n security.nesting: \"true\"\n security.privileged: \"true\"\ndescription: \"\"\ndevices:\n aadisable:\n path: /sys/module/nf_conntrack/parameters/hashsize\n source: /sys/module/nf_conntrack/parameters/hashsize\n type: disk\n aadisable2:\n path: /dev/zfs\n source: /dev/zfs\n type: disk\n aadisable3:\n path: /dev/kmsg\n source: /dev/kmsg\n type: unix-char\n aadisable4:\n path: /sys/fs/bpf\n source: /sys/fs/bpf\n type: disk\n aadisable5:\n path: /proc/sys/net/netfilter/nf_conntrack_max\n source: /proc/sys/net/netfilter/nf_conntrack_max\n type: disk\n" }, { "path": "tests/lxc/microk8s.profile", "content": "name: microk8s\nconfig:\n boot.autostart: \"true\"\n linux.kernel_modules: ip_vs,ip_vs_rr,ip_vs_wrr,ip_vs_sh,ip_tables,ip6_tables,netlink_diag,nf_nat,overlay,br_netfilter\n raw.lxc: |\n lxc.apparmor.profile=unconfined\n lxc.mount.auto=proc:rw sys:rw cgroup:rw\n lxc.cgroup.devices.allow=a\n lxc.cap.drop=\n security.nesting: \"true\"\n security.privileged: \"true\"\ndescription: \"\"\ndevices:\n aadisable:\n path: /sys/module/nf_conntrack/parameters/hashsize\n source: /sys/module/nf_conntrack/parameters/hashsize\n type: disk\n aadisable2:\n path: /dev/kmsg\n source: /dev/kmsg\n type: unix-char\n aadisable3:\n path: /sys/fs/bpf\n source: /sys/fs/bpf\n type: disk\n aadisable4:\n path: /proc/sys/net/netfilter/nf_conntrack_max\n source: /proc/sys/net/netfilter/nf_conntrack_max\n type: disk\n" }, { "path": "tests/requirements.txt", "content": "kubernetes\njinja2\npylxd\npytest\npytest-xdist\npyyaml\nsh\njsonschema>=4.0.0\npdbpp\npsutil\nnetifaces\nrequests\n" }, { "path": "tests/smoke-test.sh", "content": "#!/bin/bash\n\nset -eux\n\nn=0\nuntil [ $n -ge 10 ]\ndo\n (sudo /snap/bin/microk8s kubectl get all --all-namespaces | grep -z \"service/kubernetes\") && break\n n=$[$n+1]\n if [ $n -ge 10 ]; then\n exit 1\n fi\n sleep 20\ndone\n\nn=0\nuntil [ $n -ge 3 ]\ndo\n (sudo /snap/bin/microk8s kubectl get no | grep -z \"Ready\") && exit 0\n n=$[$n+1]\n sleep 20\ndone\n\nsudo /snap/bin/microk8s kubectl -n kube-system rollout status deployment.apps/calico-kube-controllers\nexit 1\n" }, { "path": "tests/templates/bbox-local.yaml", "content": "apiVersion: v1\nkind: Pod\nmetadata:\n name: busybox\n namespace: default\nspec:\n containers:\n - name: busybox\n image: localhost:32000/my-busybox\n command:\n - sleep\n - \"3600\"\n imagePullPolicy: IfNotPresent\n restartPolicy: Always\n" }, { "path": "tests/templates/dual-stack.yaml", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginxdualstack\nspec:\n selector:\n matchLabels:\n run: nginxdualstack\n replicas: 1\n template:\n metadata:\n labels:\n run: nginxdualstack\n spec:\n containers:\n - name: nginxdualstack\n image: cdkbot/nginxdualstack:1.0.0\n ports:\n - containerPort: 80\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: nginx6\n labels:\n run: nginxdualstack\nspec:\n type: NodePort\n ipFamilies:\n - IPv6\n ipFamilyPolicy: RequireDualStack\n ports:\n - port: 80\n protocol: TCP\n selector:\n run: nginxdualstack\n" }, { "path": "tests/templates/ingress.yaml", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n creationTimestamp: null\n labels:\n app: microbot\n name: microbot\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: microbot\n strategy: {}\n template:\n metadata:\n creationTimestamp: null\n labels:\n app: microbot\n spec:\n containers:\n - image: cdkbot/microbot-$ARCH\n imagePullPolicy: \"\"\n name: microbot\n ports:\n - containerPort: 80\n livenessProbe:\n httpGet:\n path: /\n port: 80\n initialDelaySeconds: 5\n timeoutSeconds: 30\n resources: {}\n restartPolicy: Always\n serviceAccountName: \"\"\nstatus: {}\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: microbot\n labels:\n app: microbot\nspec:\n ports:\n - port: 80\n protocol: TCP\n targetPort: 80\n selector:\n app: microbot\n---\napiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: microbot-ingress-nip\nspec:\n rules:\n - host: microbot.127.0.0.1.nip.io\n http:\n paths:\n - path: /\n pathType: Prefix\n backend:\n service:\n name: microbot\n port:\n number: 80\n" }, { "path": "tests/templates/nginx-pod.yaml", "content": "apiVersion: v1\nkind: Pod\nmetadata:\n labels:\n app: nginx\n name: nginx\n namespace: default\nspec:\n containers:\n - name: nginx\n image: nginx:latest\n restartPolicy: Always\n" }, { "path": "tests/templates/pvc.yaml", "content": "kind: PersistentVolumeClaim\napiVersion: v1\nmetadata:\n name: myclaim\nspec:\n accessModes:\n - ReadWriteOnce\n volumeMode: Filesystem\n resources:\n requests:\n storage: 1Gi\n---\nkind: Pod\napiVersion: v1\nmetadata:\n name: hostpath-test-pod\nspec:\n containers:\n - name: hostpath-test-container\n image: busybox\n command:\n [\"/bin/sh\", \"-c\", \"while true; do date >> /mnt/dates; sleep 2; done\"]\n volumeMounts:\n - name: hostpath-volume\n mountPath: \"/mnt\"\n restartPolicy: \"Never\"\n volumes:\n - name: hostpath-volume\n persistentVolumeClaim:\n claimName: myclaim\n" }, { "path": "tests/templates/registry-sc.yaml", "content": "apiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata:\n name: registry-test-sc\nprovisioner: microk8s.io/hostpath\nreclaimPolicy: Delete\nvolumeBindingMode: Immediate\n" }, { "path": "tests/templates/simple-deploy.yaml", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-deployment\n labels:\n app: nginx\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: nginx\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: nginx:1.14.2\n ports:\n - containerPort: 80\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: nginx-service\nspec:\n selector:\n app: nginx\n ports:\n - protocol: TCP\n port: 80\n targetPort: 80\n---\napiVersion: networking.k8s.io/v1\nkind: Ingress\nmetadata:\n name: nginx-ingress\nspec:\n rules:\n - http:\n paths:\n - path: /\n pathType: Prefix\n backend:\n service:\n name: nginx-service\n port:\n number: 80\n" }, { "path": "tests/test-cluster-agent.py", "content": "import requests\n\n\nclass TestClusterAgent(object):\n \"\"\"\n Validates the cluster agent in MicroK8s\n \"\"\"\n\n def test_cluster_agent_health(self):\n \"\"\"\n Query the cluster agent health endpoint to verify it is up and healthy.\n\n \"\"\"\n response = requests.get(\n \"https://127.0.0.1:25000/health\", verify=\"/var/snap/microk8s/current/certs/ca.crt\"\n )\n assert response.status_code == 200 and response.json()[\"status\"] == \"OK\"\n" }, { "path": "tests/test-cluster.py", "content": "import string\nimport random\nimport time\nimport pytest\nimport os\nimport datetime\nimport requests\nimport signal\nimport subprocess\nfrom os import path\nfrom pathlib import Path\nfrom utils import (\n is_strict,\n is_ipv6_configured,\n)\n\n\n# Provide a list of VMs you want to reuse. VMs should have already microk8s installed.\n# the test will attempt a refresh to the channel requested for testing\n# reuse_vms = ['vm-ldzcjb', 'vm-nfpgea', 'vm-pkgbtw']\nreuse_vms = None\n\n# Channel we want to test. A full path to a local snap can be used for local builds\nchannel_to_test = os.environ.get(\"CHANNEL_TO_TEST\", \"latest/stable\")\nbackend = os.environ.get(\"BACKEND\", None)\nprofile = os.environ.get(\"LXC_PROFILE\", \"lxc/microk8s.profile\")\nsnap_data = os.environ.get(\"SNAP_DATA\", \"/var/snap/microk8s/current\")\ndistro = os.environ.get(\"DISTRO\", \"ubuntu:22.04\")\n\nTEMPLATES = Path(__file__).absolute().parent / \"templates\"\n\n\nclass VM:\n \"\"\"\n This class abstracts the backend we are using. It could be either multipass or lxc.\n \"\"\"\n\n launch_config = \"\"\"---\nversion: 0.1.0\nextraCNIEnv:\n IPv4_SUPPORT: true\n IPv4_CLUSTER_CIDR: 10.3.0.0/16\n IPv4_SERVICE_CIDR: 10.153.183.0/24\n IPv6_SUPPORT: true\n IPv6_CLUSTER_CIDR: fd02::/64\n IPv6_SERVICE_CIDR: fd99::/108\nextraSANs:\n - 10.153.183.1\"\"\"\n\n def __init__(self, backend=None, attach_vm=None):\n \"\"\"Detect the available backends and instantiate a VM.\n\n If `attach_vm` is provided we just make sure the right MicroK8s is deployed.\n :param backend: either multipass of lxc\n :param attach_vm: the name of the VM we want to reuse\n \"\"\"\n rnd_letters = \"\".join(random.choice(string.ascii_lowercase) for i in range(6))\n self.backend = backend\n self.vm_name = \"vm-{}\".format(rnd_letters)\n self.attached = False\n if attach_vm:\n self.attached = True\n self.vm_name = attach_vm\n\n def setup(self, channel_or_snap):\n \"\"\"\n Setup the VM with the right snap.\n\n :param channel_or_snap: the snap channel or the path to the local snap build\n \"\"\"\n if (path.exists(\"/snap/bin/multipass\") and not self.backend) or self.backend == \"multipass\":\n print(\"Creating mulitpass VM\")\n self.backend = \"multipass\"\n self._setup_multipass(channel_or_snap)\n\n elif (path.exists(\"/snap/bin/lxc\") and not self.backend) or self.backend == \"lxc\":\n print(\"Creating lxc VM\")\n self.backend = \"lxc\"\n self._setup_lxc(channel_or_snap)\n else:\n raise Exception(\"Need to install multipass or lxc\")\n\n def _setup_lxc(self, channel_or_snap):\n if not self.attached:\n profiles = subprocess.check_output(\"/snap/bin/lxc profile list\".split())\n if \"microk8s\" not in profiles.decode():\n subprocess.check_call(\"/snap/bin/lxc profile copy default microk8s\".split())\n with open(profile, \"r+\") as fp:\n profile_string = fp.read()\n process = subprocess.Popen(\n \"/snap/bin/lxc profile edit microk8s\".split(),\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n )\n process.stdin.write(profile_string.encode())\n process.stdin.close()\n\n subprocess.check_call(\n \"/snap/bin/lxc launch -p default -p microk8s {} {}\".format(\n distro, self.vm_name\n ).split()\n )\n time.sleep(20)\n if is_ipv6_configured():\n self._load_launch_configuration_lxc()\n\n if channel_or_snap.startswith(\"/\"):\n self._transfer_install_local_snap_lxc(channel_or_snap)\n else:\n cmd = \"snap install microk8s --classic --channel {}\".format(channel_or_snap)\n time.sleep(20)\n print(\"About to run {}\".format(cmd))\n output = \"\"\n attempt = 0\n while attempt < 3:\n try:\n output = self.run(cmd)\n break\n except ChildProcessError:\n time.sleep(10)\n attempt += 1\n print(output.decode())\n else:\n if is_ipv6_configured():\n self._load_launch_configuration_lxc()\n\n if channel_or_snap.startswith(\"/\"):\n self._transfer_install_local_snap_lxc(channel_or_snap)\n else:\n cmd = \"/snap/bin/lxc exec {} -- \".format(self.vm_name).split()\n cmd.append(\"sudo snap refresh microk8s --channel {}\".format(channel_or_snap))\n subprocess.check_call(cmd)\n\n def _load_launch_configuration_lxc(self):\n # Set launch configurations before installing microk8s\n print(\"Setting launch configurations\")\n self.run(\"mkdir -p /var/snap/microk8s/common/\")\n file_path = \"microk8s.yaml\"\n print(self.launch_config)\n with open(file_path, \"w\") as file:\n file.write(self.launch_config)\n\n # Copy the file to the VM\n cmd = \"lxc file push {} {}/var/snap/microk8s/common/.microk8s.yaml\".format(\n file_path, self.vm_name\n ).split()\n subprocess.check_output(cmd)\n os.remove(file_path)\n\n def _transfer_install_local_snap_lxc(self, channel_or_snap):\n try:\n print(\"Installing snap from {}\".format(channel_or_snap))\n cmd_prefix = \"/snap/bin/lxc exec {} -- script -e -c\".format(self.vm_name).split()\n cmd = [\"rm -rf /var/tmp/microk8s.snap\"]\n subprocess.check_output(cmd_prefix + cmd)\n cmd = \"lxc file push {} {}/var/tmp/microk8s.snap\".format(\n channel_or_snap, self.vm_name\n ).split()\n subprocess.check_output(cmd)\n cmd = [\"snap install /var/tmp/microk8s.snap --dangerous --classic\"]\n subprocess.check_output(cmd_prefix + cmd)\n time.sleep(20)\n except subprocess.CalledProcessError as e:\n print(e.output.decode())\n raise\n\n def _setup_multipass(self, channel_or_snap):\n if not self.attached:\n version = distro.split(\":\")\n subprocess.check_call(\n \"/snap/bin/multipass launch {} -n {} -m 2G\".format(version[1], self.vm_name).split()\n )\n if is_ipv6_configured():\n self._load_launch_configuration_multipass()\n\n if channel_or_snap.startswith(\"/\"):\n self._transfer_install_local_snap_multipass(channel_or_snap)\n else:\n subprocess.check_call(\n \"/snap/bin/multipass exec {} -- sudo \"\n \"snap install microk8s --classic --channel {}\".format(\n self.vm_name, channel_or_snap\n ).split()\n )\n else:\n if is_ipv6_configured():\n self._load_launch_configuration_multipass()\n if channel_or_snap.startswith(\"/\"):\n self._transfer_install_local_snap_multipass(channel_or_snap)\n else:\n subprocess.check_call(\n \"/snap/bin/multipass exec {} -- sudo \"\n \"snap refresh microk8s --channel {}\".format(\n self.vm_name, channel_or_snap\n ).split()\n )\n\n def _load_launch_configuration_multipass(self):\n # Set launch configurations before installing microk8s\n print(\"Setting launch configurations\")\n self.run(\"mkdir -p /var/snap/microk8s/common/\")\n self.run(\"chmod 777 /var/snap/microk8s/common/\")\n file_path = \"microk8s.yaml\"\n print(self.launch_config)\n with open(file_path, \"w\") as file:\n file.write(self.launch_config)\n\n # Copy the file to the VM\n subprocess.check_call(\n \"/snap/bin/multipass transfer {} {}:/var/snap/microk8s/common/.microk8s.yaml\".format(\n file_path, self.vm_name\n ).split()\n )\n os.remove(file_path)\n\n def _transfer_install_local_snap_multipass(self, channel_or_snap):\n print(\"Installing snap from {}\".format(channel_or_snap))\n subprocess.check_call(\n \"/snap/bin/multipass transfer {} {}:/var/tmp/microk8s.snap\".format(\n channel_or_snap, self.vm_name\n ).split()\n )\n subprocess.check_call(\n \"/snap/bin/multipass exec {} -- sudo \"\n \"snap install /var/tmp/microk8s.snap --classic --dangerous\".format(self.vm_name).split()\n )\n\n def run(self, cmd):\n \"\"\"\n Run a command\n :param cmd: the command we are running.\n :return: the output of the command\n \"\"\"\n if self.backend == \"multipass\":\n output = subprocess.check_output(\n \"/snap/bin/multipass exec {} -- sudo \" \"{}\".format(self.vm_name, cmd).split()\n )\n return output\n elif self.backend == \"lxc\":\n cmd_prefix = \"/snap/bin/lxc exec {} -- \".format(self.vm_name)\n with subprocess.Popen(\n cmd_prefix + cmd, shell=True, stdout=subprocess.PIPE, preexec_fn=os.setsid\n ) as process:\n try:\n output = process.communicate(timeout=300)[0]\n if process.returncode != 0:\n raise ChildProcessError(\"Failed to run command\")\n except subprocess.TimeoutExpired:\n os.killpg(process.pid, signal.SIGKILL) # send signal to the process group\n print(\"Process timed out\")\n output = process.communicate()[0]\n return output\n else:\n raise Exception(\"Not implemented for backend {}\".format(self.backend))\n\n def transfer_file(self, file_path, remote_path):\n \"\"\"\n Transfer a file to the VM.\n \"\"\"\n print(\"Transferring {} to {}\".format(file_path, remote_path))\n if self.backend == \"multipass\":\n subprocess.check_call(\n \"/snap/bin/multipass transfer {} {}:{} \".format(\n file_path, self.vm_name, remote_path\n ).split()\n )\n elif self.backend == \"lxc\":\n subprocess.check_call(\n \"/snap/bin/lxc file push {} {}{}\".format(\n file_path, self.vm_name, remote_path\n ).split()\n )\n\n def release(self):\n \"\"\"\n Release a VM.\n \"\"\"\n print(\"Destroying VM in {}\".format(self.backend))\n if self.backend == \"multipass\":\n subprocess.check_call(\"/snap/bin/multipass stop {}\".format(self.vm_name).split())\n subprocess.check_call(\"/snap/bin/multipass delete {}\".format(self.vm_name).split())\n elif self.backend == \"lxc\":\n subprocess.check_call(\"/snap/bin/lxc stop {}\".format(self.vm_name).split())\n subprocess.check_call(\"/snap/bin/lxc delete {}\".format(self.vm_name).split())\n\n\nclass TestCluster(object):\n @pytest.fixture(autouse=True, scope=\"module\")\n def setup_cluster(self):\n \"\"\"\n Provision VMs and for a cluster.\n :return:\n \"\"\"\n try:\n print(\"Setting up cluster\")\n type(self).VM = []\n if not reuse_vms:\n size = 3\n for i in range(0, size):\n print(\"Creating machine {}\".format(i))\n vm = VM(backend)\n vm.setup(channel_to_test)\n print(\"Waiting for machine {}\".format(i))\n vm.run(\"/snap/bin/microk8s.status --wait-ready --timeout 120\")\n self.VM.append(vm)\n else:\n for vm_name in reuse_vms:\n vm = VM(backend, vm_name)\n vm.setup(channel_to_test)\n self.VM.append(vm)\n\n # Form cluster\n vm_master = self.VM[0]\n connected_nodes = vm_master.run(\"/snap/bin/microk8s.kubectl get no\")\n for vm in self.VM:\n if vm.vm_name in connected_nodes.decode():\n continue\n else:\n print(\"Adding machine {} to cluster\".format(vm.vm_name))\n add_node = vm_master.run(\"/snap/bin/microk8s.add-node\")\n endpoint = [ep for ep in add_node.decode().split() if \":25000/\" in ep]\n vm.run(\"/snap/bin/microk8s.join {}\".format(endpoint[0]))\n\n # Wait for nodes to be ready\n print(\"Waiting for nodes to register\")\n attempt = 0\n while attempt < 10:\n try:\n connected_nodes = vm_master.run(\"/snap/bin/microk8s.kubectl get no\")\n if \"NotReady\" in connected_nodes.decode():\n time.sleep(5)\n connected_nodes = vm_master.run(\"/snap/bin/microk8s.kubectl get no\")\n print(connected_nodes.decode())\n break\n except ChildProcessError:\n time.sleep(10)\n attempt += 1\n if attempt == 10:\n raise\n\n # Wait for CNI pods\n print(\"Waiting for cni\")\n while True:\n ready_pods = 0\n pods = vm_master.run(\"/snap/bin/microk8s.kubectl get po -n kube-system -o wide\")\n for line in pods.decode().splitlines():\n if \"calico\" in line and \"Running\" in line:\n ready_pods += 1\n if ready_pods == (len(self.VM) + 1):\n print(pods.decode())\n break\n time.sleep(5)\n\n yield\n\n finally:\n print(\"Cleanup up cluster\")\n if not reuse_vms:\n for vm in self.VM:\n print(\"Releasing machine {} in {}\".format(vm.vm_name, vm.backend))\n vm.release()\n\n def test_calico_in_nodes(self):\n \"\"\"\n Test each node has a calico pod.\n \"\"\"\n print(\"Checking calico is in all nodes\")\n pods = self.VM[0].run(\"/snap/bin/microk8s.kubectl get po -n kube-system -o wide\")\n for vm in self.VM:\n if vm.vm_name not in pods.decode():\n assert False\n print(\"Calico found in node {}\".format(vm.vm_name))\n\n @pytest.mark.skipif(\n is_strict(),\n reason=\"Skipping because calico interfaces are not removed on strict\",\n )\n def test_calico_interfaces_removed_on_snap_remove(self):\n \"\"\"\n Test that calico interfaces are not present on the node\n when the microk8s snap is removed.\n \"\"\"\n vm = VM(backend)\n vm.setup(channel_to_test)\n print(\"Waiting for machine {}\".format(vm.vm_name))\n vm.run(\"/snap/bin/microk8s.status --wait-ready --timeout 240\")\n timeout = time.time() + 240\n ready = False\n while time.time() <= timeout and not ready:\n try:\n pods = vm.run(\"/snap/bin/microk8s.kubectl get po -n kube-system -o wide\")\n for line in pods.decode().splitlines():\n if \"calico\" in line and \"Running\" in line:\n ready = True\n except ChildProcessError:\n print(\"Waiting for k8s pods to come up\")\n time.sleep(5)\n assert ready\n vm.run(\"snap remove --purge microk8s\")\n interfaces = vm.run(\"/sbin/ip a\")\n assert \"cali\" not in interfaces.decode()\n vm.release()\n\n def test_nodes_in_ha(self):\n \"\"\"\n Test all nodes are seeing the database while removing nodes\n \"\"\"\n # All nodes see the same pods\n for vm in self.VM:\n pods = vm.run(\"/snap/bin/microk8s.kubectl get po -n kube-system -o wide\")\n for other_vm in self.VM:\n if other_vm.vm_name not in pods.decode():\n assert False\n print(\"All nodes see the same pods\")\n\n attempt = 100\n while True:\n assert attempt > 0\n for vm in self.VM:\n status = vm.run(\"/snap/bin/microk8s.status\")\n if \"high-availability: yes\" not in status.decode():\n attempt += 1\n continue\n break\n\n # remove a node\n print(\"Removing machine {}\".format(self.VM[0].vm_name))\n self.VM[0].run(\"/snap/bin/microk8s.leave\")\n self.VM[1].run(\"/snap/bin/microk8s.remove-node {}\".format(self.VM[0].vm_name))\n # allow for some time for the leader to hand over leadership\n time.sleep(10)\n attempt = 100\n while True:\n ready_pods = 0\n pods = self.VM[1].run(\"/snap/bin/microk8s.kubectl get po -n kube-system -o wide\")\n for line in pods.decode().splitlines():\n if \"calico\" in line and \"Running\" in line:\n ready_pods += 1\n if ready_pods == (len(self.VM)):\n print(pods.decode())\n break\n attempt -= 1\n if attempt <= 0:\n assert False\n time.sleep(5)\n print(\"Checking calico is on the nodes running\")\n\n leftVMs = [self.VM[1], self.VM[2]]\n attempt = 100\n while True:\n assert attempt > 0\n for vm in leftVMs:\n status = vm.run(\"/snap/bin/microk8s.status\")\n if \"high-availability: no\" not in status.decode():\n attempt += 1\n time.sleep(2)\n continue\n break\n\n for vm in leftVMs:\n pods = vm.run(\"/snap/bin/microk8s.kubectl get po -n kube-system -o wide\")\n for other_vm in leftVMs:\n if other_vm.vm_name not in pods.decode():\n time.sleep(2)\n assert False\n print(\"Remaining nodes see the same pods\")\n\n print(\"Waiting for two ingress to appear\")\n self.VM[1].run(\"/snap/bin/microk8s.enable ingress\")\n # wait for two ingress to appear\n time.sleep(10)\n attempt = 100\n while True:\n ready_pods = 0\n pods = self.VM[1].run(\"/snap/bin/microk8s.kubectl get po -A -o wide\")\n for line in pods.decode().splitlines():\n if \"ingress\" in line and \"Running\" in line:\n ready_pods += 1\n if ready_pods == (len(self.VM) - 1):\n print(pods.decode())\n break\n attempt -= 1\n if attempt <= 0:\n assert False\n time.sleep(5)\n\n print(\"Rejoin the node\")\n add_node = self.VM[1].run(\"/snap/bin/microk8s.add-node\")\n endpoint = [ep for ep in add_node.decode().split() if \":25000/\" in ep]\n self.VM[0].run(\"/snap/bin/microk8s.join {}\".format(endpoint[0]))\n\n print(\"Waiting for nodes to be ready\")\n attempt = 0\n while attempt < 10:\n try:\n connected_nodes = self.VM[0].run(\"/snap/bin/microk8s.kubectl get no\")\n if \"NotReady\" in connected_nodes.decode():\n time.sleep(5)\n continue\n print(connected_nodes.decode())\n break\n except ChildProcessError:\n time.sleep(10)\n attempt += 1\n if attempt == 10:\n raise\n\n attempt = 100\n while True:\n assert attempt > 0\n for vm in self.VM:\n status = vm.run(\"/snap/bin/microk8s.status\")\n if \"high-availability: yes\" not in status.decode():\n attempt += 1\n time.sleep(2)\n continue\n break\n\n def test_worker_node(self):\n \"\"\"\n Test a worker node is setup\n \"\"\"\n print(\"Setting up a worker node\")\n vm = VM(backend)\n vm.setup(channel_to_test)\n self.VM.append(vm)\n\n # Form cluster\n vm_master = self.VM[0]\n print(\"Adding machine {} to cluster\".format(vm.vm_name))\n add_node = vm_master.run(\"/snap/bin/microk8s.add-node\")\n endpoint = [ep for ep in add_node.decode().split() if \":25000/\" in ep]\n vm.run(\"/snap/bin/microk8s.join {} --worker\".format(endpoint[0]))\n ep_parts = endpoint[0].split(\":\")\n master_ip = ep_parts[0]\n\n # Wait for nodes to be ready\n print(\"Waiting for node to register\")\n attempt = 0\n while attempt < 10:\n try:\n connected_nodes = vm_master.run(\"/snap/bin/microk8s.kubectl get no\")\n if (\n \"NotReady\" in connected_nodes.decode()\n or vm.vm_name not in connected_nodes.decode()\n ):\n time.sleep(5)\n continue\n print(connected_nodes.decode())\n break\n except ChildProcessError:\n time.sleep(10)\n attempt += 1\n if attempt == 10:\n raise\n\n # Check that kubelet talks to the control plane node via the local proxy\n print(\"Checking the worker's configuration\")\n provider = vm.run(\"cat /var/snap/microk8s/current/args/traefik/provider.yaml\")\n assert master_ip in provider.decode()\n kubelet = vm.run(\"cat /var/snap/microk8s/current/credentials/kubelet.config\")\n assert \"127.0.0.1\" in kubelet.decode()\n\n # Leave the worker node from the cluster\n print(\"Leaving the worker node {} from the cluster\".format(vm.vm_name))\n vm.run(\"/snap/bin/microk8s.leave\")\n vm_master.run(\"/snap/bin/microk8s.remove-node {}\".format(vm.vm_name))\n\n # Wait for worker node to leave the cluster\n attempt = 0\n while attempt < 10:\n try:\n connected_nodes = vm_master.run(\"/snap/bin/microk8s.kubectl get no\")\n print(connected_nodes.decode())\n if \"NotReady\" in connected_nodes.decode() or vm.vm_name in connected_nodes.decode():\n time.sleep(5)\n continue\n print(connected_nodes.decode())\n break\n except ChildProcessError:\n time.sleep(10)\n attempt += 1\n if attempt == 10:\n raise\n\n # Check that the worker node is Ready\n print(\"Checking that the worker node {} is working and Ready\".format(vm.vm_name))\n worker_node = vm.run(\"/snap/bin/microk8s status --wait-ready\")\n print(worker_node.decode())\n assert \"microk8s is running\" in worker_node.decode()\n\n # The removed node now isn't part of the cluster and will re-issue certificates.\n # This will interfere when testing `test_no_cert_reissue_in_nodes`.\n # Hence, we remove this machine from the VM list.\n print(\"Remove machine {}\".format(vm.vm_name))\n self.VM.remove(vm)\n vm.release()\n\n def test_no_cert_reissue_in_nodes(self):\n \"\"\"\n Test that each node has the cert no-reissue lock.\n \"\"\"\n print(\"Checking for the no re-issue lock\")\n for vm in self.VM:\n lock_files = vm.run(\"ls /var/snap/microk8s/current/var/lock/\")\n assert \"no-cert-reissue\" in lock_files.decode()\n\n @pytest.mark.skipif(\n # If the host system does not have IPv6 support or is not configured for IPv6,\n # it won't be able to create VMs with IPv6 connectivity.\n not is_ipv6_configured,\n reason=\"Skipping dual stack tests on VMs which are not lxc based and not dual-stack enabled\",\n )\n def test_dual_stack_cluster(self):\n vm = self.VM[0]\n # Deploy the test deployment and service\n manifest = TEMPLATES / \"dual-stack.yaml\"\n remote_path = \"{}/tmp/dual-stack.yaml\".format(snap_data)\n vm.transfer_file(manifest, remote_path)\n vm.run(\"ls -al {}\".format(remote_path))\n vm.run(\"/snap/bin/microk8s.kubectl apply -f {}\".format(remote_path))\n\n # Wait for the deployment to become ready\n print(\"Waiting for nginx deployment\")\n while True:\n ready_pods = 0\n pods = vm.run(\"/snap/bin/microk8s.kubectl get po -o wide\")\n for line in pods.decode().splitlines():\n if \"nginxdualstack\" in line and \"Running\" in line:\n ready_pods = 1\n if ready_pods == 1:\n print(pods.decode())\n break\n time.sleep(5)\n\n # ping the service attached with the deployment\n ep = (\n \"/snap/bin/microk8s.kubectl get service nginx6 \"\n \"-o jsonpath='{.spec.clusterIP}' --output='jsonpath=['{.spec.clusterIP}']'\"\n )\n ipv6_endpoint = vm.run(ep).decode()\n print(\"Pinging endpoint: http://{}/\".format(ipv6_endpoint))\n url = f\"http://{ipv6_endpoint}/\"\n attempt = 10\n while attempt >= 0:\n try:\n resp = vm.run(\"curl {}\".format(url))\n if \"Kubernetes IPv6 nginx\" in resp.decode():\n print(resp)\n break\n except subprocess.CalledProcessError as e:\n print(\"Error occurred during the request:\", str(e))\n raise\n attempt -= 1\n time.sleep(2)\n\n\nclass TestUpgradeCluster(object):\n @pytest.fixture(autouse=True, scope=\"module\")\n def setup_old_versioned_cluster(self):\n \"\"\"\n Provision VMs of the previous version and form the cluster.\n \"\"\"\n try:\n print(\"Setting up cluster of an older version\")\n if channel_to_test.startswith(\"latest\") or \"/\" not in channel_to_test:\n attempt = 0\n release_url = \"https://dl.k8s.io/release/stable.txt\"\n while attempt < 10:\n try:\n r = requests.get(release_url)\n if r.status_code == 200:\n last_stable_str = r.content.decode().strip()\n last_stable_str = last_stable_str.replace(\"v\", \"\")\n last_stable_str = \".\".join(last_stable_str.split(\".\")[:2])\n break\n except TimeoutError:\n time.sleep(3)\n attempt += 1\n if attempt == 10:\n raise\n\n track, *_ = channel_to_test.split(\"/\")\n if track == \"latest\":\n track = last_stable_str\n\n # For eksd and stable tracks, we need a previous version on these tracks.\n # Eg, to test 1.24-eksd, we need 1.23-eksd track.\n major = track.split(\".\")[0]\n minor = track.split(\".\")[1].split(\"-\")[0]\n branch = track.split(\"-\")[1] if len(track.split(\"-\")) > 1 else \"\"\n\n if minor == \"0\" and major >= \"1\":\n major = str(int(major) - 1)\n minor = \"9\"\n else:\n if \"-\" in track:\n minor = str(int(minor.split(\"-\")[0]) - 1)\n else:\n minor = str(int(minor) - 1)\n\n branch = \"-\" + branch if branch != \"\" else \"\"\n older_version = major + \".\" + minor + branch + \"/\" + \"stable\"\n print(\"Old version is {}\".format(older_version))\n\n type(self).VM = []\n if not reuse_vms:\n print(\"Creating machine\")\n vm = VM(backend)\n vm.setup(older_version)\n print(\"Waiting for machine\")\n vm.run(\"/snap/bin/microk8s.status --wait-ready --timeout 120\")\n self.VM.append(vm)\n else:\n vm = VM(backend, reuse_vms[0])\n vm.setup(older_version)\n self.VM.append(vm)\n\n vm_older_version = self.VM[0]\n\n # Wait for CNI pods\n print(\"Waiting for cni\")\n while True:\n ready_pods = 0\n attempt = 0\n try:\n pods = vm_older_version.run(\n \"/snap/bin/microk8s.kubectl get po -n kube-system -o wide\"\n )\n for line in pods.decode().splitlines():\n if \"calico\" in line and \"Running\" in line:\n ready_pods += 1\n if ready_pods == (len(self.VM) + 1):\n print(pods.decode())\n break\n time.sleep(5)\n except ChildProcessError:\n time.sleep(10)\n attempt += 1\n if attempt == 10:\n raise\n time.sleep(5)\n yield\n\n finally:\n print(\"Cleanup up cluster\")\n if not reuse_vms:\n for vm in self.VM:\n print(\"Releasing machine {} in {}\".format(vm.vm_name, vm.backend))\n vm.release()\n\n @pytest.mark.skipif(\n is_strict() and backend == \"lxc\",\n reason=\"Skipping test of multi-version cluster on strict and lxc\",\n )\n def test_mixed_version_join(self):\n \"\"\"\n Test n versioned node joining a n-1 versioned cluster.\n \"\"\"\n print(\"Setting up an newer versioned node\")\n vm = VM(backend)\n vm.setup(channel_to_test)\n self.VM.append(vm)\n\n # Form cluster\n vm_older_version = self.VM[0]\n print(\"Adding newer versioned machine {} to cluster\".format(vm.vm_name))\n add_node = vm_older_version.run(\"/snap/bin/microk8s.add-node\")\n endpoint = [ep for ep in add_node.decode().split() if \":25000/\" in ep]\n vm.run(\"/snap/bin/microk8s.join {}\".format(endpoint[0]))\n\n # Wait for nodes to be ready\n print(\"Waiting for two node to be Ready\")\n attempt = 0\n timeout_insec = 300\n deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec)\n while attempt < 10:\n try:\n # Timeout after few minutes.\n if datetime.datetime.now() > deadline:\n raise TimeoutError(\n \"Nodes not in Ready state after {} seconds.\".format(timeout_insec)\n )\n\n connected_nodes = vm_older_version.run(\"/snap/bin/microk8s.kubectl get no\")\n num_nodes = connected_nodes.count(b\" Ready\")\n if num_nodes != 2:\n time.sleep(5)\n continue\n print(connected_nodes.decode())\n break\n except ChildProcessError:\n time.sleep(10)\n attempt += 1\n if attempt == 10:\n raise\n" }, { "path": "tests/test-distro.sh", "content": "#!/usr/bin/env bash\n\nsource tests/libs/utils.sh\n\nTEMP=$(getopt -o \"h\" \\\n --long help,distro:,from-channel:,to-channel:,proxy: \\\n -n \"$(basename \"$0\")\" -- \"$@\")\n\nif [ $? != 0 ] ; then echo \"Terminating...\" >&2 ; exit 1 ; fi\n\neval set -- \"$TEMP\"\n\nDISTRO=\"${DISTRO-}\"\nFROM_CHANNEL=\"${FROM_CHANNEL-}\"\nTO_CHANNEL=\"${TO_CHANNEL-}\"\nPROXY=\"${PROXY-}\"\n\nwhile true; do\n case \"$1\" in\n --distro ) DISTRO=\"$2\"; shift 2 ;;\n --from-channel ) FROM_CHANNEL=\"$2\"; shift 2 ;;\n --to-channel ) TO_CHANNEL=\"$2\"; shift 2 ;;\n --proxy ) PROXY=\"$2\"; shift 2 ;;\n -h | --help ) \n prog=$(basename -s.wrapper \"$0\")\n echo \"Usage: $prog [options...] \"\n echo \" --distro Distro image to be used for LXD containers Eg. ubuntu:18.04\"\n echo \" Can also be set by using DISTRO environment variable\"\n echo \" --from-channel Channel to upgrade from to the channel under testing Eg. latest/beta\"\n echo \" Can also be set by using FROM_CHANNEL environment variable\"\n echo \" --to-channel Channel to be tested Eg. latest/edge\"\n echo \" Can also be set by using TO_CHANNEL environment variable\"\n echo \" --proxy Proxy url to be used by the nodes\"\n echo \" Can also be set by using PROXY environment variable\"\n echo\n exit ;;\n -- ) shift; break ;;\n * ) break ;;\n esac\ndone\n\nset -uex\n\nsetup_tests \"$@\"\n\nDISABLE_AIRGAP_TESTS=\"${DISABLE_AIRGAP_TESTS:-0}\"\nif [ \"x${DISABLE_AIRGAP_TESTS}\" != \"x1\" ]; then\n . tests/libs/airgap.sh\nfi\n\n. tests/libs/clustering.sh\n\n. tests/libs/addons-upgrade.sh\n\n. tests/libs/upgrade-path.sh\n\n. tests/libs/addons.sh\n" }, { "path": "tests/test-simple.py", "content": "import subprocess\nimport time\nimport requests\nimport os.path\n\nimport utils\n\n\nclass TestSimple(object):\n def test_microk8s_nodes_ready(self):\n # Get MicroK8s node status\n output = subprocess.check_output([\"microk8s\", \"kubectl\", \"get\", \"nodes\"])\n\n # Split the output into lines and skip the header\n lines = output.decode(\"utf-8\").split(\"\\n\")[1:]\n\n # Iterate through the lines to check the node status\n for line in lines:\n # Split the line by whitespace\n parts = line.split()\n # Check if the node status is \"Ready\"\n if len(parts) > 1 and parts[1] != \"Ready\":\n # If any node is not in \"Ready\" status, fail the test\n assert False, f\"Node {parts[0]} is not in Ready status\"\n\n def test_calico_cni_pods_running(self):\n # Get Calico CNI pod status\n output = subprocess.check_output(\n [\"microk8s\", \"kubectl\", \"get\", \"pods\", \"-n\", \"kube-system\"]\n )\n\n # Split the output into lines and skip the header\n lines = output.decode(\"utf-8\").split(\"\\n\")[1:]\n\n # Iterate through the lines to check the Calico CNI pod status\n for line in lines:\n # Split the line by whitespace\n parts = line.split()\n # Check if the pod is related to Calico CNI and not in the \"Running\" state\n if len(parts) >= 4 and parts[0].startswith(\"calico\") and parts[2] != \"Running\":\n # If any Calico CNI pod is not in the \"Running\" state, fail the test\n assert False, f\"Calico CNI pod {parts[0]} is not running\"\n\n def test_nginx_ingress(self):\n # Create Ingress resource for the Nginx pod\n subprocess.run(\n [\"microk8s\", \"kubectl\", \"apply\", \"-f\", \"tests/templates/simple-deploy.yaml\"], check=True\n )\n\n # Wait for the pod to be in a ready state\n subprocess.run(\n [\n \"microk8s\",\n \"kubectl\",\n \"rollout\",\n \"status\",\n \"deployment\",\n \"nginx-deployment\",\n \"-n\",\n \"default\",\n \"--timeout=90s\",\n ],\n check=True,\n )\n\n # Send a curl request to the Ingress IP\n if \"core/ingress: enabled\" not in subprocess.check_output(\n [\"microk8s\", \"status\", \"--format\", \"short\"]\n ).decode(\"utf-8\"):\n output = subprocess.check_output(\n [\n \"microk8s\",\n \"kubectl\",\n \"get\",\n \"svc\",\n \"nginx-service\",\n \"-o\",\n \"jsonpath={.spec.clusterIP}\",\n ]\n )\n response = requests.get(f\"http://{output.decode('utf-8')}:80\", timeout=15)\n else:\n # Wait for ingress to be ready\n time.sleep(3)\n response = requests.get(\"http://127.0.0.1:80\", timeout=15)\n\n subprocess.run(\n [\"microk8s\", \"kubectl\", \"delete\", \"-f\", \"tests/templates/simple-deploy.yaml\"],\n check=True,\n )\n\n # Verify the HTTP status code is 200\n assert response.status_code == 200\n\n def test_microk8s_services_running(self):\n # Define the services to check for control plane and worker nodes\n node_services = [\n \"snap.microk8s.daemon-kubelite.service\",\n \"snap.microk8s.daemon-containerd.service\",\n \"snap.microk8s.daemon-cluster-agent.service\",\n ]\n control_plane_services = [\n \"snap.microk8s.daemon-apiserver-kicker.service\",\n \"snap.microk8s.daemon-k8s-dqlite.service\",\n ]\n worker_node_services = [\"snap.microk8s.daemon-apiserver-proxy.service\"]\n\n if os.path.exists(\"/var/snap/microk8s/current/var/lock/clustered.lock\"):\n node_services += worker_node_services\n else:\n node_services += control_plane_services\n\n # Get the list of running systemd services\n output = subprocess.check_output([\"systemctl\", \"list-units\", \"--state=running\"])\n\n # Split the output into lines and skip the header\n lines = output.decode(\"utf-8\").split(\"\\n\")[1:]\n\n # Create sets to store the running services for control plane and worker nodes\n running_node_services = set()\n\n # Iterate through the lines to check the running services\n for line in lines:\n # Split the line by whitespace\n parts = line.split()\n\n # Check if the line contains a running service\n if len(parts) >= 2 and parts[0].endswith(\".service\"):\n service_name = parts[0]\n\n # Check if the running service is in the control plane services list\n if service_name in node_services:\n running_node_services.add(service_name)\n\n # Verify that all node services are running\n assert running_node_services == set(node_services), \"Not all node services are running\"\n\n def test_microk8s_stop_start(self):\n coredns_procs = utils._get_process(\"coredns\")\n assert len(coredns_procs) > 0, \"Expected to find a coredns process running.\"\n\n utils.run_until_success(\"/snap/bin/microk8s.stop\", timeout_insec=180)\n\n new_coredns_procs = utils._get_process(\"coredns\")\n assert len(new_coredns_procs) == 0, \"coredns found still running after microk8s stop.\"\n\n utils.run_until_success(\"/snap/bin/microk8s.start\", timeout_insec=180)\n\n new_coredns_procs = utils._get_process(\"coredns\")\n assert len(new_coredns_procs) > 0, \"Expected to find a new coredns process running.\"\n" }, { "path": "tests/test-upgrade-path.py", "content": "import pytest\nimport os\nimport time\nimport requests\nfrom utils import (\n wait_for_installation,\n run_until_success,\n is_strict,\n)\n\nupgrade_from = os.environ.get(\"UPGRADE_MICROK8S_FROM\", \"beta\")\n# Have UPGRADE_MICROK8S_TO point to a file to upgrade to that file\nupgrade_to = os.environ.get(\"UPGRADE_MICROK8S_TO\", \"edge\")\n\n\nclass TestUpgradePath(object):\n \"\"\"\n Validates a microk8s upgrade path\n \"\"\"\n\n @pytest.mark.skipif(\n os.environ.get(\"UNDER_TIME_PRESSURE\", \"\").lower() == \"true\",\n reason=\"Skipping refresh path test as we are under time pressure\",\n )\n def test_refresh_path(self):\n \"\"\"\n Deploy an old snap and try to refresh until the current one.\n\n \"\"\"\n start_channel = 24\n if is_strict():\n start_channel = 25\n last_stable_minor = None\n if upgrade_from.startswith(\"latest\") or \"/\" not in upgrade_from:\n attempt = 0\n release_url = \"https://dl.k8s.io/release/stable.txt\"\n while attempt < 10 and not last_stable_minor:\n r = requests.get(release_url)\n if r.status_code == 200:\n last_stable_str = r.content.decode().strip()\n # We have \"v1.18.4\" and we need the \"18\"\n last_stable_parts = last_stable_str.split(\".\")\n last_stable_minor = int(last_stable_parts[1])\n else:\n time.sleep(3)\n attempt += 1\n else:\n channel_parts = upgrade_from.split(\".\")\n channel_parts = channel_parts[1].split(\"/\")\n if is_strict():\n channel_parts = channel_parts[0].split(\"-\")\n print(channel_parts)\n last_stable_minor = int(channel_parts[0])\n\n last_stable_minor -= 1\n\n print(\"\")\n print(\n \"Testing refresh path from 1.{} to 1.{} and finally refresh to {}\".format(\n \"{}-strict\".format(start_channel) if is_strict() else start_channel,\n \"{}-strict\".format(last_stable_minor) if is_strict() else last_stable_minor,\n upgrade_to,\n )\n )\n assert last_stable_minor is not None\n\n channel = \"1.{}/stable\".format(\n \"{}-strict\".format(start_channel) if is_strict() else start_channel\n )\n print(\"Installing {}\".format(channel))\n cmd = \"sudo snap install microk8s --channel={} {}\".format(\n channel, \"\" if is_strict() else \"--classic\"\n )\n run_until_success(cmd)\n wait_for_installation()\n channel_minor = start_channel\n channel_minor += 1\n while channel_minor <= last_stable_minor:\n channel = \"1.{}/stable\".format(\n \"{}-strict\".format(channel_minor) if is_strict() else channel_minor\n )\n print(\"Refreshing to {}\".format(channel))\n cmd = \"sudo snap refresh microk8s --channel={} {}\".format(\n channel, \"\" if is_strict() else \"--classic\"\n )\n run_until_success(cmd)\n wait_for_installation()\n time.sleep(30)\n channel_minor += 1\n\n print(\"Installing {}\".format(upgrade_to))\n if upgrade_to.endswith(\".snap\"):\n cmd = \"sudo snap install {} --dangerous {}\".format(\n upgrade_to, \"\" if is_strict() else \"--classic\"\n )\n else:\n cmd = \"sudo snap refresh microk8s --channel={}\".format(upgrade_to)\n run_until_success(cmd, timeout_insec=600)\n # Allow for the refresh to be processed\n time.sleep(20)\n wait_for_installation(timeout_insec=1200)\n" }, { "path": "tests/test-upgrade.py", "content": "import os\nimport platform\nimport time\nfrom validators import (\n validate_dns_dashboard,\n validate_storage,\n validate_ingress,\n validate_registry,\n validate_forward,\n validate_metrics_server,\n validate_metallb_config,\n validate_dual_stack,\n)\nfrom subprocess import check_call, CalledProcessError\nfrom utils import (\n microk8s_enable,\n wait_for_pod_state,\n wait_for_installation,\n run_until_success,\n is_container,\n is_ipv6_configured,\n kubectl,\n _get_process,\n)\n\nupgrade_from = os.environ.get(\"UPGRADE_MICROK8S_FROM\", \"beta\")\n# Have UPGRADE_MICROK8S_TO point to a file to upgrade to that file\nupgrade_to = os.environ.get(\"UPGRADE_MICROK8S_TO\", \"edge\")\nunder_time_pressure = os.environ.get(\"UNDER_TIME_PRESSURE\", \"false\").lower()\n\n\nclass TestUpgrade(object):\n \"\"\"\n Validates a microk8s upgrade path\n \"\"\"\n\n def test_upgrade(self):\n \"\"\"\n Deploy, probe, upgrade, validate nothing broke.\n\n \"\"\"\n print(\"Testing upgrade from {} to {}\".format(upgrade_from, upgrade_to))\n if is_ipv6_configured:\n print(\"IPv6 is configured, will test dual stack\")\n launch_config = \"\"\"---\nversion: 0.1.0\nextraCNIEnv:\n IPv4_SUPPORT: true\n IPv4_CLUSTER_CIDR: 10.3.0.0/16\n IPv4_SERVICE_CIDR: 10.153.183.0/24\n IPv6_SUPPORT: true\n IPv6_CLUSTER_CIDR: fd02::/64\n IPv6_SERVICE_CIDR: fd99::/108\nextraSANs:\n - 10.153.183.1\"\"\"\n lc_config_dir = \"/var/snap/microk8s/common/\"\n if not os.path.exists(lc_config_dir):\n os.makedirs(lc_config_dir)\n\n file_path = os.path.join(lc_config_dir, \".microk8s.yaml\")\n with open(file_path, \"w\") as file:\n file.write(launch_config)\n\n cmd = \"sudo snap install microk8s --classic --channel={}\".format(upgrade_from)\n run_until_success(cmd)\n wait_for_installation()\n\n if is_ipv6_configured:\n kubectl(\"set env daemonset/calico-node -n kube-system IP=10.3.0.0/16 IP6=fd02::/64\")\n\n # Run through the validators and\n # select those that were valid for the original snap\n test_matrix = {}\n try:\n enable = microk8s_enable(\"dns\")\n wait_for_pod_state(\"\", \"kube-system\", \"running\", label=\"k8s-app=kube-dns\")\n assert \"Nothing to do for\" not in enable\n enable = microk8s_enable(\"dashboard\")\n assert \"Nothing to do for\" not in enable\n validate_dns_dashboard()\n test_matrix[\"dns_dashboard\"] = validate_dns_dashboard\n except CalledProcessError:\n print(\"Will not test dns-dashboard\")\n\n try:\n enable = microk8s_enable(\"storage\")\n assert \"Nothing to do for\" not in enable\n validate_storage()\n test_matrix[\"storage\"] = validate_storage\n except CalledProcessError:\n print(\"Will not test storage\")\n\n try:\n enable = microk8s_enable(\"ingress\")\n assert \"Nothing to do for\" not in enable\n validate_ingress()\n test_matrix[\"ingress\"] = validate_ingress\n except CalledProcessError:\n print(\"Will not test ingress\")\n\n try:\n enable = microk8s_enable(\"registry\")\n assert \"Nothing to do for\" not in enable\n validate_registry()\n test_matrix[\"registry\"] = validate_registry\n except CalledProcessError:\n print(\"Will not test registry\")\n\n try:\n validate_forward()\n test_matrix[\"forward\"] = validate_forward\n except CalledProcessError:\n print(\"Will not test port forward\")\n\n try:\n enable = microk8s_enable(\"metrics-server\")\n assert \"Nothing to do for\" not in enable\n validate_metrics_server()\n test_matrix[\"metrics_server\"] = validate_metrics_server\n except CalledProcessError:\n print(\"Will not test the metrics server\")\n\n # AMD64 only tests\n if platform.machine() == \"x86_64\" and under_time_pressure == \"false\":\n try:\n ip_ranges = (\n \"192.168.0.105-192.168.0.105,192.168.0.110-192.168.0.111,192.168.1.240/28\"\n )\n enable = microk8s_enable(\"{}:{}\".format(\"metallb\", ip_ranges), timeout_insec=500)\n assert \"MetalLB is enabled\" in enable and \"Nothing to do for\" not in enable\n validate_metallb_config(ip_ranges)\n test_matrix[\"metallb\"] = validate_metallb_config\n except CalledProcessError:\n print(\"Will not test the metallb addon\")\n\n if is_ipv6_configured:\n try:\n validate_dual_stack()\n test_matrix[\"dual_stack\"] = validate_dual_stack\n except CalledProcessError:\n print(\"Will not test the dual stack configuration\")\n\n # Refresh the snap to the target\n if upgrade_to.endswith(\".snap\"):\n cmd = \"sudo snap install {} --classic --dangerous\".format(upgrade_to)\n else:\n cmd = \"sudo snap refresh microk8s --channel={}\".format(upgrade_to)\n run_until_success(cmd)\n # Allow for the refresh to be processed\n time.sleep(10)\n wait_for_installation()\n\n # Test any validations that were valid for the original snap\n for test, validation in test_matrix.items():\n print(\"Testing {}\".format(test))\n validation()\n\n if not is_container():\n # On lxc umount docker overlay is not permitted.\n check_call(\"sudo snap remove microk8s\".split())\n coredns_procs = _get_process(\"coredns\")\n assert len(coredns_procs) == 0, \"Expected to have 0 coredns processes running.\"\n" }, { "path": "tests/unit/cluster/test_join.py", "content": "import os\nfrom shutil import rmtree\nfrom unittest import mock\n\nimport join\nfrom click.testing import CliRunner\nfrom join import join as command\n\n\ndef test_command_help_arguments():\n runner = CliRunner()\n for help_arg in (\"-h\", \"--help\"):\n result = runner.invoke(command, [help_arg])\n assert result.exit_code == 0\n assert \"Join the node to a cluster\" in result.output\n\n\ndef test_command_errors_if_no_arguments():\n runner = CliRunner()\n result = runner.invoke(command, [])\n assert result.exit_code != 0\n assert \"Error: Missing argument\" in result.output\n\n\n@mock.patch(\"subprocess.check_call\")\n@mock.patch(\"os.chown\")\n@mock.patch(\"os.chmod\")\n@mock.patch(\"subprocess.check_output\")\n@mock.patch(\"time.sleep\")\ndef test_join_dqlite_master_node(\n mock_sleep,\n mock_subprocess_check_output,\n mock_chmod,\n mock_chown,\n mock_subprocess_check_call,\n tmp_path,\n):\n \"\"\"\n Test the join operation. Create a directory layout and let the join operation work with it.\n \"\"\"\n snapcommon = tmp_path / \"snapcommon\"\n snapdata = tmp_path / \"snapdata\" / \"rev-123\"\n snapdatacurrnet = tmp_path / \"snapdata\" / \"current\"\n snap = tmp_path / \"snap\"\n args = snapdata / \"args\"\n certs = snapdata / \"certs\"\n credentials = snapdata / \"credentials\"\n dqlite = snapdata / \"var\" / \"kubernetes\" / \"backend\"\n\n join.snapdata_path = snapdata\n join.snap_path = snap\n join.cluster_dir = f\"{snapdata}/var/kubernetes/backend\"\n join.cluster_backup_dir = f\"{snapdata}/var/kubernetes/backend.backup\"\n join.cluster_cert_file = f\"{join.cluster_dir}/cluster.crt\"\n join.cluster_key_file = f\"{join.cluster_dir}/cluster.key\"\n\n os.environ[\"SNAP\"] = str(snap)\n os.environ[\"SNAP_DATA\"] = str(snapdata)\n os.environ[\"SNAP_COMMON\"] = str(snapcommon)\n\n def create_dir_layout(tokens):\n for d in [\n snapcommon,\n snapdata,\n dqlite,\n snap / \"meta\",\n args / \"cni-network\",\n snapdata / \"var\" / \"lock\",\n certs,\n credentials,\n ]:\n if os.path.exists(d):\n rmtree(d, ignore_errors=True)\n os.makedirs(d, exist_ok=True)\n\n if os.path.exists(snapdatacurrnet):\n os.remove(snapdatacurrnet)\n snapdatacurrnet\n os.symlink(snapdata, snapdatacurrnet)\n for cert in [\"ca\", \"client\", \"controller\", \"proxy\", \"scheduler\", \"kubelet\"]:\n with open(certs / f\"{cert}.crt\", \"w\") as ca:\n ca.write(f\"{cert}_data\")\n with open(certs / f\"{cert}.key\", \"w\") as ca:\n ca.write(f\"{cert}_key_data\")\n\n with open(snap / \"meta\" / \"snap.yaml\", \"w\") as f:\n f.write(\"confinement: classic\")\n with open(certs / \"serviceaccount.key\", \"w\") as f:\n f.write(\"service_account_key_data\")\n with open(args / \"kube-apiserver\", \"w\") as f:\n f.write(\"--some-argument\")\n with open(dqlite / \"info.yaml\", \"w\") as f:\n f.write(\"Address: 127.0.0.1:19001\\nID: 3297041220608546238\\nRole: 0\\n\")\n with open(args / \"cni-network\" / \"cni.yaml\", \"w\") as f:\n f.write(\"can-reach\\n\")\n with open(snap / \"client.config.template\", \"w\") as f:\n f.write(\"token\\nUSERNAME\")\n with open(snap / \"client-x509.config.template\", \"w\") as f:\n f.write(\"x509\\nUSERNAME\")\n for config in [\n \"client.config\",\n \"controller.config\",\n \"proxy.config\",\n \"scheduler.config\",\n \"kubelet.config\",\n ]:\n with open(credentials / config, \"w\") as f:\n f.write(\"kubeconfig\")\n\n create_dir_layout(tokens=False)\n\n info = {\n \"hostname_override\": \"no-host\",\n \"ca\": \"new_ca_bytes\",\n \"ca_key\": \"new_ca_key_bytes\",\n \"service_account_key\": \"srv_account_key_bytes\",\n \"kubelet_args\": \"--some-kubelet-arg\",\n \"callback_token\": \"123callback456\",\n \"cluster_cert\": \"dqlite_cert\",\n \"cluster_key\": \"dqlite_cert_key\",\n \"voters\": \"none\",\n }\n\n join.join_dqlite_master_node(info, \"123.123.123.123\")\n\n # Assert we stored the CA and the dqlite certs\n mock_chmod.assert_any_call(str(certs / \"ca.crt\"), 0o660)\n mock_chmod.assert_any_call(str(certs / \"ca.key\"), 0o660)\n mock_chmod.assert_any_call(str(dqlite / \"cluster.crt\"), 0o660)\n mock_chmod.assert_any_call(str(dqlite / \"cluster.key\"), 0o660)\n\n # Assert we restart services\n mock_subprocess_check_call.assert_any_call(\"snapctl stop microk8s.daemon-kubelite\".split())\n mock_subprocess_check_call.assert_any_call(\"snapctl start microk8s.daemon-kubelite\".split())\n mock_subprocess_check_call.assert_any_call(\"snapctl stop microk8s.daemon-k8s-dqlite\".split())\n mock_subprocess_check_call.assert_any_call(\"snapctl start microk8s.daemon-k8s-dqlite\".split())\n\n # Assert we created admin kubeconfig from certificate\n mock_subprocess_check_call.assert_any_call(\n [f\"{snap}/actions/common/utils.sh\", \"create_user_certs_and_configs\"], stdout=-3, stderr=-3\n )\n\n mock_subprocess_check_call.reset_mock()\n\n # Check joining with tokens based\n create_dir_layout(tokens=True)\n mock_subprocess_check_call.reset_mock()\n info[\"admin_token\"] = \"some-token\"\n join.join_dqlite_master_node(info, \"123.123.123.123\")\n\n mock_subprocess_check_call.assert_any_call(\n [f\"{snap}/actions/common/utils.sh\", \"create_user_certs_and_configs\"], stdout=-3, stderr=-3\n )\n" }, { "path": "tests/unit/cluster/test_leave.py", "content": "from click.testing import CliRunner\nfrom leave import leave as command\n\n\ndef test_command_help_arguments():\n runner = CliRunner()\n for help_arg in (\"-h\", \"--help\"):\n result = runner.invoke(command, [help_arg])\n assert result.exit_code == 0\n assert \"The node will depart from the cluster it is in\" in result.output\n" }, { "path": "tests/unit/test_addons.py", "content": "import os\nimport stat\nimport shutil\nfrom contextlib import contextmanager\nfrom copy import deepcopy\nfrom pathlib import Path\nfrom unittest.mock import mock_open, patch, Mock\n\nimport pytest\nimport yaml\n\nfrom addons import (\n AddonsYamlFormatError,\n AddonsYamlNotFoundError,\n MissingHookError,\n WrongHookPermissionsError,\n add,\n get_addons_list,\n load_addons_yaml,\n update,\n validate_addons_file,\n validate_addons_repo,\n)\nfrom common.utils import parse_xable_addon_args, get_available_addons\n\n\nADDONS = [\n (\"core\", \"addon1\"),\n (\"core\", \"addon2\"),\n (\"community\", \"addon3\"),\n (\"core\", \"conflict\"),\n (\"community\", \"conflict\"),\n]\n\n\nREPO_YAML = \"\"\"\nmicrok8s-addons:\n description: \"List of all addons included in Microk8s.\"\n addons:\n - name: \"dns\"\n description: \"CoreDNS\"\n version: \"1.9.0\"\n check_status: \"pod/coredns\"\n supported_architectures:\n - amd64\n - arm64\n - s390x\n\n - name: \"rbac\"\n description: \"Role-Based Access Control for authorisation\"\n version: \"\"\n check_status: \"clusterrole.rbac.authorization.k8s.io/cluster-admin\"\n confinement: \"strict\"\n supported_architectures:\n - arm64\n - amd64\n - s390x\n\n - name: \"dashboard\"\n description: \"The Kubernetes dashboard\"\n version: \"2.3.0\"\n check_status: \"pod/kubernetes-dashboard\"\n confinement: \"classic\"\n supported_architectures:\n - arm64\n - amd64\n - s390x\n\"\"\"\n\n\n@pytest.mark.parametrize(\n \"confinement, result\", [(True, [\"dns\", \"rbac\"]), (False, [\"dns\", \"dashboard\"])]\n)\n@patch(\"common.utils.is_strict\")\n@patch(\"os.listdir\")\n@patch(\"common.utils.snap_common\")\ndef test_get_available_addons(snap_common_mock, listdir_mock, strict_mock, confinement, result):\n strict_mock.return_value = confinement\n listdir_mock.return_value = [\"/a/path/\"]\n snap_common_mock.return_value = Path(\"/common_dir\")\n\n with patch(\"common.utils.open\", mock_open(read_data=REPO_YAML)):\n addons = get_available_addons(\"amd64\")\n assert len(addons) == len(result)\n for addon in addons:\n assert addon[\"name\"] in result\n\n\n@pytest.mark.parametrize(\n \"args, result\",\n [\n ([\"addon1\"], [(\"core\", \"addon1\", [])]),\n ([\"core/conflict\"], [(\"core\", \"conflict\", [])]),\n ([\"community/conflict\"], [(\"community\", \"conflict\", [])]),\n ([\"community/conflict\", \"--with-arg\"], [(\"community\", \"conflict\", [\"--with-arg\"])]),\n ([\"addon1\", \"--with-arg\"], [(\"core\", \"addon1\", [\"--with-arg\"])]),\n (\n [\"addon1:arg1\", \"addon2:arg2\", \"addon3\"],\n [\n (\"core\", \"addon1\", [\"arg1\"]),\n (\"core\", \"addon2\", [\"arg2\"]),\n (\"community\", \"addon3\", []),\n ],\n ),\n (\n [\"addon1:arg1\", \"addon2:arg2\", \"community/conflict\"],\n [\n (\"core\", \"addon1\", [\"arg1\"]),\n (\"core\", \"addon2\", [\"arg2\"]),\n (\"community\", \"conflict\", []),\n ],\n ),\n (\n [\"core/addon1:arg1\", \"addon2:arg2\", \"community/conflict:arg3\"],\n [\n (\"core\", \"addon1\", [\"arg1\"]),\n (\"core\", \"addon2\", [\"arg2\"]),\n (\"community\", \"conflict\", [\"arg3\"]),\n ],\n ),\n ],\n)\ndef test_parse_addons_args(args, result):\n addons = parse_xable_addon_args(args, ADDONS)\n assert addons == result\n\n\nTEST_ADDON_NAME = \"foo\"\nVALID_ADDONS = {\n \"microk8s-addons\": {\n \"addons\": [\n {\n \"name\": TEST_ADDON_NAME,\n \"description\": \"bar\",\n \"version\": \"1.1.1\",\n \"check_status\": \"foobar\",\n \"supported_architectures\": [\"arm64\", \"amd64\"],\n }\n ]\n }\n}\nINVALID_ADDONS = deepcopy(VALID_ADDONS)\n# Remove one of the required properties of an addon in the addons list\nINVALID_ADDONS[\"microk8s-addons\"][\"addons\"][0].pop(\"check_status\")\n\n\n@patch(\"addons.load_addons_yaml\", return_value=VALID_ADDONS)\ndef test_validate_addons_file(load_addons_mock):\n validate_addons_file(Mock())\n\n\n@patch(\"addons.load_addons_yaml\", return_value=INVALID_ADDONS)\ndef test_validate_addons_file_raises_if_invalid_format(load_addons_mock):\n with pytest.raises(AddonsYamlFormatError) as err:\n validate_addons_file(Mock())\n assert \"Invalid addons.yaml file\" in err.value.message\n\n\ndef test_load_addons_raises_on_file_not_found():\n # When addons.yaml file is not found\n with pytest.raises(AddonsYamlNotFoundError) as exc:\n load_addons_yaml(Path(\"/some/repo\"))\n assert exc.value.message == \"Error: repository repo does not contain an addons.yaml file\"\n\n\ndef test_load_addons_raises_on_invalid_yaml_contents(repo_dir):\n with patch(\"addons.open\", mock_open(read_data=\"unbalanced blackets: ][\")):\n with pytest.raises(AddonsYamlFormatError) as exc:\n load_addons_yaml(repo_dir)\n assert (\n exc.value.message\n == \"Yaml format error in addons.yaml file: while parsing a block node expected the node content, but found ']'\" # noqa\n )\n\n\ndef test_get_addons_list():\n with patch(\"addons.open\", mock_open(read_data=yaml.dump(VALID_ADDONS))):\n assert get_addons_list(Path(\"/some/path\")) == [TEST_ADDON_NAME]\n\n\ndef test_validate_addons_repo(repo_dir):\n validate_addons_repo(repo_dir)\n\n\ndef test_validate_addons_repo_raises_on_missing_enable_hook(addon_missing_enable_hook):\n with pytest.raises(MissingHookError) as err:\n validate_addons_repo(addon_missing_enable_hook)\n assert err.value.message == \"Missing enable hook for foo addon\"\n\n\ndef test_validate_addons_repo_raises_on_missing_disable_hook(addon_missing_disable_hook):\n with pytest.raises(MissingHookError) as err:\n validate_addons_repo(addon_missing_disable_hook)\n assert err.value.message == \"Missing disable hook for foo addon\"\n\n\ndef test_validate_addons_repo_raises_on_enable_not_executable(enable_not_executable):\n with pytest.raises(WrongHookPermissionsError) as err:\n validate_addons_repo(enable_not_executable)\n assert err.value.message == \"enable hook for foo addon needs execute permissions\"\n\n\ndef test_validate_addons_repo_raises_on_disable_not_executable(disable_not_executable):\n with pytest.raises(WrongHookPermissionsError) as err:\n validate_addons_repo(disable_not_executable)\n assert err.value.message == \"disable hook for foo addon needs execute permissions\"\n\n\n@patch(\"addons.subprocess\")\n@patch(\"addons.snap_common\", return_value=Path(\"/tmp/\"))\n@patch(\"addons.get_group\", return_value=\"microk8s\")\n@patch(\"addons.validate_addons_repo\", side_effect=AddonsYamlFormatError(\"foo\"))\n@patch(\"addons.shutil.rmtree\")\ndef test_add_removes_repo_on_validation_error(\n rm_mock,\n validate_addons_repo_mock,\n get_group_mock,\n snap_common_mock,\n subprocess_mock,\n):\n with pytest.raises(SystemExit):\n add.callback(\"myrepo\", \"http://github.com/me/myrepo\", None, False)\n\n repo_dir = Path(\"/tmp/addons/myrepo\")\n validate_addons_repo_mock.assert_called_once_with(repo_dir)\n rm_mock.assert_called_once_with(repo_dir)\n\n\n@pytest.fixture(scope=\"function\")\ndef repo_dir(tmp_path):\n repo_dir = tmp_path / \"myrepo\"\n repo_dir.mkdir()\n addons_yaml = repo_dir / \"addons.yaml\"\n addons_yaml.write_text(yaml.dump(VALID_ADDONS))\n\n addons = repo_dir / \"addons\"\n addons.mkdir()\n myaddon = addons / TEST_ADDON_NAME\n myaddon.mkdir()\n\n everyone_can_exec = stat.S_IXGRP | stat.S_IXUSR | stat.S_IEXEC\n\n enable = myaddon / \"enable\"\n enable.write_text(\"echo 1\")\n enable.chmod(everyone_can_exec)\n\n disable = myaddon / \"disable\"\n disable.write_text(\"echo 0\")\n disable.chmod(everyone_can_exec)\n\n yield repo_dir\n\n\n@pytest.fixture(scope=\"function\")\ndef invalid_addons_yaml(tmp_path):\n repo_dir = tmp_path / \"invalid_repo\"\n repo_dir.mkdir()\n addons_yaml = repo_dir / \"addons.yaml\"\n addons_yaml.write_text(yaml.dump(INVALID_ADDONS))\n yield repo_dir\n\n\n@pytest.fixture(scope=\"function\")\ndef missing_addons_yaml(tmp_path):\n repo_dir = tmp_path / \"invalid_repo\"\n repo_dir.mkdir()\n yield repo_dir\n\n\n@pytest.fixture(scope=\"function\")\ndef addon_missing_enable_hook(repo_dir):\n addon_dir = repo_dir / \"addons\" / TEST_ADDON_NAME\n os.remove(addon_dir / \"enable\")\n yield repo_dir\n\n\n@pytest.fixture(scope=\"function\")\ndef addon_missing_disable_hook(repo_dir):\n addon_dir = repo_dir / \"addons\" / TEST_ADDON_NAME\n os.remove(addon_dir / \"disable\")\n yield repo_dir\n\n\n@pytest.fixture(scope=\"function\")\ndef enable_not_executable(repo_dir):\n addon_dir = repo_dir / \"addons\" / TEST_ADDON_NAME\n (addon_dir / \"enable\").chmod(stat.S_IREAD)\n yield repo_dir\n\n\n@pytest.fixture(scope=\"function\")\ndef disable_not_executable(repo_dir):\n addon_dir = repo_dir / \"addons\" / TEST_ADDON_NAME\n (addon_dir / \"disable\").chmod(stat.S_IREAD)\n yield repo_dir\n\n\n@contextmanager\ndef create_test_repo(repo_name: str):\n # Create temporary dummy test git repository\n addons = Path(\"/tmp/addons\")\n os.mkdir(Path(addons))\n repo_dir = addons / repo_name\n os.mkdir(Path(repo_dir))\n\n # Create the .git file\n with open(repo_dir / \".git\", mode=\"w+\"):\n pass\n\n yield repo_dir\n\n # Cleanup\n shutil.rmtree(Path(addons))\n\n\n@patch(\"addons.git_rollback\")\n@patch(\"addons.git_current_commit\")\n@patch(\"addons.subprocess\")\n@patch(\"addons.snap_common\", return_value=Path(\"/tmp/\"))\n@patch(\"addons.validate_addons_repo\", side_effect=AddonsYamlFormatError(\"foo\"))\ndef test_update_rollbacks_repo_on_validation_error(\n validate_addons_repo_mock,\n snap_common_mock,\n subprocess_mock,\n git_current_commit_mock,\n git_rollback_mock,\n):\n with create_test_repo(\"repo_to_update\") as repo_dir:\n\n with pytest.raises(SystemExit):\n update.callback(\"repo_to_update\", skip_check_root=True)\n\n validate_addons_repo_mock.assert_called_once_with(repo_dir)\n git_current_commit_mock.assert_called_once_with(repo_dir)\n git_rollback_mock.assert_called_once_with(git_current_commit_mock.return_value, repo_dir)\n" }, { "path": "tests/unit/test_addtoken.py", "content": "from unittest import mock\nfrom add_token import print_short\n\n\ndef test_single(capsys):\n with mock.patch(\n \"add_token.get_network_info\", return_value=[\"10.23.53.54\", [\"10.23.53.54\"], \"32\"]\n ):\n print_short(\"t\", \"c\")\n captured = capsys.readouterr()\n output = captured.out.strip()\n assert output == \"microk8s join 10.23.53.54:32/t/c\"\n\n\ndef test_multiple(capsys):\n with mock.patch(\n \"add_token.get_network_info\", return_value=[\"d_ip\", [\"ip1\", \"ip2\", \"d_ip\"], \"4\"]\n ):\n print_short(\"t\", \"c\")\n captured = capsys.readouterr()\n all_outputs = captured.out.strip().split(\"\\n\")\n assert all_outputs == [\n \"microk8s join d_ip:4/t/c\",\n \"microk8s join ip1:4/t/c\",\n \"microk8s join ip2:4/t/c\",\n ]\n" }, { "path": "tests/unit/test_dashboard_proxy.py", "content": "from click.testing import CliRunner\nfrom dashboard_proxy import dashboard_proxy as command\n\n\ndef test_command_help_arguments():\n runner = CliRunner()\n for help_arg in (\"-h\", \"--help\"):\n result = runner.invoke(command, [help_arg])\n assert result.exit_code == 0\n assert \"Enable the dashboard add-on and configures port-forwarding\" in result.output\n" }, { "path": "tests/unit/test_disable.py", "content": "from click.testing import CliRunner\nfrom disable import disable as command\nfrom unittest.mock import patch\n\n\ndef test_command_help_arguments():\n runner = CliRunner()\n for help_arg in (\"-h\", \"--help\"):\n result = runner.invoke(command, [help_arg])\n assert result.exit_code == 0\n assert \"Disable one or more MicroK8s addons\" in result.output\n\n\ndef test_command_errors_if_no_arguments():\n runner = CliRunner()\n result = runner.invoke(command, [])\n assert result.exit_code != 0\n assert \"Error: Missing argument\" in result.output\n\n\n@patch(\"disable.xable\")\ndef test_command_shows_addon_help_message(xable_mock):\n runner = CliRunner()\n for help_flag in (\"-h\", \"--help\"):\n result = runner.invoke(command, [\"dns\", \"--\", help_flag])\n assert result.output.startswith(\"Addon dns does not yet have a help message.\")\n xable_mock.assert_not_called()\n" }, { "path": "tests/unit/test_enable.py", "content": "from click.testing import CliRunner\nfrom enable import enable as command\nfrom unittest.mock import patch\n\n\ndef test_command_help_arguments():\n runner = CliRunner()\n for help_arg in (\"-h\", \"--help\"):\n result = runner.invoke(command, [help_arg])\n assert result.exit_code == 0\n assert \"Enable a MicroK8s addon\" in result.output\n\n\ndef test_command_errors_if_no_arguments():\n runner = CliRunner()\n result = runner.invoke(command, [])\n assert result.exit_code != 0\n assert \"Error: Missing argument\" in result.output\n\n\n@patch(\"enable.xable\")\ndef test_command_shows_addon_help_message(xable_mock):\n runner = CliRunner()\n for help_flag in (\"-h\", \"--help\"):\n result = runner.invoke(command, [\"dns\", \"--\", help_flag])\n assert result.output.startswith(\"Addon dns does not yet have a help message.\")\n xable_mock.assert_not_called()\n" }, { "path": "tests/unit/test_refresh_certs.py", "content": "import os\nimport pytest\nfrom unittest.mock import patch, call, Mock\nfrom click.testing import CliRunner\n\nfrom refresh_certs import (\n refresh_certs,\n restart,\n reproduce_all_root_ca_certs,\n reproduce_front_proxy_client_cert,\n reproduce_server_cert,\n)\n\n\nclass TestRefreshCerts(object):\n @patch(\"subprocess.check_call\")\n def test_restart(self, mock_check_call):\n restart()\n # We stop and start microk8s\n assert mock_check_call.call_count == 2\n\n @patch(\"common.cluster.utils.get_arg\")\n @patch(\"subprocess.Popen\")\n @patch(\"subprocess.check_call\")\n def test_reproduce_all_root_ca_certs(self, mock_check_call, mock_subproc_popen, mock_get_arg):\n process_mock = Mock()\n mock_subproc_popen.return_value = process_mock\n mock_get_arg.return_value = \"known_tokens.csv\"\n\n reproduce_all_root_ca_certs()\n\n \"\"\"\n Make sure we:\n 1. remove the ca.crt\n 2. call produce_certs\n 3. call update_configs that also restarts microk8s\n \"\"\"\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n assert (\n call(\"rm -rf {}/certs/ca.crt\".format(snapdata_path).split())\n in mock_check_call.call_args_list\n )\n assert mock_check_call.called\n assert mock_subproc_popen.called\n assert self.is_argument_in_call(mock_subproc_popen, \"produce_certs\")\n assert self.is_argument_in_call(mock_subproc_popen, \"update_configs\")\n\n @patch(\"refresh_certs.restart\")\n @patch(\"subprocess.check_call\")\n def test_reproduce_front_proxy_client_cert(self, mock_check_call, mock_restart):\n \"\"\"\n Make sure we:\n 1. remove the front-proxy-client.crt\n 2. call gen_proxy_client_cert\n 3. restart microk8s\n \"\"\"\n reproduce_front_proxy_client_cert()\n\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n cmd = \"rm -rf {}/certs/front-proxy-client.crt\".format(snapdata_path).split()\n assert call(cmd) in mock_check_call.call_args_list\n assert mock_check_call.called\n assert mock_restart.called\n assert self.is_argument_in_call(mock_check_call, \"gen_proxy_client_cert\")\n\n @patch(\"refresh_certs.restart\")\n @patch(\"subprocess.check_call\")\n def test_reproduce_server_cert(self, mock_check_call, mock_restart):\n \"\"\"\n Make sure we:\n 1. remove the server.crt\n 2. call gen_server_cert\n 3. restart microk8s\n \"\"\"\n reproduce_server_cert()\n\n snapdata_path = os.environ.get(\"SNAP_DATA\")\n assert (\n call(\"rm -rf {}/certs/server.crt\".format(snapdata_path).split())\n in mock_check_call.call_args_list\n )\n assert mock_check_call.called\n assert mock_restart.called\n assert self.is_argument_in_call(mock_check_call, \"gen_server_cert\")\n\n def is_argument_in_call(self, mock_function, argument_substring):\n \"\"\"Search for a substring in the list of arguments in all calls of a mocked function\"\"\"\n for calls in mock_function.call_args_list:\n # calls is the list of calls\n for arglist in calls.args:\n # list of arguments in call\n for arg in arglist:\n if argument_substring in arg:\n return True\n return False\n\n @pytest.mark.parametrize(\n \"ca_dir,undo,check,cert,help,expected_output,expected_err_code\",\n [\n (None, None, True, \"ca.crt\", None, \"Please select only one of the options\", 2),\n (None, True, True, None, None, \"Please select only one of the options\", 2),\n (None, True, None, \"ca.crt\", None, \"Please select only one of the options\", 2),\n (None, True, None, \"ca.crt\", True, \"Usage:\", 0),\n (\"/some/path\", True, None, None, None, \"does not exist\", 2),\n (\"/\", True, None, None, None, \"options in combination\", 1),\n (None, True, None, \"wrong_file\", None, \"Invalid value\", 2),\n ],\n )\n def test_refresh_cert_errors(\n self, ca_dir, undo, check, cert, help, expected_output, expected_err_code\n ):\n \"\"\"\n Test conditions under which the upgrade should not continue\n \"\"\"\n runner = CliRunner()\n args = []\n if ca_dir:\n args.append(ca_dir)\n if undo:\n args.append(\"-u\")\n if check:\n args.append(\"-c\")\n if cert:\n args.append(\"-e\")\n args.append(cert)\n if help:\n args.append(\"-h\")\n result = runner.invoke(refresh_certs, args)\n assert expected_output in result.output\n assert result.exit_code == expected_err_code\n\n\ndef test_command_help_arguments():\n runner = CliRunner()\n for help_arg in (\"-h\", \"--help\"):\n result = runner.invoke(refresh_certs, [help_arg])\n assert result.exit_code == 0\n assert \"Replace the CA certificates with the ca.crt and ca.key\" in result.output\n" }, { "path": "tests/unit/test_reset.py", "content": "from click.testing import CliRunner\nfrom reset import reset as command\n\n\ndef test_command_help_arguments():\n runner = CliRunner()\n for help_arg in (\"-h\", \"--help\"):\n result = runner.invoke(command, [help_arg])\n assert result.exit_code == 0\n assert \"Return the MicroK8s node to the default initial state\" in result.output\n" }, { "path": "tests/unit/test_upgrade_calico_cni.py", "content": "import os\nimport shutil\n\nfrom calico.upgrade import (\n try_upgrade,\n get_installed_version_of_calico,\n get_calicos_autodetection_method,\n mark_apply_needed,\n)\n\n\nclass TestCNIUpgrade(object):\n def setup_class(self):\n dirname, filename = os.path.split(os.path.abspath(__file__))\n self._invalid_yaml = os.path.join(dirname, \"yamls/invalid.yaml\")\n self._calico_new_yaml = os.path.join(dirname, \"yamls/calico-new.yaml\")\n self._calico_old_yaml = os.path.join(dirname, \"yamls/cni.yaml\")\n self._calico_old_copy_yaml = os.path.join(dirname, \"yamls/cni.yaml.copy\")\n self._calico_old_copy_backup_yaml = os.path.join(dirname, \"yamls/cni.yaml.copy.backup\")\n self._lock_file = os.path.join(dirname, \"yamls/lock_file\")\n self._cni_no_manage_file = os.path.join(dirname, \"yamls/cni_no_manage\")\n\n def test_no_op(self):\n \"\"\"\n Test conditions under which the upgrade should not continue\n \"\"\"\n res = try_upgrade(\"foo\", \"bar\")\n assert res is False\n res = try_upgrade(self._calico_new_yaml, self._calico_new_yaml)\n assert res is False\n res = try_upgrade(self._invalid_yaml, self._calico_new_yaml)\n assert res is False\n shutil.copyfile(self._invalid_yaml, self._cni_no_manage_file)\n res = try_upgrade(self._calico_new_yaml, self._calico_new_yaml, self._cni_no_manage_file)\n assert res is False\n os.remove(self._cni_no_manage_file)\n\n def test_get_version(self):\n \"\"\"\n Test extracting the Calico version\n \"\"\"\n res = get_installed_version_of_calico(self._calico_new_yaml)\n assert res == \"v3.23.4\"\n\n def test_get_autodetect_method(self):\n \"\"\"\n Test extracting the IP autodetection method\n \"\"\"\n res = get_calicos_autodetection_method(self._calico_new_yaml)\n assert res == \"first-found\"\n\n def test_patch(self):\n \"\"\"\n Test patching the manifest\n \"\"\"\n shutil.copyfile(self._calico_old_yaml, self._calico_old_copy_yaml)\n res = get_calicos_autodetection_method(self._calico_old_copy_yaml)\n assert res == \"can-reach=192.168.1.43\"\n res = get_calicos_autodetection_method(self._calico_new_yaml)\n assert res == \"first-found\"\n res = get_installed_version_of_calico(self._calico_old_copy_yaml)\n assert res == \"v3.21.1\"\n res = try_upgrade(self._calico_old_copy_yaml, self._calico_new_yaml)\n assert os.path.exists(self._calico_old_copy_backup_yaml)\n res = get_installed_version_of_calico(self._calico_old_copy_yaml)\n assert res == \"v3.23.4\"\n os.remove(self._calico_old_copy_yaml)\n os.remove(self._calico_old_copy_backup_yaml)\n\n def test_mark(self):\n \"\"\"\n Test marking the need for reapplying the manifest\n \"\"\"\n shutil.copyfile(self._calico_old_yaml, self._lock_file)\n mark_apply_needed(self._lock_file)\n assert not os.path.exists(self._lock_file)\n" }, { "path": "tests/unit/test_version.py", "content": "from version import get_snap_version, get_snap_revision\nimport os\n\n\ndef test_get_snap_version():\n os.environ[\"SNAP_VERSION\"] = \"v1.21.11\"\n assert get_snap_version() == \"v1.21.11\"\n\n\ndef test_get_snap_revision():\n os.environ[\"SNAP_REVISION\"] = \"3086\"\n assert get_snap_revision() == \"3086\"\n" }, { "path": "tests/unit/yamls/calico-new.yaml", "content": "---\n# Source: calico/templates/calico-config.yaml\n# This ConfigMap is used to configure a self-hosted Calico installation.\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-config\n namespace: kube-system\ndata:\n # Typha is disabled.\n typha_service_name: \"none\"\n # Configure the backend to use.\n calico_backend: \"vxlan\"\n\n # Configure the MTU to use for workload interfaces and tunnels.\n # By default, MTU is auto-detected, and explicitly setting this field should not be required.\n # You can override auto-detection by providing a non-zero value.\n veth_mtu: \"0\"\n\n # The CNI network configuration to install on each node. The special\n # values in this config will be automatically populated.\n cni_network_config: |-\n {\n \"name\": \"k8s-pod-network\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": [\n {\n \"type\": \"calico\",\n \"nodename_file_optional\": true,\n \"log_level\": \"info\",\n \"log_file_path\": \"/var/log/calico/cni/cni.log\",\n \"datastore_type\": \"kubernetes\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n \"mtu\": __CNI_MTU__,\n \"ipam\": {\n \"type\": \"calico-ipam\"\n },\n \"policy\": {\n \"type\": \"k8s\"\n },\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n }\n },\n {\n \"type\": \"portmap\",\n \"snat\": true,\n \"capabilities\": {\"portMappings\": true}\n },\n {\n \"type\": \"bandwidth\",\n \"capabilities\": {\"bandwidth\": true}\n }\n ]\n }\n\n---\n# Source: calico/templates/kdd-crds.yaml\n\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: bgpconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPConfiguration\n listKind: BGPConfigurationList\n plural: bgpconfigurations\n singular: bgpconfiguration\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: BGPConfiguration contains the configuration for any BGP routing.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPConfigurationSpec contains the values of the BGP configuration.\n properties:\n asNumber:\n description: 'ASNumber is the default AS number used by a node. [Default:\n 64512]'\n format: int32\n type: integer\n communities:\n description: Communities is a list of BGP community values and their\n arbitrary names for tagging routes.\n items:\n description: Community contains standard or large community value\n and its name.\n properties:\n name:\n description: Name given to community value.\n type: string\n value:\n description: Value must be of format `aa:nn` or `aa:nn:mm`.\n For standard community use `aa:nn` format, where `aa` and\n `nn` are 16 bit number. For large community use `aa:nn:mm`\n format, where `aa`, `nn` and `mm` are 32 bit number. Where,\n `aa` is an AS Number, `nn` and `mm` are per-AS identifier.\n pattern: ^(\\d+):(\\d+)$|^(\\d+):(\\d+):(\\d+)$\n type: string\n type: object\n type: array\n listenPort:\n description: ListenPort is the port where BGP protocol should listen.\n Defaults to 179\n maximum: 65535\n minimum: 1\n type: integer\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: INFO]'\n type: string\n nodeToNodeMeshEnabled:\n description: 'NodeToNodeMeshEnabled sets whether full node to node\n BGP mesh is enabled. [Default: true]'\n type: boolean\n prefixAdvertisements:\n description: PrefixAdvertisements contains per-prefix advertisement\n configuration.\n items:\n description: PrefixAdvertisement configures advertisement properties\n for the specified CIDR.\n properties:\n cidr:\n description: CIDR for which properties should be advertised.\n type: string\n communities:\n description: Communities can be list of either community names\n already defined in `Specs.Communities` or community value\n of format `aa:nn` or `aa:nn:mm`. For standard community use\n `aa:nn` format, where `aa` and `nn` are 16 bit number. For\n large community use `aa:nn:mm` format, where `aa`, `nn` and\n `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and\n `mm` are per-AS identifier.\n items:\n type: string\n type: array\n type: object\n type: array\n serviceClusterIPs:\n description: ServiceClusterIPs are the CIDR blocks from which service\n cluster IPs are allocated. If specified, Calico will advertise these\n blocks, as well as any cluster IPs within them.\n items:\n description: ServiceClusterIPBlock represents a single allowed ClusterIP\n CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n serviceExternalIPs:\n description: ServiceExternalIPs are the CIDR blocks for Kubernetes\n Service External IPs. Kubernetes Service ExternalIPs will only be\n advertised if they are within one of these blocks.\n items:\n description: ServiceExternalIPBlock represents a single allowed\n External IP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n serviceLoadBalancerIPs:\n description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes\n Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress\n IPs will only be advertised if they are within one of these blocks.\n items:\n description: ServiceLoadBalancerIPBlock represents a single allowed\n LoadBalancer IP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: bgppeers.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPPeer\n listKind: BGPPeerList\n plural: bgppeers\n singular: bgppeer\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPPeerSpec contains the specification for a BGPPeer resource.\n properties:\n asNumber:\n description: The AS Number of the peer.\n format: int32\n type: integer\n keepOriginalNextHop:\n description: Option to keep the original nexthop field when routes\n are sent to a BGP Peer. Setting \"true\" configures the selected BGP\n Peers node to use the \"next hop keep;\" instead of \"next hop self;\"(default)\n in the specific branch of the Node on \"bird.cfg\".\n type: boolean\n maxRestartTime:\n description: Time to allow for software restart. When specified,\n this is configured as the graceful restart timeout. When not specified,\n the BIRD default of 120s is used.\n type: string\n node:\n description: The node name identifying the Calico node instance that\n is targeted by this peer. If this is not set, and no nodeSelector\n is specified, then this BGP peer selects all nodes in the cluster.\n type: string\n nodeSelector:\n description: Selector for the nodes that should have this peering. When\n this is set, the Node field must be empty.\n type: string\n password:\n description: Optional BGP password for the peerings generated by this\n BGPPeer resource.\n properties:\n secretKeyRef:\n description: Selects a key of a secret in the node pod's namespace.\n properties:\n key:\n description: The key of the secret to select from. Must be\n a valid secret key.\n type: string\n name:\n description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Add other useful fields. apiVersion, kind, uid?'\n type: string\n optional:\n description: Specify whether the Secret or its key must be\n defined\n type: boolean\n required:\n - key\n type: object\n type: object\n peerIP:\n description: The IP address of the peer followed by an optional port\n number to peer with. If port number is given, format should be `[]:port`\n or `:` for IPv4. If optional port number is not set,\n and this peer IP and ASNumber belongs to a calico/node with ListenPort\n set in BGPConfiguration, then we use that port to peer.\n type: string\n peerSelector:\n description: Selector for the remote nodes to peer with. When this\n is set, the PeerIP and ASNumber fields must be empty. For each\n peering between the local node and selected remote nodes, we configure\n an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified,\n and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The\n remote AS number comes from the remote node's NodeBGPSpec.ASNumber,\n or the global default if that is not set.\n type: string\n sourceAddress:\n description: Specifies whether and how to configure a source address\n for the peerings generated by this BGPPeer resource. Default value\n \"UseNodeIP\" means to configure the node IP as the source address. \"None\"\n means not to configure a source address.\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: blockaffinities.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BlockAffinity\n listKind: BlockAffinityList\n plural: blockaffinities\n singular: blockaffinity\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BlockAffinitySpec contains the specification for a BlockAffinity\n resource.\n properties:\n cidr:\n type: string\n deleted:\n description: Deleted indicates that this block affinity is being deleted.\n This field is a string for compatibility with older releases that\n mistakenly treat this field as a string.\n type: string\n node:\n type: string\n state:\n type: string\n required:\n - cidr\n - deleted\n - node\n - state\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: caliconodestatuses.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: CalicoNodeStatus\n listKind: CalicoNodeStatusList\n plural: caliconodestatuses\n singular: caliconodestatus\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus\n resource.\n properties:\n classes:\n description: Classes declares the types of information to monitor\n for this calico/node, and allows for selective status reporting\n about certain subsets of information.\n items:\n type: string\n type: array\n node:\n description: The node name identifies the Calico node instance for\n node status.\n type: string\n updatePeriodSeconds:\n description: UpdatePeriodSeconds is the period at which CalicoNodeStatus\n should be updated. Set to 0 to disable CalicoNodeStatus refresh.\n Maximum update period is one day.\n format: int32\n type: integer\n type: object\n status:\n description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus.\n No validation needed for status since it is updated by Calico.\n properties:\n agent:\n description: Agent holds agent status on the node.\n properties:\n birdV4:\n description: BIRDV4 represents the latest observed status of bird4.\n properties:\n lastBootTime:\n description: LastBootTime holds the value of lastBootTime\n from bird.ctl output.\n type: string\n lastReconfigurationTime:\n description: LastReconfigurationTime holds the value of lastReconfigTime\n from bird.ctl output.\n type: string\n routerID:\n description: Router ID used by bird.\n type: string\n state:\n description: The state of the BGP Daemon.\n type: string\n version:\n description: Version of the BGP daemon\n type: string\n type: object\n birdV6:\n description: BIRDV6 represents the latest observed status of bird6.\n properties:\n lastBootTime:\n description: LastBootTime holds the value of lastBootTime\n from bird.ctl output.\n type: string\n lastReconfigurationTime:\n description: LastReconfigurationTime holds the value of lastReconfigTime\n from bird.ctl output.\n type: string\n routerID:\n description: Router ID used by bird.\n type: string\n state:\n description: The state of the BGP Daemon.\n type: string\n version:\n description: Version of the BGP daemon\n type: string\n type: object\n type: object\n bgp:\n description: BGP holds node BGP status.\n properties:\n numberEstablishedV4:\n description: The total number of IPv4 established bgp sessions.\n type: integer\n numberEstablishedV6:\n description: The total number of IPv6 established bgp sessions.\n type: integer\n numberNotEstablishedV4:\n description: The total number of IPv4 non-established bgp sessions.\n type: integer\n numberNotEstablishedV6:\n description: The total number of IPv6 non-established bgp sessions.\n type: integer\n peersV4:\n description: PeersV4 represents IPv4 BGP peers status on the node.\n items:\n description: CalicoNodePeer contains the status of BGP peers\n on the node.\n properties:\n peerIP:\n description: IP address of the peer whose condition we are\n reporting.\n type: string\n since:\n description: Since the state or reason last changed.\n type: string\n state:\n description: State is the BGP session state.\n type: string\n type:\n description: Type indicates whether this peer is configured\n via the node-to-node mesh, or via en explicit global or\n per-node BGPPeer object.\n type: string\n type: object\n type: array\n peersV6:\n description: PeersV6 represents IPv6 BGP peers status on the node.\n items:\n description: CalicoNodePeer contains the status of BGP peers\n on the node.\n properties:\n peerIP:\n description: IP address of the peer whose condition we are\n reporting.\n type: string\n since:\n description: Since the state or reason last changed.\n type: string\n state:\n description: State is the BGP session state.\n type: string\n type:\n description: Type indicates whether this peer is configured\n via the node-to-node mesh, or via en explicit global or\n per-node BGPPeer object.\n type: string\n type: object\n type: array\n required:\n - numberEstablishedV4\n - numberEstablishedV6\n - numberNotEstablishedV4\n - numberNotEstablishedV6\n type: object\n lastUpdated:\n description: LastUpdated is a timestamp representing the server time\n when CalicoNodeStatus object last updated. It is represented in\n RFC3339 form and is in UTC.\n format: date-time\n nullable: true\n type: string\n routes:\n description: Routes reports routes known to the Calico BGP daemon\n on the node.\n properties:\n routesV4:\n description: RoutesV4 represents IPv4 routes on the node.\n items:\n description: CalicoNodeRoute contains the status of BGP routes\n on the node.\n properties:\n destination:\n description: Destination of the route.\n type: string\n gateway:\n description: Gateway for the destination.\n type: string\n interface:\n description: Interface for the destination\n type: string\n learnedFrom:\n description: LearnedFrom contains information regarding\n where this route originated.\n properties:\n peerIP:\n description: If sourceType is NodeMesh or BGPPeer, IP\n address of the router that sent us this route.\n type: string\n sourceType:\n description: Type of the source where a route is learned\n from.\n type: string\n type: object\n type:\n description: Type indicates if the route is being used for\n forwarding or not.\n type: string\n type: object\n type: array\n routesV6:\n description: RoutesV6 represents IPv6 routes on the node.\n items:\n description: CalicoNodeRoute contains the status of BGP routes\n on the node.\n properties:\n destination:\n description: Destination of the route.\n type: string\n gateway:\n description: Gateway for the destination.\n type: string\n interface:\n description: Interface for the destination\n type: string\n learnedFrom:\n description: LearnedFrom contains information regarding\n where this route originated.\n properties:\n peerIP:\n description: If sourceType is NodeMesh or BGPPeer, IP\n address of the router that sent us this route.\n type: string\n sourceType:\n description: Type of the source where a route is learned\n from.\n type: string\n type: object\n type:\n description: Type indicates if the route is being used for\n forwarding or not.\n type: string\n type: object\n type: array\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: clusterinformations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: ClusterInformation\n listKind: ClusterInformationList\n plural: clusterinformations\n singular: clusterinformation\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: ClusterInformation contains the cluster specific information.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: ClusterInformationSpec contains the values of describing\n the cluster.\n properties:\n calicoVersion:\n description: CalicoVersion is the version of Calico that the cluster\n is running\n type: string\n clusterGUID:\n description: ClusterGUID is the GUID of the cluster\n type: string\n clusterType:\n description: ClusterType describes the type of the cluster\n type: string\n datastoreReady:\n description: DatastoreReady is used during significant datastore migrations\n to signal to components such as Felix that it should wait before\n accessing the datastore.\n type: boolean\n variant:\n description: Variant declares which variant of Calico should be active.\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: felixconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: FelixConfiguration\n listKind: FelixConfigurationList\n plural: felixconfigurations\n singular: felixconfiguration\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: Felix Configuration contains the configuration for Felix.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: FelixConfigurationSpec contains the values of the Felix configuration.\n properties:\n allowIPIPPacketsFromWorkloads:\n description: 'AllowIPIPPacketsFromWorkloads controls whether Felix\n will add a rule to drop IPIP encapsulated traffic from workloads\n [Default: false]'\n type: boolean\n allowVXLANPacketsFromWorkloads:\n description: 'AllowVXLANPacketsFromWorkloads controls whether Felix\n will add a rule to drop VXLAN encapsulated traffic from workloads\n [Default: false]'\n type: boolean\n awsSrcDstCheck:\n description: 'Set source-destination-check on AWS EC2 instances. Accepted\n value must be one of \"DoNothing\", \"Enable\" or \"Disable\". [Default:\n DoNothing]'\n enum:\n - DoNothing\n - Enable\n - Disable\n type: string\n bpfConnectTimeLoadBalancingEnabled:\n description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode,\n controls whether Felix installs the connection-time load balancer. The\n connect-time load balancer is required for the host to be able to\n reach Kubernetes services and it improves the performance of pod-to-service\n connections. The only reason to disable it is for debugging purposes. [Default:\n true]'\n type: boolean\n bpfDataIfacePattern:\n description: BPFDataIfacePattern is a regular expression that controls\n which interfaces Felix should attach BPF programs to in order to\n catch traffic to/from the network. This needs to match the interfaces\n that Calico workload traffic flows over as well as any interfaces\n that handle incoming traffic to nodeports and services from outside\n the cluster. It should not match the workload interfaces (usually\n named cali...).\n type: string\n bpfDisableUnprivileged:\n description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled\n sysctl to disable unprivileged use of BPF. This ensures that unprivileged\n users cannot access Calico''s BPF maps and cannot insert their own\n BPF programs to interfere with Calico''s. [Default: true]'\n type: boolean\n bpfEnabled:\n description: 'BPFEnabled, if enabled Felix will use the BPF dataplane.\n [Default: false]'\n type: boolean\n bpfExtToServiceConnmark:\n description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit\n mark that is set on connections from an external client to a local\n service. This mark allows us to control how packets of that connection\n are routed within the host and how is routing interpreted by RPF\n check. [Default: 0]'\n type: integer\n bpfExternalServiceMode:\n description: 'BPFExternalServiceMode in BPF mode, controls how connections\n from outside the cluster to services (node ports and cluster IPs)\n are forwarded to remote workloads. If set to \"Tunnel\" then both\n request and response traffic is tunneled to the remote node. If\n set to \"DSR\", the request traffic is tunneled but the response traffic\n is sent directly from the remote node. In \"DSR\" mode, the remote\n node appears to use the IP of the ingress node; this requires a\n permissive L2 network. [Default: Tunnel]'\n type: string\n bpfKubeProxyEndpointSlicesEnabled:\n description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls\n whether Felix's embedded kube-proxy accepts EndpointSlices or not.\n type: boolean\n bpfKubeProxyIptablesCleanupEnabled:\n description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF\n mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s\n iptables chains. Should only be enabled if kube-proxy is not running. [Default:\n true]'\n type: boolean\n bpfKubeProxyMinSyncPeriod:\n description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the\n minimum time between updates to the dataplane for Felix''s embedded\n kube-proxy. Lower values give reduced set-up latency. Higher values\n reduce Felix CPU usage by batching up more work. [Default: 1s]'\n type: string\n bpfLogLevel:\n description: 'BPFLogLevel controls the log level of the BPF programs\n when in BPF dataplane mode. One of \"Off\", \"Info\", or \"Debug\". The\n logs are emitted to the BPF trace pipe, accessible with the command\n `tc exec bpf debug`. [Default: Off].'\n type: string\n chainInsertMode:\n description: 'ChainInsertMode controls whether Felix hooks the kernel''s\n top-level iptables chains by inserting a rule at the top of the\n chain or by appending a rule at the bottom. insert is the safe default\n since it prevents Calico''s rules from being bypassed. If you switch\n to append mode, be sure that the other rules in the chains signal\n acceptance by falling through to the Calico rules, otherwise the\n Calico policy will be bypassed. [Default: insert]'\n type: string\n dataplaneDriver:\n type: string\n debugDisableLogDropping:\n type: boolean\n debugMemoryProfilePath:\n type: string\n debugSimulateCalcGraphHangAfter:\n type: string\n debugSimulateDataplaneHangAfter:\n type: string\n defaultEndpointToHostAction:\n description: 'DefaultEndpointToHostAction controls what happens to\n traffic that goes from a workload endpoint to the host itself (after\n the traffic hits the endpoint egress policy). By default Calico\n blocks traffic from workload endpoints to the host itself with an\n iptables \"DROP\" action. If you want to allow some or all traffic\n from endpoint to host, set this parameter to RETURN or ACCEPT. Use\n RETURN if you have your own rules in the iptables \"INPUT\" chain;\n Calico will insert its rules at the top of that chain, then \"RETURN\"\n packets to the \"INPUT\" chain once it has completed processing workload\n endpoint egress policy. Use ACCEPT to unconditionally accept packets\n from workloads after processing workload endpoint egress policy.\n [Default: Drop]'\n type: string\n deviceRouteProtocol:\n description: This defines the route protocol added to programmed device\n routes, by default this will be RTPROT_BOOT when left blank.\n type: integer\n deviceRouteSourceAddress:\n description: This is the source address to use on programmed device\n routes. By default the source address is left blank, leaving the\n kernel to choose the source address used.\n type: string\n disableConntrackInvalidCheck:\n type: boolean\n endpointReportingDelay:\n type: string\n endpointReportingEnabled:\n type: boolean\n externalNodesList:\n description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes\n which may source tunnel traffic and have the tunneled traffic be\n accepted at calico nodes.\n items:\n type: string\n type: array\n failsafeInboundHostPorts:\n description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports\n and CIDRs that Felix will allow incoming traffic to host endpoints\n on irrespective of the security policy. This is useful to avoid\n accidentally cutting off a host with incorrect configuration. For\n back-compatibility, if the protocol is not specified, it defaults\n to \"tcp\". If a CIDR is not specified, it will allow traffic from\n all addresses. To disable all inbound host ports, use the value\n none. The default value allows ssh access and DHCP. [Default: tcp:22,\n udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]'\n items:\n description: ProtoPort is combination of protocol, port, and CIDR.\n Protocol and port must be specified.\n properties:\n net:\n type: string\n port:\n type: integer\n protocol:\n type: string\n required:\n - port\n - protocol\n type: object\n type: array\n failsafeOutboundHostPorts:\n description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports\n and CIDRs that Felix will allow outgoing traffic from host endpoints\n to irrespective of the security policy. This is useful to avoid\n accidentally cutting off a host with incorrect configuration. For\n back-compatibility, if the protocol is not specified, it defaults\n to \"tcp\". If a CIDR is not specified, it will allow traffic from\n all addresses. To disable all outbound host ports, use the value\n none. The default value opens etcd''s standard ports to ensure that\n Felix does not get cut off from etcd as well as allowing DHCP and\n DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666,\n tcp:6667, udp:53, udp:67]'\n items:\n description: ProtoPort is combination of protocol, port, and CIDR.\n Protocol and port must be specified.\n properties:\n net:\n type: string\n port:\n type: integer\n protocol:\n type: string\n required:\n - port\n - protocol\n type: object\n type: array\n featureDetectOverride:\n description: FeatureDetectOverride is used to override the feature\n detection. Values are specified in a comma separated list with no\n spaces, example; \"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\".\n \"true\" or \"false\" will force the feature, empty or omitted values\n are auto-detected.\n type: string\n genericXDPEnabled:\n description: 'GenericXDPEnabled enables Generic XDP so network cards\n that don''t support XDP offload or driver modes can use XDP. This\n is not recommended since it doesn''t provide better performance\n than iptables. [Default: false]'\n type: boolean\n healthEnabled:\n type: boolean\n healthHost:\n type: string\n healthPort:\n type: integer\n interfaceExclude:\n description: 'InterfaceExclude is a comma-separated list of interfaces\n that Felix should exclude when monitoring for host endpoints. The\n default value ensures that Felix ignores Kubernetes'' IPVS dummy\n interface, which is used internally by kube-proxy. If you want to\n exclude multiple interface names using a single value, the list\n supports regular expressions. For regular expressions you must wrap\n the value with ''/''. For example having values ''/^kube/,veth1''\n will exclude all interfaces that begin with ''kube'' and also the\n interface ''veth1''. [Default: kube-ipvs0]'\n type: string\n interfacePrefix:\n description: 'InterfacePrefix is the interface name prefix that identifies\n workload endpoints and so distinguishes them from host endpoint\n interfaces. Note: in environments other than bare metal, the orchestrators\n configure this appropriately. For example our Kubernetes and Docker\n integrations set the ''cali'' value, and our OpenStack integration\n sets the ''tap'' value. [Default: cali]'\n type: string\n interfaceRefreshInterval:\n description: InterfaceRefreshInterval is the period at which Felix\n rescans local interfaces to verify their state. The rescan can be\n disabled by setting the interval to 0.\n type: string\n ipipEnabled:\n type: boolean\n ipipMTU:\n description: 'IPIPMTU is the MTU to set on the tunnel device. See\n Configuring MTU [Default: 1440]'\n type: integer\n ipsetsRefreshInterval:\n description: 'IpsetsRefreshInterval is the period at which Felix re-checks\n all iptables state to ensure that no other process has accidentally\n broken Calico''s rules. Set to 0 to disable iptables refresh. [Default:\n 90s]'\n type: string\n iptablesBackend:\n description: IptablesBackend specifies which backend of iptables will\n be used. The default is legacy.\n type: string\n iptablesFilterAllowAction:\n type: string\n iptablesLockFilePath:\n description: 'IptablesLockFilePath is the location of the iptables\n lock file. You may need to change this if the lock file is not in\n its standard location (for example if you have mapped it into Felix''s\n container at a different path). [Default: /run/xtables.lock]'\n type: string\n iptablesLockProbeInterval:\n description: 'IptablesLockProbeInterval is the time that Felix will\n wait between attempts to acquire the iptables lock if it is not\n available. Lower values make Felix more responsive when the lock\n is contended, but use more CPU. [Default: 50ms]'\n type: string\n iptablesLockTimeout:\n description: 'IptablesLockTimeout is the time that Felix will wait\n for the iptables lock, or 0, to disable. To use this feature, Felix\n must share the iptables lock file with all other processes that\n also take the lock. When running Felix inside a container, this\n requires the /run directory of the host to be mounted into the calico/node\n or calico/felix container. [Default: 0s disabled]'\n type: string\n iptablesMangleAllowAction:\n type: string\n iptablesMarkMask:\n description: 'IptablesMarkMask is the mask that Felix selects its\n IPTables Mark bits from. Should be a 32 bit hexadecimal number with\n at least 8 bits set, none of which clash with any other mark bits\n in use on the system. [Default: 0xff000000]'\n format: int32\n type: integer\n iptablesNATOutgoingInterfaceFilter:\n type: string\n iptablesPostWriteCheckInterval:\n description: 'IptablesPostWriteCheckInterval is the period after Felix\n has done a write to the dataplane that it schedules an extra read\n back in order to check the write was not clobbered by another process.\n This should only occur if another application on the system doesn''t\n respect the iptables lock. [Default: 1s]'\n type: string\n iptablesRefreshInterval:\n description: 'IptablesRefreshInterval is the period at which Felix\n re-checks the IP sets in the dataplane to ensure that no other process\n has accidentally broken Calico''s rules. Set to 0 to disable IP\n sets refresh. Note: the default for this value is lower than the\n other refresh intervals as a workaround for a Linux kernel bug that\n was fixed in kernel version 4.11. If you are using v4.11 or greater\n you may want to set this to, a higher value to reduce Felix CPU\n usage. [Default: 10s]'\n type: string\n ipv6Support:\n type: boolean\n kubeNodePortRanges:\n description: 'KubeNodePortRanges holds list of port ranges used for\n service node ports. Only used if felix detects kube-proxy running\n in ipvs mode. Felix uses these ranges to separate host and workload\n traffic. [Default: 30000:32767].'\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n logFilePath:\n description: 'LogFilePath is the full path to the Felix log. Set to\n none to disable file logging. [Default: /var/log/calico/felix.log]'\n type: string\n logPrefix:\n description: 'LogPrefix is the log prefix that Felix uses when rendering\n LOG rules. [Default: calico-packet]'\n type: string\n logSeverityFile:\n description: 'LogSeverityFile is the log severity above which logs\n are sent to the log file. [Default: Info]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n type: string\n logSeveritySys:\n description: 'LogSeveritySys is the log severity above which logs\n are sent to the syslog. Set to None for no logging to syslog. [Default:\n Info]'\n type: string\n maxIpsetSize:\n type: integer\n metadataAddr:\n description: 'MetadataAddr is the IP address or domain name of the\n server that can answer VM queries for cloud-init metadata. In OpenStack,\n this corresponds to the machine running nova-api (or in Ubuntu,\n nova-api-metadata). A value of none (case insensitive) means that\n Felix should not set up any NAT rule for the metadata path. [Default:\n 127.0.0.1]'\n type: string\n metadataPort:\n description: 'MetadataPort is the port of the metadata server. This,\n combined with global.MetadataAddr (if not ''None''), is used to\n set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort.\n In most cases this should not need to be changed [Default: 8775].'\n type: integer\n mtuIfacePattern:\n description: MTUIfacePattern is a regular expression that controls\n which interfaces Felix should scan in order to calculate the host's\n MTU. This should not match workload interfaces (usually named cali...).\n type: string\n natOutgoingAddress:\n description: NATOutgoingAddress specifies an address to use when performing\n source NAT for traffic in a natOutgoing pool that is leaving the\n network. By default the address used is an address on the interface\n the traffic is leaving on (ie it uses the iptables MASQUERADE target)\n type: string\n natPortRange:\n anyOf:\n - type: integer\n - type: string\n description: NATPortRange specifies the range of ports that is used\n for port mapping when doing outgoing NAT. When unset the default\n behavior of the network stack is used.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n netlinkTimeout:\n type: string\n openstackRegion:\n description: 'OpenstackRegion is the name of the region that a particular\n Felix belongs to. In a multi-region Calico/OpenStack deployment,\n this must be configured somehow for each Felix (here in the datamodel,\n or in felix.cfg or the environment on each compute node), and must\n match the [calico] openstack_region value configured in neutron.conf\n on each node. [Default: Empty]'\n type: string\n policySyncPathPrefix:\n description: 'PolicySyncPathPrefix is used to by Felix to communicate\n policy changes to external services, like Application layer policy.\n [Default: Empty]'\n type: string\n prometheusGoMetricsEnabled:\n description: 'PrometheusGoMetricsEnabled disables Go runtime metrics\n collection, which the Prometheus client does by default, when set\n to false. This reduces the number of metrics reported, reducing\n Prometheus load. [Default: true]'\n type: boolean\n prometheusMetricsEnabled:\n description: 'PrometheusMetricsEnabled enables the Prometheus metrics\n server in Felix if set to true. [Default: false]'\n type: boolean\n prometheusMetricsHost:\n description: 'PrometheusMetricsHost is the host that the Prometheus\n metrics server should bind to. [Default: empty]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. [Default: 9091]'\n type: integer\n prometheusProcessMetricsEnabled:\n description: 'PrometheusProcessMetricsEnabled disables process metrics\n collection, which the Prometheus client does by default, when set\n to false. This reduces the number of metrics reported, reducing\n Prometheus load. [Default: true]'\n type: boolean\n prometheusWireGuardMetricsEnabled:\n description: 'PrometheusWireGuardMetricsEnabled disables wireguard\n metrics collection, which the Prometheus client does by default,\n when set to false. This reduces the number of metrics reported,\n reducing Prometheus load. [Default: true]'\n type: boolean\n removeExternalRoutes:\n description: Whether or not to remove device routes that have not\n been programmed by Felix. Disabling this will allow external applications\n to also add device routes. This is enabled by default which means\n we will remove externally added routes.\n type: boolean\n reportingInterval:\n description: 'ReportingInterval is the interval at which Felix reports\n its status into the datastore or 0 to disable. Must be non-zero\n in OpenStack deployments. [Default: 30s]'\n type: string\n reportingTTL:\n description: 'ReportingTTL is the time-to-live setting for process-wide\n status reports. [Default: 90s]'\n type: string\n routeRefreshInterval:\n description: 'RouteRefreshInterval is the period at which Felix re-checks\n the routes in the dataplane to ensure that no other process has\n accidentally broken Calico''s rules. Set to 0 to disable route refresh.\n [Default: 90s]'\n type: string\n routeSource:\n description: 'RouteSource configures where Felix gets its routing\n information. - WorkloadIPs: use workload endpoints to construct\n routes. - CalicoIPAM: the default - use IPAM data to construct routes.'\n type: string\n routeTableRange:\n description: Calico programs additional Linux route tables for various\n purposes. RouteTableRange specifies the indices of the route tables\n that Calico should use.\n properties:\n max:\n type: integer\n min:\n type: integer\n required:\n - max\n - min\n type: object\n serviceLoopPrevention:\n description: 'When service IP advertisement is enabled, prevent routing\n loops to service IPs that are not in use, by dropping or rejecting\n packets that do not get DNAT''d by kube-proxy. Unless set to \"Disabled\",\n in which case such routing loops continue to be allowed. [Default:\n Drop]'\n type: string\n sidecarAccelerationEnabled:\n description: 'SidecarAccelerationEnabled enables experimental sidecar\n acceleration [Default: false]'\n type: boolean\n usageReportingEnabled:\n description: 'UsageReportingEnabled reports anonymous Calico version\n number and cluster size to projectcalico.org. Logs warnings returned\n by the usage server. For example, if a significant security vulnerability\n has been discovered in the version of Calico being used. [Default:\n true]'\n type: boolean\n usageReportingInitialDelay:\n description: 'UsageReportingInitialDelay controls the minimum delay\n before Felix makes a report. [Default: 300s]'\n type: string\n usageReportingInterval:\n description: 'UsageReportingInterval controls the interval at which\n Felix makes reports. [Default: 86400s]'\n type: string\n useInternalDataplaneDriver:\n type: boolean\n vxlanEnabled:\n type: boolean\n vxlanMTU:\n description: 'VXLANMTU is the MTU to set on the tunnel device. See\n Configuring MTU [Default: 1440]'\n type: integer\n vxlanPort:\n type: integer\n vxlanVNI:\n type: integer\n wireguardEnabled:\n description: 'WireguardEnabled controls whether Wireguard is enabled.\n [Default: false]'\n type: boolean\n wireguardHostEncryptionEnabled:\n description: 'WireguardHostEncryptionEnabled controls whether Wireguard\n host-to-host encryption is enabled. [Default: false]'\n type: boolean\n wireguardInterfaceName:\n description: 'WireguardInterfaceName specifies the name to use for\n the Wireguard interface. [Default: wg.calico]'\n type: string\n wireguardListeningPort:\n description: 'WireguardListeningPort controls the listening port used\n by Wireguard. [Default: 51820]'\n type: integer\n wireguardMTU:\n description: 'WireguardMTU controls the MTU on the Wireguard interface.\n See Configuring MTU [Default: 1420]'\n type: integer\n wireguardRoutingRulePriority:\n description: 'WireguardRoutingRulePriority controls the priority value\n to use for the Wireguard routing rule. [Default: 99]'\n type: integer\n xdpEnabled:\n description: 'XDPEnabled enables XDP acceleration for suitable untracked\n incoming deny rules. [Default: true]'\n type: boolean\n xdpRefreshInterval:\n description: 'XDPRefreshInterval is the period at which Felix re-checks\n all XDP state to ensure that no other process has accidentally broken\n Calico''s BPF maps or attached programs. Set to 0 to disable XDP\n refresh. [Default: 90s]'\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: globalnetworkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: GlobalNetworkPolicy\n listKind: GlobalNetworkPolicyList\n plural: globalnetworkpolicies\n singular: globalnetworkpolicy\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n properties:\n applyOnForward:\n description: ApplyOnForward indicates to apply the rules in this policy\n on forward traffic.\n type: boolean\n doNotTrack:\n description: DoNotTrack indicates whether packets matched by the rules\n in this policy should go through the data plane's connection tracking,\n such as Linux conntrack. If True, the rules in this policy are\n applied before any data plane connection tracking, and packets allowed\n by this policy are marked as not to be tracked.\n type: boolean\n egress:\n description: The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description: The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n namespaceSelector:\n description: NamespaceSelector is an optional field for an expression\n used to select a pod based on namespaces.\n type: string\n order:\n description: Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order. If the order is omitted, it may be\n considered to be \"infinite\" - i.e. the policy will be applied last. Policies\n with identical order will be applied in alphanumerical order based\n on the Policy \"Name\".\n type: number\n preDNAT:\n description: PreDNAT indicates to apply the rules in this policy before\n any DNAT.\n type: boolean\n selector:\n description: \"The selector is an expression used to pick pick out\n the endpoints that the policy should be applied to. \\n Selector\n expressions follow this syntax: \\n \\tlabel == \\\"string_literal\\\"\n \\ -> comparison, e.g. my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\"\n \\ -> not equal; also matches if label is not present \\tlabel in\n { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is\n one of \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\",\n ... } -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\",\n \\\"c\\\" \\thas(label_name) -> True if that label is present \\t! expr\n -> negation of expr \\texpr && expr -> Short-circuit and \\texpr\n || expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description: ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n types:\n description: \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress rules are present in the policy. The\n default is: \\n - [ PolicyTypeIngress ], if there are no Egress rules\n (including the case where there are also no Ingress rules) \\n\n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress\n rules \\n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are\n both Ingress and Egress rules. \\n When the policy is read back again,\n Types will always be one of these values, never empty or nil.\"\n items:\n description: PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: globalnetworksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: GlobalNetworkSet\n listKind: GlobalNetworkSetList\n plural: globalnetworksets\n singular: globalnetworkset\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs\n that share labels to allow rules to refer to them via selectors. The labels\n of GlobalNetworkSet are not namespaced.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: GlobalNetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: hostendpoints.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: HostEndpoint\n listKind: HostEndpointList\n plural: hostendpoints\n singular: hostendpoint\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: HostEndpointSpec contains the specification for a HostEndpoint\n resource.\n properties:\n expectedIPs:\n description: \"The expected IP addresses (IPv4 and IPv6) of the endpoint.\n If \\\"InterfaceName\\\" is not present, Calico will look for an interface\n matching any of the IPs in the list and apply policy to that. Note:\n \\tWhen using the selector match criteria in an ingress or egress\n security Policy \\tor Profile, Calico converts the selector into\n a set of IP addresses. For host \\tendpoints, the ExpectedIPs field\n is used for that purpose. (If only the interface \\tname is specified,\n Calico does not learn the IPs of the interface for use in match\n \\tcriteria.)\"\n items:\n type: string\n type: array\n interfaceName:\n description: \"Either \\\"*\\\", or the name of a specific Linux interface\n to apply policy to; or empty. \\\"*\\\" indicates that this HostEndpoint\n governs all traffic to, from or through the default network namespace\n of the host named by the \\\"Node\\\" field; entering and leaving that\n namespace via any interface, including those from/to non-host-networked\n local workloads. \\n If InterfaceName is not \\\"*\\\", this HostEndpoint\n only governs traffic that enters or leaves the host through the\n specific interface named by InterfaceName, or - when InterfaceName\n is empty - through the specific interface that has one of the IPs\n in ExpectedIPs. Therefore, when InterfaceName is empty, at least\n one expected IP must be specified. Only external interfaces (such\n as \\\"eth0\\\") are supported here; it isn't possible for a HostEndpoint\n to protect traffic through a specific local workload interface.\n \\n Note: Only some kinds of policy are implemented for \\\"*\\\" HostEndpoints;\n initially just pre-DNAT policy. Please check Calico documentation\n for the latest position.\"\n type: string\n node:\n description: The node name identifying the Calico node instance.\n type: string\n ports:\n description: Ports contains the endpoint's named ports, which may\n be referenced in security policy rules.\n items:\n properties:\n name:\n type: string\n port:\n type: integer\n protocol:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n required:\n - name\n - port\n - protocol\n type: object\n type: array\n profiles:\n description: A list of identifiers of security Profile objects that\n apply to this endpoint. Each profile is applied in the order that\n they appear in this list. Profile rules are applied after the selector-based\n security policy.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamblocks.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMBlock\n listKind: IPAMBlockList\n plural: ipamblocks\n singular: ipamblock\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMBlockSpec contains the specification for an IPAMBlock\n resource.\n properties:\n affinity:\n type: string\n allocations:\n items:\n type: integer\n # TODO: This nullable is manually added in. We should update controller-gen\n # to handle []*int properly itself.\n nullable: true\n type: array\n attributes:\n items:\n properties:\n handle_id:\n type: string\n secondary:\n additionalProperties:\n type: string\n type: object\n type: object\n type: array\n cidr:\n type: string\n deleted:\n type: boolean\n strictAffinity:\n type: boolean\n unallocated:\n items:\n type: integer\n type: array\n required:\n - allocations\n - attributes\n - cidr\n - strictAffinity\n - unallocated\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMConfigSpec contains the specification for an IPAMConfig\n resource.\n properties:\n autoAllocateBlocks:\n type: boolean\n maxBlocksPerHost:\n description: MaxBlocksPerHost, if non-zero, is the max number of blocks\n that can be affine to each host.\n type: integer\n strictAffinity:\n type: boolean\n required:\n - autoAllocateBlocks\n - strictAffinity\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamhandles.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMHandle\n listKind: IPAMHandleList\n plural: ipamhandles\n singular: ipamhandle\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMHandleSpec contains the specification for an IPAMHandle\n resource.\n properties:\n block:\n additionalProperties:\n type: integer\n type: object\n deleted:\n type: boolean\n handleID:\n type: string\n required:\n - block\n - handleID\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPPool\n listKind: IPPoolList\n plural: ippools\n singular: ippool\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPPoolSpec contains the specification for an IPPool resource.\n properties:\n allowedUses:\n description: AllowedUse controls what the IP pool will be used for. If\n not specified or empty, defaults to [\"Tunnel\", \"Workload\"] for back-compatibility\n items:\n type: string\n type: array\n blockSize:\n description: The block size to use for IP address assignments from\n this pool. Defaults to 26 for IPv4 and 112 for IPv6.\n type: integer\n cidr:\n description: The pool CIDR.\n type: string\n disabled:\n description: When disabled is true, Calico IPAM will not assign addresses\n from this pool.\n type: boolean\n disableBGPExport:\n description: 'Disable exporting routes from this IP Pool’s CIDR over\n BGP. [Default: false]'\n type: boolean\n ipip:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n properties:\n enabled:\n description: When enabled is true, ipip tunneling will be used\n to deliver packets to destinations within this pool.\n type: boolean\n mode:\n description: The IPIP mode. This can be one of \"always\" or \"cross-subnet\". A\n mode of \"always\" will also use IPIP tunneling for routing to\n destination IP addresses within this pool. A mode of \"cross-subnet\"\n will only use IPIP tunneling when the destination node is on\n a different subnet to the originating node. The default value\n (if not specified) is \"always\".\n type: string\n type: object\n ipipMode:\n description: Contains configuration for IPIP tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. IPIP tunneling\n is disabled).\n type: string\n nat-outgoing:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n type: boolean\n natOutgoing:\n description: When nat-outgoing is true, packets sent from Calico networked\n containers in this pool to destinations outside of this pool will\n be masqueraded.\n type: boolean\n nodeSelector:\n description: Allows IPPool to allocate for a specific node by label\n selector.\n type: string\n vxlanMode:\n description: Contains configuration for VXLAN tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. VXLAN\n tunneling is disabled).\n type: string\n required:\n - cidr\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipreservations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPReservation\n listKind: IPReservationList\n plural: ipreservations\n singular: ipreservation\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPReservationSpec contains the specification for an IPReservation\n resource.\n properties:\n reservedCIDRs:\n description: ReservedCIDRs is a list of CIDRs and/or IP addresses\n that Calico IPAM will exclude from new allocations.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: kubecontrollersconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: KubeControllersConfiguration\n listKind: KubeControllersConfigurationList\n plural: kubecontrollersconfigurations\n singular: kubecontrollersconfiguration\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: KubeControllersConfigurationSpec contains the values of the\n Kubernetes controllers configuration.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description: Namespace enables and configures the namespace controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host endpoints.\n Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation of\n host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the controller\n to determine if an IP address has been leaked. Set to 0\n to disable IP garbage collection. [Default: 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n type: object\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default: 9094]'\n type: integer\n required:\n - controllers\n type: object\n status:\n description: KubeControllersConfigurationStatus represents the status\n of the configuration. It's useful for admins to be able to see the actual\n config that was applied, which can be modified by environment variables\n on the kube-controllers process.\n properties:\n environmentVars:\n additionalProperties:\n type: string\n description: EnvironmentVars contains the environment variables on\n the kube-controllers that influenced the RunningConfig.\n type: object\n runningConfig:\n description: RunningConfig contains the effective config that is running\n in the kube-controllers pod, after merging the API resource with\n any environment variables.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description: Namespace enables and configures the namespace\n controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host\n endpoints. Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation\n of host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the\n controller to determine if an IP address has been leaked.\n Set to 0 to disable IP garbage collection. [Default:\n 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n type: object\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which\n logs are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default:\n 9094]'\n type: integer\n required:\n - controllers\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkPolicy\n listKind: NetworkPolicyList\n plural: networkpolicies\n singular: networkpolicy\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n properties:\n egress:\n description: The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description: The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n order:\n description: Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order. If the order is omitted, it may be\n considered to be \"infinite\" - i.e. the policy will be applied last. Policies\n with identical order will be applied in alphanumerical order based\n on the Policy \"Name\".\n type: number\n selector:\n description: \"The selector is an expression used to pick pick out\n the endpoints that the policy should be applied to. \\n Selector\n expressions follow this syntax: \\n \\tlabel == \\\"string_literal\\\"\n \\ -> comparison, e.g. my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\"\n \\ -> not equal; also matches if label is not present \\tlabel in\n { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is\n one of \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\",\n ... } -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\",\n \\\"c\\\" \\thas(label_name) -> True if that label is present \\t! expr\n -> negation of expr \\texpr && expr -> Short-circuit and \\texpr\n || expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description: ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n types:\n description: \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress are present in the policy. The default\n is: \\n - [ PolicyTypeIngress ], if there are no Egress rules (including\n the case where there are also no Ingress rules) \\n - [ PolicyTypeEgress\n ], if there are Egress rules but no Ingress rules \\n - [ PolicyTypeIngress,\n PolicyTypeEgress ], if there are both Ingress and Egress rules.\n \\n When the policy is read back again, Types will always be one\n of these values, never empty or nil.\"\n items:\n description: PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkSet\n listKind: NetworkSetList\n plural: networksets\n singular: networkset\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: NetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\n---\n# Source: calico/templates/calico-kube-controllers-rbac.yaml\n\n# Include a clusterrole for the kube-controllers component,\n# and bind it to the calico-kube-controllers serviceaccount.\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-kube-controllers\nrules:\n # Nodes are watched to monitor for deletions.\n - apiGroups: [\"\"]\n resources:\n - nodes\n verbs:\n - watch\n - list\n - get\n # Pods are watched to check for existence as part of IPAM controller.\n - apiGroups: [\"\"]\n resources:\n - pods\n verbs:\n - get\n - list\n - watch\n # IPAM resources are manipulated when nodes are deleted.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ippools\n - ipreservations\n verbs:\n - list\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n verbs:\n - get\n - list\n - create\n - update\n - delete\n - watch\n # kube-controllers manages hostendpoints.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - hostendpoints\n verbs:\n - get\n - list\n - create\n - update\n - delete\n # Needs access to update clusterinformations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - clusterinformations\n verbs:\n - get\n - create\n - update\n # KubeControllersConfiguration is where it gets its config\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - kubecontrollersconfigurations\n verbs:\n # read its own config\n - get\n # create a default if none exists\n - create\n # update status\n - update\n # watch for changes\n - watch\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-kube-controllers\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-kube-controllers\nsubjects:\n- kind: ServiceAccount\n name: calico-kube-controllers\n namespace: kube-system\n---\n\n---\n# Source: calico/templates/calico-node-rbac.yaml\n# Include a clusterrole for the calico-node DaemonSet,\n# and bind it to the calico-node serviceaccount.\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-node\nrules:\n # The CNI plugin needs to get pods, nodes, and namespaces.\n - apiGroups: [\"\"]\n resources:\n - pods\n - nodes\n - namespaces\n verbs:\n - get\n # EndpointSlices are used for Service-based network policy rule\n # enforcement.\n - apiGroups: [\"discovery.k8s.io\"]\n resources:\n - endpointslices\n verbs:\n - watch \n - list\n - apiGroups: [\"\"]\n resources:\n - endpoints\n - services\n verbs:\n # Used to discover service IPs for advertisement.\n - watch\n - list\n # Used to discover Typhas.\n - get\n # Pod CIDR auto-detection on kubeadm needs access to config maps.\n - apiGroups: [\"\"]\n resources:\n - configmaps\n verbs:\n - get\n - apiGroups: [\"\"]\n resources:\n - nodes/status\n verbs:\n # Needed for clearing NodeNetworkUnavailable flag.\n - patch\n # Calico stores some configuration information in node annotations.\n - update\n # Watch for changes to Kubernetes NetworkPolicies.\n - apiGroups: [\"networking.k8s.io\"]\n resources:\n - networkpolicies\n verbs:\n - watch\n - list\n # Used by Calico for policy information.\n - apiGroups: [\"\"]\n resources:\n - pods\n - namespaces\n - serviceaccounts\n verbs:\n - list\n - watch\n # The CNI plugin patches pods/status.\n - apiGroups: [\"\"]\n resources:\n - pods/status\n verbs:\n - patch\n # Calico monitors various CRDs for config.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - globalfelixconfigs\n - felixconfigurations\n - bgppeers\n - globalbgpconfigs\n - bgpconfigurations\n - ippools\n - ipreservations\n - ipamblocks\n - globalnetworkpolicies\n - globalnetworksets\n - networkpolicies\n - networksets\n - clusterinformations\n - hostendpoints\n - blockaffinities\n - caliconodestatuses\n verbs:\n - get\n - list\n - watch\n # Calico must create and update some CRDs on startup.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ippools\n - felixconfigurations\n - clusterinformations\n verbs:\n - create\n - update\n # Calico must update some CRDs.\n - apiGroups: [ \"crd.projectcalico.org\" ]\n resources:\n - caliconodestatuses\n verbs:\n - update\n # Calico stores some configuration information on the node.\n - apiGroups: [\"\"]\n resources:\n - nodes\n verbs:\n - get\n - list\n - watch\n # These permissions are only required for upgrade from v2.6, and can\n # be removed after upgrade or on fresh installations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - bgpconfigurations\n - bgppeers\n verbs:\n - create\n - update\n # These permissions are required for Calico CNI to perform IPAM allocations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n verbs:\n - get\n - list\n - create\n - update\n - delete\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ipamconfigs\n verbs:\n - get\n # Block affinities must also be watchable by confd for route aggregation.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n verbs:\n - watch\n # The Calico IPAM migration needs to get daemonsets. These permissions can be\n # removed if not upgrading from an installation using host-local IPAM.\n - apiGroups: [\"apps\"]\n resources:\n - daemonsets\n verbs:\n - get\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-node\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-node\nsubjects:\n- kind: ServiceAccount\n name: calico-node\n namespace: kube-system\n\n---\n# Source: calico/templates/calico-node.yaml\n# This manifest installs the calico-node container, as well\n# as the CNI plugins and network config on\n# each master and worker node in a Kubernetes cluster.\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: calico-node\n namespace: kube-system\n labels:\n k8s-app: calico-node\nspec:\n selector:\n matchLabels:\n k8s-app: calico-node\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n template:\n metadata:\n labels:\n k8s-app: calico-node\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n # Make sure calico-node gets scheduled on all nodes.\n - effect: NoSchedule\n operator: Exists\n # Mark the pod as a critical add-on for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - effect: NoExecute\n operator: Exists\n serviceAccountName: calico-node\n # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a \"force\n # deletion\": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.\n terminationGracePeriodSeconds: 0\n priorityClassName: system-node-critical\n initContainers:\n # This container performs upgrade from host-local IPAM to calico-ipam.\n # It can be deleted if this is a fresh installation, or if you have already\n # upgraded to use calico-ipam.\n - name: upgrade-ipam\n image: docker.io/calico/cni:v3.23.4\n command: [\"/opt/cni/bin/calico-ipam\", \"-upgrade\"]\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n - name: KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: CALICO_NETWORKING_BACKEND\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: calico_backend\n volumeMounts:\n - mountPath: /var/lib/cni/networks\n name: host-local-net-dir\n - mountPath: /host/opt/cni/bin\n name: cni-bin-dir\n securityContext:\n privileged: true\n # This container installs the CNI binaries\n # and CNI network config file on each node.\n - name: install-cni\n image: docker.io/calico/cni:v3.23.4\n command: [\"/opt/cni/bin/install\"]\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: \"10-calico.conflist\"\n # The CNI network config to install on each node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: cni_network_config\n # Set the hostname based on the k8s node name.\n - name: KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Prevents the container from sleeping forever.\n - name: SLEEP\n value: \"false\"\n - name: CNI_NET_DIR\n value: \"/var/snap/microk8s/current/args/cni-network\"\n volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: cni-net-dir\n securityContext:\n privileged: true\n # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes\n # to communicate with Felix over the Policy Sync API.\n - name: flexvol-driver\n image: docker.io/calico/pod2daemon-flexvol:v3.23.4\n volumeMounts:\n - name: flexvol-driver-host\n mountPath: /host/driver\n securityContext:\n privileged: true\n containers:\n # Runs calico-node container on each Kubernetes node. This\n # container programs network policy and routes on each\n # host.\n - name: calico-node\n image: docker.io/calico/node:v3.23.4\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # Use Kubernetes API as the backing datastore.\n - name: DATASTORE_TYPE\n value: \"kubernetes\"\n # Wait for the datastore.\n - name: WAIT_FOR_DATASTORE\n value: \"true\"\n # Set based on the k8s node name.\n - name: NODENAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n # Choose the backend to use.\n - name: CALICO_NETWORKING_BACKEND\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: calico_backend\n # Cluster type to identify the deployment type\n - name: CLUSTER_TYPE\n value: \"k8s,bgp\"\n # Auto-detect the BGP IP address.\n - name: IP\n value: \"autodetect\"\n - name: IP_AUTODETECTION_METHOD\n value: \"first-found\"\n # Enable IPIP\n #- name: CALICO_IPV4POOL_IPIP\n # value: \"Always\"\n # Enable or Disable VXLAN on the default IP pool.\n - name: CALICO_IPV4POOL_VXLAN\n value: \"Always\"\n # Set MTU for tunnel device used if ipip is enabled\n - name: FELIX_IPINIPMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Set MTU for the VXLAN tunnel device.\n - name: FELIX_VXLANMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Set MTU for the Wireguard tunnel device.\n - name: FELIX_WIREGUARDMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # The default IPv4 pool to create on startup if none exists. Pod IPs will be\n # chosen from this range. Changing this value after installation will have\n # no effect. This should fall within `--cluster-cidr`.\n - name: CALICO_IPV4POOL_CIDR\n value: \"10.1.0.0/16\"\n # Disable file logging so `kubectl logs` works.\n - name: CALICO_DISABLE_FILE_LOGGING\n value: \"true\"\n # Set Felix endpoint to host default action to ACCEPT.\n - name: FELIX_DEFAULTENDPOINTTOHOSTACTION\n value: \"ACCEPT\"\n # Disable IPv6 on Kubernetes.\n - name: FELIX_IPV6SUPPORT\n value: \"false\"\n # Set Felix logging to \"error\"\n - name: FELIX_LOGSEVERITYSCREEN\n value: \"error\"\n - name: FELIX_HEALTHENABLED\n value: \"true\"\n securityContext:\n privileged: true\n resources:\n requests:\n cpu: 250m\n lifecycle:\n preStop:\n exec:\n command:\n - /bin/calico-node\n - -shutdown\n livenessProbe:\n exec:\n command:\n - /bin/calico-node\n - -felix-live\n # - -bird-live\n periodSeconds: 10\n initialDelaySeconds: 10\n failureThreshold: 6\n timeoutSeconds: 10\n readinessProbe:\n exec:\n command:\n - /bin/calico-node\n - -felix-ready\n # - -bird-ready\n periodSeconds: 10\n timeoutSeconds: 10\n volumeMounts:\n # For maintaining CNI plugin API credentials.\n - mountPath: /host/etc/cni/net.d\n name: cni-net-dir\n readOnly: false\n - mountPath: /lib/modules\n name: lib-modules\n readOnly: true\n - mountPath: /run/xtables.lock\n name: xtables-lock\n readOnly: false\n - mountPath: /var/run/calico\n name: var-run-calico\n readOnly: false\n - mountPath: /var/lib/calico\n name: var-lib-calico\n readOnly: false\n - name: policysync\n mountPath: /var/run/nodeagent\n # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the\n # parent directory.\n # - name: sysfs\n # mountPath: /sys/fs/\n # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.\n # If the host is known to mount that filesystem already then Bidirectional can be omitted.\n # mountPropagation: Bidirectional\n - name: cni-log-dir\n mountPath: /var/log/calico/cni\n readOnly: true\n volumes:\n # Used by calico-node.\n - name: lib-modules\n hostPath:\n path: /lib/modules\n - name: var-run-calico\n hostPath:\n path: /var/snap/microk8s/current/var/run/calico\n - name: var-lib-calico\n hostPath:\n path: /var/snap/microk8s/current/var/lib/calico\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n - name: sysfs\n hostPath:\n path: /sys/fs/\n type: DirectoryOrCreate\n # Used to install CNI.\n - name: cni-bin-dir\n hostPath:\n path: /var/snap/microk8s/current/opt/cni/bin\n - name: cni-net-dir\n hostPath:\n path: /var/snap/microk8s/current/args/cni-network\n # Used to access CNI logs.\n - name: cni-log-dir\n hostPath:\n path: /var/snap/microk8s/common/var/log/calico/cni\n # Mount in the directory for host-local IPAM allocations. This is\n # used when upgrading from host-local to calico-ipam, and can be removed\n # if not using the upgrade-ipam init container.\n - name: host-local-net-dir\n hostPath:\n path: /var/snap/microk8s/current/var/lib/cni/networks\n # Used to create per-pod Unix Domain Sockets\n - name: policysync\n hostPath:\n type: DirectoryOrCreate\n path: /var/snap/microk8s/current/var/run/nodeagent\n # Used to install Flex Volume Driver\n - name: flexvol-driver-host\n hostPath:\n type: DirectoryOrCreate\n path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds\n---\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-node\n namespace: kube-system\n\n---\n# Source: calico/templates/calico-kube-controllers.yaml\n# See https://github.com/projectcalico/kube-controllers\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\nspec:\n # The controllers can only have a single active instance.\n replicas: 1\n selector:\n matchLabels:\n k8s-app: calico-kube-controllers\n strategy:\n type: Recreate\n template:\n metadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n tolerations:\n # Mark the pod as a critical add-on for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n serviceAccountName: calico-kube-controllers\n priorityClassName: system-cluster-critical\n containers:\n - name: calico-kube-controllers\n image: docker.io/calico/kube-controllers:v3.23.4\n env:\n # Choose which controllers to run.\n - name: ENABLED_CONTROLLERS\n value: node\n - name: DATASTORE_TYPE\n value: kubernetes\n livenessProbe:\n exec:\n command:\n - /usr/bin/check-status\n - -l\n periodSeconds: 10\n initialDelaySeconds: 10\n failureThreshold: 6\n timeoutSeconds: 10\n readinessProbe:\n exec:\n command:\n - /usr/bin/check-status\n - -r\n periodSeconds: 10\n\n---\n\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n\n---\n\n# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict\n\napiVersion: policy/v1beta1\nkind: PodDisruptionBudget\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\nspec:\n maxUnavailable: 1\n selector:\n matchLabels:\n k8s-app: calico-kube-controllers\n\n---\n# Source: calico/templates/calico-etcd-secrets.yaml\n\n---\n# Source: calico/templates/calico-typha.yaml\n\n---\n# Source: calico/templates/configure-canal.yaml\n\n\n" }, { "path": "tests/unit/yamls/cni.yaml", "content": "---\n# Source: calico/templates/calico-config.yaml\n# This ConfigMap is used to configure a self-hosted Calico installation.\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-config\n namespace: kube-system\ndata:\n # Typha is disabled.\n typha_service_name: \"none\"\n # Configure the backend to use.\n calico_backend: \"vxlan\"\n\n # Configure the MTU to use for workload interfaces and tunnels.\n # By default, MTU is auto-detected, and explicitly setting this field should not be required.\n # You can override auto-detection by providing a non-zero value.\n veth_mtu: \"1440\"\n\n # The CNI network configuration to install on each node. The special\n # values in this config will be automatically populated.\n cni_network_config: |-\n {\n \"name\": \"k8s-pod-network\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": [\n {\n \"type\": \"calico\",\n \"log_level\": \"info\",\n \"datastore_type\": \"kubernetes\",\n \"nodename_file_optional\": true,\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n \"mtu\": __CNI_MTU__,\n \"ipam\": {\n \"type\": \"calico-ipam\"\n },\n \"policy\": {\n \"type\": \"k8s\"\n },\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n }\n },\n {\n \"type\": \"portmap\",\n \"snat\": true,\n \"capabilities\": {\"portMappings\": true}\n },\n {\n \"type\": \"bandwidth\",\n \"capabilities\": {\"bandwidth\": true}\n }\n ]\n }\n\n---\n# Source: calico/templates/kdd-crds.yaml\n\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: bgpconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPConfiguration\n listKind: BGPConfigurationList\n plural: bgpconfigurations\n singular: bgpconfiguration\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: BGPConfiguration contains the configuration for any BGP routing.\n properties:\n apiVersion:\n description: \"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description: \"Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description: BGPConfigurationSpec contains the values of the BGP configuration.\n properties:\n asNumber:\n description: \"ASNumber is the default AS number used by a node. [Default: 64512]\"\n format: int32\n type: integer\n communities:\n description: Communities is a list of BGP community values and their arbitrary names for tagging routes.\n items:\n description: Community contains standard or large community value and its name.\n properties:\n name:\n description: Name given to community value.\n type: string\n value:\n description: Value must be of format `aa:nn` or `aa:nn:mm`. For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. Where, `aa` is an AS Number, `nn` and `mm` are per-AS identifier.\n pattern: ^(\\d+):(\\d+)$|^(\\d+):(\\d+):(\\d+)$\n type: string\n type: object\n type: array\n listenPort:\n description: ListenPort is the port where BGP protocol should listen. Defaults to 179\n maximum: 65535\n minimum: 1\n type: integer\n logSeverityScreen:\n description: \"LogSeverityScreen is the log severity above which logs are sent to the stdout. [Default: INFO]\"\n type: string\n nodeToNodeMeshEnabled:\n description: \"NodeToNodeMeshEnabled sets whether full node to node BGP mesh is enabled. [Default: true]\"\n type: boolean\n prefixAdvertisements:\n description: PrefixAdvertisements contains per-prefix advertisement configuration.\n items:\n description: PrefixAdvertisement configures advertisement properties for the specified CIDR.\n properties:\n cidr:\n description: CIDR for which properties should be advertised.\n type: string\n communities:\n description: Communities can be list of either community names already defined in `Specs.Communities` or community value of format `aa:nn` or `aa:nn:mm`. For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and `mm` are per-AS identifier.\n items:\n type: string\n type: array\n type: object\n type: array\n serviceClusterIPs:\n description: ServiceClusterIPs are the CIDR blocks from which service cluster IPs are allocated. If specified, Calico will advertise these blocks, as well as any cluster IPs within them.\n items:\n description: ServiceClusterIPBlock represents a single allowed ClusterIP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n serviceExternalIPs:\n description: ServiceExternalIPs are the CIDR blocks for Kubernetes Service External IPs. Kubernetes Service ExternalIPs will only be advertised if they are within one of these blocks.\n items:\n description: ServiceExternalIPBlock represents a single allowed External IP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n serviceLoadBalancerIPs:\n description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress IPs will only be advertised if they are within one of these blocks.\n items:\n description: ServiceLoadBalancerIPBlock represents a single allowed LoadBalancer IP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n type: object\n type: object\n served: true\n storage: true\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: bgppeers.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPPeer\n listKind: BGPPeerList\n plural: bgppeers\n singular: bgppeer\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description: BGPPeerSpec contains the specification for a BGPPeer resource.\n properties:\n asNumber:\n description: The AS Number of the peer.\n format: int32\n type: integer\n keepOriginalNextHop:\n description:\n Option to keep the original nexthop field when routes\n are sent to a BGP Peer. Setting \"true\" configures the selected BGP\n Peers node to use the \"next hop keep;\" instead of \"next hop self;\"(default)\n in the specific branch of the Node on \"bird.cfg\".\n type: boolean\n node:\n description:\n The node name identifying the Calico node instance that\n is targeted by this peer. If this is not set, and no nodeSelector\n is specified, then this BGP peer selects all nodes in the cluster.\n type: string\n nodeSelector:\n description:\n Selector for the nodes that should have this peering. When\n this is set, the Node field must be empty.\n type: string\n password:\n description:\n Optional BGP password for the peerings generated by this\n BGPPeer resource.\n properties:\n secretKeyRef:\n description: Selects a key of a secret in the node pod's namespace.\n properties:\n key:\n description:\n The key of the secret to select from. Must be\n a valid secret key.\n type: string\n name:\n description:\n \"Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Add other useful fields. apiVersion, kind, uid?\"\n type: string\n optional:\n description:\n Specify whether the Secret or its key must be\n defined\n type: boolean\n required:\n - key\n type: object\n type: object\n peerIP:\n description:\n The IP address of the peer followed by an optional port\n number to peer with. If port number is given, format should be `[]:port`\n or `:` for IPv4. If optional port number is not set,\n and this peer IP and ASNumber belongs to a calico/node with ListenPort\n set in BGPConfiguration, then we use that port to peer.\n type: string\n peerSelector:\n description:\n Selector for the remote nodes to peer with. When this\n is set, the PeerIP and ASNumber fields must be empty. For each\n peering between the local node and selected remote nodes, we configure\n an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified,\n and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The\n remote AS number comes from the remote node's NodeBGPSpec.ASNumber,\n or the global default if that is not set.\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: blockaffinities.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BlockAffinity\n listKind: BlockAffinityList\n plural: blockaffinities\n singular: blockaffinity\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n BlockAffinitySpec contains the specification for a BlockAffinity\n resource.\n properties:\n cidr:\n type: string\n deleted:\n description:\n Deleted indicates that this block affinity is being deleted.\n This field is a string for compatibility with older releases that\n mistakenly treat this field as a string.\n type: string\n node:\n type: string\n state:\n type: string\n required:\n - cidr\n - deleted\n - node\n - state\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: clusterinformations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: ClusterInformation\n listKind: ClusterInformationList\n plural: clusterinformations\n singular: clusterinformation\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: ClusterInformation contains the cluster specific information.\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n ClusterInformationSpec contains the values of describing\n the cluster.\n properties:\n calicoVersion:\n description:\n CalicoVersion is the version of Calico that the cluster\n is running\n type: string\n clusterGUID:\n description: ClusterGUID is the GUID of the cluster\n type: string\n clusterType:\n description: ClusterType describes the type of the cluster\n type: string\n datastoreReady:\n description:\n DatastoreReady is used during significant datastore migrations\n to signal to components such as Felix that it should wait before\n accessing the datastore.\n type: boolean\n variant:\n description: Variant declares which variant of Calico should be active.\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: felixconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: FelixConfiguration\n listKind: FelixConfigurationList\n plural: felixconfigurations\n singular: felixconfiguration\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: Felix Configuration contains the configuration for Felix.\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description: FelixConfigurationSpec contains the values of the Felix configuration.\n properties:\n allowIPIPPacketsFromWorkloads:\n description:\n \"AllowIPIPPacketsFromWorkloads controls whether Felix\n will add a rule to drop IPIP encapsulated traffic from workloads\n [Default: false]\"\n type: boolean\n allowVXLANPacketsFromWorkloads:\n description:\n \"AllowVXLANPacketsFromWorkloads controls whether Felix\n will add a rule to drop VXLAN encapsulated traffic from workloads\n [Default: false]\"\n type: boolean\n awsSrcDstCheck:\n description:\n 'Set source-destination-check on AWS EC2 instances. Accepted\n value must be one of \"DoNothing\", \"Enabled\" or \"Disabled\". [Default:\n DoNothing]'\n enum:\n - DoNothing\n - Enable\n - Disable\n type: string\n bpfConnectTimeLoadBalancingEnabled:\n description:\n \"BPFConnectTimeLoadBalancingEnabled when in BPF mode,\n controls whether Felix installs the connection-time load balancer. The\n connect-time load balancer is required for the host to be able to\n reach Kubernetes services and it improves the performance of pod-to-service\n connections. The only reason to disable it is for debugging purposes. [Default:\n true]\"\n type: boolean\n bpfDataIfacePattern:\n description:\n BPFDataIfacePattern is a regular expression that controls\n which interfaces Felix should attach BPF programs to in order to\n catch traffic to/from the network. This needs to match the interfaces\n that Calico workload traffic flows over as well as any interfaces\n that handle incoming traffic to nodeports and services from outside\n the cluster. It should not match the workload interfaces (usually\n named cali...).\n type: string\n bpfDisableUnprivileged:\n description:\n \"BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled\n sysctl to disable unprivileged use of BPF. This ensures that unprivileged\n users cannot access Calico's BPF maps and cannot insert their own\n BPF programs to interfere with Calico's. [Default: true]\"\n type: boolean\n bpfEnabled:\n description:\n \"BPFEnabled, if enabled Felix will use the BPF dataplane.\n [Default: false]\"\n type: boolean\n bpfExternalServiceMode:\n description:\n 'BPFExternalServiceMode in BPF mode, controls how connections\n from outside the cluster to services (node ports and cluster IPs)\n are forwarded to remote workloads. If set to \"Tunnel\" then both\n request and response traffic is tunneled to the remote node. If\n set to \"DSR\", the request traffic is tunneled but the response traffic\n is sent directly from the remote node. In \"DSR\" mode, the remote\n node appears to use the IP of the ingress node; this requires a\n permissive L2 network. [Default: Tunnel]'\n type: string\n bpfKubeProxyEndpointSlicesEnabled:\n description:\n BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls\n whether Felix's embedded kube-proxy accepts EndpointSlices or not.\n type: boolean\n bpfKubeProxyIptablesCleanupEnabled:\n description:\n \"BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF\n mode, Felix will proactively clean up the upstream Kubernetes kube-proxy's\n iptables chains. Should only be enabled if kube-proxy is not running. [Default:\n true]\"\n type: boolean\n bpfKubeProxyMinSyncPeriod:\n description:\n \"BPFKubeProxyMinSyncPeriod, in BPF mode, controls the\n minimum time between updates to the dataplane for Felix's embedded\n kube-proxy. Lower values give reduced set-up latency. Higher values\n reduce Felix CPU usage by batching up more work. [Default: 1s]\"\n type: string\n bpfLogLevel:\n description:\n 'BPFLogLevel controls the log level of the BPF programs\n when in BPF dataplane mode. One of \"Off\", \"Info\", or \"Debug\". The\n logs are emitted to the BPF trace pipe, accessible with the command\n `tc exec bpf debug`. [Default: Off].'\n type: string\n chainInsertMode:\n description:\n \"ChainInsertMode controls whether Felix hooks the kernel's\n top-level iptables chains by inserting a rule at the top of the\n chain or by appending a rule at the bottom. insert is the safe default\n since it prevents Calico's rules from being bypassed. If you switch\n to append mode, be sure that the other rules in the chains signal\n acceptance by falling through to the Calico rules, otherwise the\n Calico policy will be bypassed. [Default: insert]\"\n type: string\n dataplaneDriver:\n type: string\n debugDisableLogDropping:\n type: boolean\n debugMemoryProfilePath:\n type: string\n debugSimulateCalcGraphHangAfter:\n type: string\n debugSimulateDataplaneHangAfter:\n type: string\n defaultEndpointToHostAction:\n description:\n \"DefaultEndpointToHostAction controls what happens to\n traffic that goes from a workload endpoint to the host itself (after\n the traffic hits the endpoint egress policy). By default Calico\n blocks traffic from workload endpoints to the host itself with an\n iptables DROP action. If you want to allow some or all traffic\n from endpoint to host, set this parameter to RETURN or ACCEPT. Use\n RETURN if you have your own rules in the iptables INPUT chain;\n Calico will insert its rules at the top of that chain, then RETURN\n packets to the INPUT chain once it has completed processing workload\n endpoint egress policy. Use ACCEPT to unconditionally accept packets\n from workloads after processing workload endpoint egress policy.\n [Default: Drop]\"\n type: string\n deviceRouteProtocol:\n description:\n This defines the route protocol added to programmed device\n routes, by default this will be RTPROT_BOOT when left blank.\n type: integer\n deviceRouteSourceAddress:\n description:\n This is the source address to use on programmed device\n routes. By default the source address is left blank, leaving the\n kernel to choose the source address used.\n type: string\n disableConntrackInvalidCheck:\n type: boolean\n endpointReportingDelay:\n type: string\n endpointReportingEnabled:\n type: boolean\n externalNodesList:\n description:\n ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes\n which may source tunnel traffic and have the tunneled traffic be\n accepted at calico nodes.\n items:\n type: string\n type: array\n failsafeInboundHostPorts:\n description:\n \"FailsafeInboundHostPorts is a comma-delimited list of\n UDP/TCP ports that Felix will allow incoming traffic to host endpoints\n on irrespective of the security policy. This is useful to avoid\n accidentally cutting off a host with incorrect configuration. Each\n port should be specified as tcp: or udp:.\n For back-compatibility, if the protocol is not specified, it defaults\n to tcp. To disable all inbound host ports, use the value none.\n The default value allows ssh access and DHCP. [Default: tcp:22,\n udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]\"\n items:\n description:\n ProtoPort is combination of protocol and port, both\n must be specified.\n properties:\n port:\n type: integer\n protocol:\n type: string\n required:\n - port\n - protocol\n type: object\n type: array\n failsafeOutboundHostPorts:\n description:\n \"FailsafeOutboundHostPorts is a comma-delimited list\n of UDP/TCP ports that Felix will allow outgoing traffic from host\n endpoints to irrespective of the security policy. This is useful\n to avoid accidentally cutting off a host with incorrect configuration.\n Each port should be specified as tcp: or udp:.\n For back-compatibility, if the protocol is not specified, it defaults\n to tcp. To disable all outbound host ports, use the value none.\n The default value opens etcd's standard ports to ensure that Felix\n does not get cut off from etcd as well as allowing DHCP and DNS.\n [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667,\n udp:53, udp:67]\"\n items:\n description:\n ProtoPort is combination of protocol and port, both\n must be specified.\n properties:\n port:\n type: integer\n protocol:\n type: string\n required:\n - port\n - protocol\n type: object\n type: array\n featureDetectOverride:\n description:\n FeatureDetectOverride is used to override the feature\n detection. Values are specified in a comma separated list with no\n spaces, example; \"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\".\n \"true\" or \"false\" will force the feature, empty or omitted values\n are auto-detected.\n type: string\n genericXDPEnabled:\n description:\n \"GenericXDPEnabled enables Generic XDP so network cards\n that don't support XDP offload or driver modes can use XDP. This\n is not recommended since it doesn't provide better performance\n than iptables. [Default: false]\"\n type: boolean\n healthEnabled:\n type: boolean\n healthHost:\n type: string\n healthPort:\n type: integer\n interfaceExclude:\n description:\n \"InterfaceExclude is a comma-separated list of interfaces\n that Felix should exclude when monitoring for host endpoints. The\n default value ensures that Felix ignores Kubernetes' IPVS dummy\n interface, which is used internally by kube-proxy. If you want to\n exclude multiple interface names using a single value, the list\n supports regular expressions. For regular expressions you must wrap\n the value with '/'. For example having values '/^kube/,veth1'\n will exclude all interfaces that begin with 'kube' and also the\n interface 'veth1'. [Default: kube-ipvs0]\"\n type: string\n interfacePrefix:\n description:\n \"InterfacePrefix is the interface name prefix that identifies\n workload endpoints and so distinguishes them from host endpoint\n interfaces. Note: in environments other than bare metal, the orchestrators\n configure this appropriately. For example our Kubernetes and Docker\n integrations set the 'cali' value, and our OpenStack integration\n sets the 'tap' value. [Default: cali]\"\n type: string\n interfaceRefreshInterval:\n description:\n InterfaceRefreshInterval is the period at which Felix\n rescans local interfaces to verify their state. The rescan can be\n disabled by setting the interval to 0.\n type: string\n ipipEnabled:\n type: boolean\n ipipMTU:\n description:\n \"IPIPMTU is the MTU to set on the tunnel device. See\n Configuring MTU [Default: 1440]\"\n type: integer\n ipsetsRefreshInterval:\n description:\n \"IpsetsRefreshInterval is the period at which Felix re-checks\n all iptables state to ensure that no other process has accidentally\n broken Calico's rules. Set to 0 to disable iptables refresh. [Default:\n 90s]\"\n type: string\n iptablesBackend:\n description:\n IptablesBackend specifies which backend of iptables will\n be used. The default is legacy.\n type: string\n iptablesFilterAllowAction:\n type: string\n iptablesLockFilePath:\n description:\n \"IptablesLockFilePath is the location of the iptables\n lock file. You may need to change this if the lock file is not in\n its standard location (for example if you have mapped it into Felix's\n container at a different path). [Default: /run/xtables.lock]\"\n type: string\n iptablesLockProbeInterval:\n description:\n \"IptablesLockProbeInterval is the time that Felix will\n wait between attempts to acquire the iptables lock if it is not\n available. Lower values make Felix more responsive when the lock\n is contended, but use more CPU. [Default: 50ms]\"\n type: string\n iptablesLockTimeout:\n description:\n \"IptablesLockTimeout is the time that Felix will wait\n for the iptables lock, or 0, to disable. To use this feature, Felix\n must share the iptables lock file with all other processes that\n also take the lock. When running Felix inside a container, this\n requires the /run directory of the host to be mounted into the calico/node\n or calico/felix container. [Default: 0s disabled]\"\n type: string\n iptablesMangleAllowAction:\n type: string\n iptablesMarkMask:\n description:\n \"IptablesMarkMask is the mask that Felix selects its\n IPTables Mark bits from. Should be a 32 bit hexadecimal number with\n at least 8 bits set, none of which clash with any other mark bits\n in use on the system. [Default: 0xff000000]\"\n format: int32\n type: integer\n iptablesNATOutgoingInterfaceFilter:\n type: string\n iptablesPostWriteCheckInterval:\n description:\n \"IptablesPostWriteCheckInterval is the period after Felix\n has done a write to the dataplane that it schedules an extra read\n back in order to check the write was not clobbered by another process.\n This should only occur if another application on the system doesn't\n respect the iptables lock. [Default: 1s]\"\n type: string\n iptablesRefreshInterval:\n description:\n \"IptablesRefreshInterval is the period at which Felix\n re-checks the IP sets in the dataplane to ensure that no other process\n has accidentally broken Calico's rules. Set to 0 to disable IP\n sets refresh. Note: the default for this value is lower than the\n other refresh intervals as a workaround for a Linux kernel bug that\n was fixed in kernel version 4.11. If you are using v4.11 or greater\n you may want to set this to, a higher value to reduce Felix CPU\n usage. [Default: 10s]\"\n type: string\n ipv6Support:\n type: boolean\n kubeNodePortRanges:\n description:\n \"KubeNodePortRanges holds list of port ranges used for\n service node ports. Only used if felix detects kube-proxy running\n in ipvs mode. Felix uses these ranges to separate host and workload\n traffic. [Default: 30000:32767].\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n logFilePath:\n description:\n \"LogFilePath is the full path to the Felix log. Set to\n none to disable file logging. [Default: /var/log/calico/felix.log]\"\n type: string\n logPrefix:\n description:\n \"LogPrefix is the log prefix that Felix uses when rendering\n LOG rules. [Default: calico-packet]\"\n type: string\n logSeverityFile:\n description:\n \"LogSeverityFile is the log severity above which logs\n are sent to the log file. [Default: Info]\"\n type: string\n logSeverityScreen:\n description:\n \"LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]\"\n type: string\n logSeveritySys:\n description:\n \"LogSeveritySys is the log severity above which logs\n are sent to the syslog. Set to None for no logging to syslog. [Default:\n Info]\"\n type: string\n maxIpsetSize:\n type: integer\n metadataAddr:\n description:\n \"MetadataAddr is the IP address or domain name of the\n server that can answer VM queries for cloud-init metadata. In OpenStack,\n this corresponds to the machine running nova-api (or in Ubuntu,\n nova-api-metadata). A value of none (case insensitive) means that\n Felix should not set up any NAT rule for the metadata path. [Default:\n 127.0.0.1]\"\n type: string\n metadataPort:\n description:\n \"MetadataPort is the port of the metadata server. This,\n combined with global.MetadataAddr (if not 'None'), is used to\n set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort.\n In most cases this should not need to be changed [Default: 8775].\"\n type: integer\n mtuIfacePattern:\n description:\n MTUIfacePattern is a regular expression that controls\n which interfaces Felix should scan in order to calculate the host's\n MTU. This should not match workload interfaces (usually named cali...).\n type: string\n natOutgoingAddress:\n description:\n NATOutgoingAddress specifies an address to use when performing\n source NAT for traffic in a natOutgoing pool that is leaving the\n network. By default the address used is an address on the interface\n the traffic is leaving on (ie it uses the iptables MASQUERADE target)\n type: string\n natPortRange:\n anyOf:\n - type: integer\n - type: string\n description:\n NATPortRange specifies the range of ports that is used\n for port mapping when doing outgoing NAT. When unset the default\n behavior of the network stack is used.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n netlinkTimeout:\n type: string\n openstackRegion:\n description:\n \"OpenstackRegion is the name of the region that a particular\n Felix belongs to. In a multi-region Calico/OpenStack deployment,\n this must be configured somehow for each Felix (here in the datamodel,\n or in felix.cfg or the environment on each compute node), and must\n match the [calico] openstack_region value configured in neutron.conf\n on each node. [Default: Empty]\"\n type: string\n policySyncPathPrefix:\n description:\n \"PolicySyncPathPrefix is used to by Felix to communicate\n policy changes to external services, like Application layer policy.\n [Default: Empty]\"\n type: string\n prometheusGoMetricsEnabled:\n description:\n \"PrometheusGoMetricsEnabled disables Go runtime metrics\n collection, which the Prometheus client does by default, when set\n to false. This reduces the number of metrics reported, reducing\n Prometheus load. [Default: true]\"\n type: boolean\n prometheusMetricsEnabled:\n description:\n \"PrometheusMetricsEnabled enables the Prometheus metrics\n server in Felix if set to true. [Default: false]\"\n type: boolean\n prometheusMetricsHost:\n description:\n \"PrometheusMetricsHost is the host that the Prometheus\n metrics server should bind to. [Default: empty]\"\n type: string\n prometheusMetricsPort:\n description:\n \"PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. [Default: 9091]\"\n type: integer\n prometheusProcessMetricsEnabled:\n description:\n \"PrometheusProcessMetricsEnabled disables process metrics\n collection, which the Prometheus client does by default, when set\n to false. This reduces the number of metrics reported, reducing\n Prometheus load. [Default: true]\"\n type: boolean\n removeExternalRoutes:\n description:\n Whether or not to remove device routes that have not\n been programmed by Felix. Disabling this will allow external applications\n to also add device routes. This is enabled by default which means\n we will remove externally added routes.\n type: boolean\n reportingInterval:\n description:\n \"ReportingInterval is the interval at which Felix reports\n its status into the datastore or 0 to disable. Must be non-zero\n in OpenStack deployments. [Default: 30s]\"\n type: string\n reportingTTL:\n description:\n \"ReportingTTL is the time-to-live setting for process-wide\n status reports. [Default: 90s]\"\n type: string\n routeRefreshInterval:\n description:\n \"RouteRefreshInterval is the period at which Felix re-checks\n the routes in the dataplane to ensure that no other process has\n accidentally broken Calico's rules. Set to 0 to disable route refresh.\n [Default: 90s]\"\n type: string\n routeSource:\n description:\n \"RouteSource configures where Felix gets its routing\n information. - WorkloadIPs: use workload endpoints to construct\n routes. - CalicoIPAM: the default - use IPAM data to construct routes.\"\n type: string\n routeTableRange:\n description:\n Calico programs additional Linux route tables for various\n purposes. RouteTableRange specifies the indices of the route tables\n that Calico should use.\n properties:\n max:\n type: integer\n min:\n type: integer\n required:\n - max\n - min\n type: object\n serviceLoopPrevention:\n description:\n 'When service IP advertisement is enabled, prevent routing\n loops to service IPs that are not in use, by dropping or rejecting\n packets that do not get DNAT''d by kube-proxy. Unless set to \"Disabled\",\n in which case such routing loops continue to be allowed. [Default:\n Drop]'\n type: string\n sidecarAccelerationEnabled:\n description:\n \"SidecarAccelerationEnabled enables experimental sidecar\n acceleration [Default: false]\"\n type: boolean\n usageReportingEnabled:\n description:\n \"UsageReportingEnabled reports anonymous Calico version\n number and cluster size to projectcalico.org. Logs warnings returned\n by the usage server. For example, if a significant security vulnerability\n has been discovered in the version of Calico being used. [Default:\n true]\"\n type: boolean\n usageReportingInitialDelay:\n description:\n \"UsageReportingInitialDelay controls the minimum delay\n before Felix makes a report. [Default: 300s]\"\n type: string\n usageReportingInterval:\n description:\n \"UsageReportingInterval controls the interval at which\n Felix makes reports. [Default: 86400s]\"\n type: string\n useInternalDataplaneDriver:\n type: boolean\n vxlanEnabled:\n type: boolean\n vxlanMTU:\n description:\n \"VXLANMTU is the MTU to set on the tunnel device. See\n Configuring MTU [Default: 1440]\"\n type: integer\n vxlanPort:\n type: integer\n vxlanVNI:\n type: integer\n wireguardEnabled:\n description:\n \"WireguardEnabled controls whether Wireguard is enabled.\n [Default: false]\"\n type: boolean\n wireguardInterfaceName:\n description:\n \"WireguardInterfaceName specifies the name to use for\n the Wireguard interface. [Default: wg.calico]\"\n type: string\n wireguardListeningPort:\n description:\n \"WireguardListeningPort controls the listening port used\n by Wireguard. [Default: 51820]\"\n type: integer\n wireguardMTU:\n description:\n \"WireguardMTU controls the MTU on the Wireguard interface.\n See Configuring MTU [Default: 1420]\"\n type: integer\n wireguardRoutingRulePriority:\n description:\n \"WireguardRoutingRulePriority controls the priority value\n to use for the Wireguard routing rule. [Default: 99]\"\n type: integer\n xdpEnabled:\n description:\n \"XDPEnabled enables XDP acceleration for suitable untracked\n incoming deny rules. [Default: true]\"\n type: boolean\n xdpRefreshInterval:\n description:\n \"XDPRefreshInterval is the period at which Felix re-checks\n all XDP state to ensure that no other process has accidentally broken\n Calico's BPF maps or attached programs. Set to 0 to disable XDP\n refresh. [Default: 90s]\"\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: globalnetworkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: GlobalNetworkPolicy\n listKind: GlobalNetworkPolicyList\n plural: globalnetworkpolicies\n singular: globalnetworkpolicy\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n properties:\n applyOnForward:\n description:\n ApplyOnForward indicates to apply the rules in this policy\n on forward traffic.\n type: boolean\n doNotTrack:\n description:\n DoNotTrack indicates whether packets matched by the rules\n in this policy should go through the data plane's connection tracking,\n such as Linux conntrack. If True, the rules in this policy are\n applied before any data plane connection tracking, and packets allowed\n by this policy are marked as not to be tracked.\n type: boolean\n egress:\n description:\n The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description:\n \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with Not. All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description:\n Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description:\n \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and Selector are defined on the same rule, then only workload\n endpoints that are matched by both selectors will be selected\n by the rule. \\n For NetworkPolicy, an empty NamespaceSelector\n implies that the Selector is limited to selecting only\n workload endpoints in the same namespace as the NetworkPolicy.\n \\n For NetworkPolicy, `global()` NamespaceSelector implies\n that the Selector is limited to selecting only GlobalNetworkSet\n or HostEndpoint. \\n For GlobalNetworkPolicy, an empty\n NamespaceSelector implies the Selector applies to workload\n endpoints across all namespaces.\"\n type: string\n nets:\n description:\n Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description:\n NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description:\n NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description:\n NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description:\n \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description:\n \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label my_label. \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label my_label.\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description:\n ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description:\n Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description:\n Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n type: object\n http:\n description:\n HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description:\n Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description:\n \"Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR'd together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.\"\n items:\n description:\n \"HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix\"\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description:\n ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description:\n Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description:\n Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description:\n IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description:\n Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description:\n Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description:\n Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description:\n Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description:\n NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description:\n \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description:\n Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description:\n \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and Selector are defined on the same rule, then only workload\n endpoints that are matched by both selectors will be selected\n by the rule. \\n For NetworkPolicy, an empty NamespaceSelector\n implies that the Selector is limited to selecting only\n workload endpoints in the same namespace as the NetworkPolicy.\n \\n For NetworkPolicy, `global()` NamespaceSelector implies\n that the Selector is limited to selecting only GlobalNetworkSet\n or HostEndpoint. \\n For GlobalNetworkPolicy, an empty\n NamespaceSelector implies the Selector applies to workload\n endpoints across all namespaces.\"\n type: string\n nets:\n description:\n Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description:\n NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description:\n NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description:\n NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description:\n \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description:\n \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label my_label. \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label my_label.\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description:\n ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description:\n Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description:\n Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description:\n The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description:\n \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with Not. All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description:\n Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description:\n \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and Selector are defined on the same rule, then only workload\n endpoints that are matched by both selectors will be selected\n by the rule. \\n For NetworkPolicy, an empty NamespaceSelector\n implies that the Selector is limited to selecting only\n workload endpoints in the same namespace as the NetworkPolicy.\n \\n For NetworkPolicy, `global()` NamespaceSelector implies\n that the Selector is limited to selecting only GlobalNetworkSet\n or HostEndpoint. \\n For GlobalNetworkPolicy, an empty\n NamespaceSelector implies the Selector applies to workload\n endpoints across all namespaces.\"\n type: string\n nets:\n description:\n Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description:\n NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description:\n NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description:\n NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description:\n \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description:\n \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label my_label. \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label my_label.\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description:\n ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description:\n Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description:\n Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n type: object\n http:\n description:\n HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description:\n Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description:\n \"Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR'd together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.\"\n items:\n description:\n \"HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix\"\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description:\n ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description:\n Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description:\n Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description:\n IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description:\n Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description:\n Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description:\n Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description:\n Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description:\n NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description:\n \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description:\n Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description:\n \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and Selector are defined on the same rule, then only workload\n endpoints that are matched by both selectors will be selected\n by the rule. \\n For NetworkPolicy, an empty NamespaceSelector\n implies that the Selector is limited to selecting only\n workload endpoints in the same namespace as the NetworkPolicy.\n \\n For NetworkPolicy, `global()` NamespaceSelector implies\n that the Selector is limited to selecting only GlobalNetworkSet\n or HostEndpoint. \\n For GlobalNetworkPolicy, an empty\n NamespaceSelector implies the Selector applies to workload\n endpoints across all namespaces.\"\n type: string\n nets:\n description:\n Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description:\n NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description:\n NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description:\n NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description:\n \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description:\n \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label my_label. \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label my_label.\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description:\n ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description:\n Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description:\n Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n namespaceSelector:\n description:\n NamespaceSelector is an optional field for an expression\n used to select a pod based on namespaces.\n type: string\n order:\n description:\n Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order. If the order is omitted, it may be\n considered to be \"infinite\" - i.e. the policy will be applied last. Policies\n with identical order will be applied in alphanumerical order based\n on the Policy \"Name\".\n type: number\n preDNAT:\n description:\n PreDNAT indicates to apply the rules in this policy before\n any DNAT.\n type: boolean\n selector:\n description:\n \"The selector is an expression used to pick pick out\n the endpoints that the policy should be applied to. \\n Selector\n expressions follow this syntax: \\n \\tlabel == \\\"string_literal\\\"\n \\ -> comparison, e.g. my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\"\n \\ -> not equal; also matches if label is not present \\tlabel in\n { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is\n one of \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\",\n ... } -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\",\n \\\"c\\\" \\thas(label_name) -> True if that label is present \\t! expr\n -> negation of expr \\texpr && expr -> Short-circuit and \\texpr\n || expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description:\n ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n types:\n description:\n \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress rules are present in the policy. The\n default is: \\n - [ PolicyTypeIngress ], if there are no Egress rules\n (including the case where there are also no Ingress rules) \\n\n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress\n rules \\n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are\n both Ingress and Egress rules. \\n When the policy is read back again,\n Types will always be one of these values, never empty or nil.\"\n items:\n description:\n PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: globalnetworksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: GlobalNetworkSet\n listKind: GlobalNetworkSetList\n plural: globalnetworksets\n singular: globalnetworkset\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description:\n GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs\n that share labels to allow rules to refer to them via selectors. The labels\n of GlobalNetworkSet are not namespaced.\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n GlobalNetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: hostendpoints.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: HostEndpoint\n listKind: HostEndpointList\n plural: hostendpoints\n singular: hostendpoint\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n HostEndpointSpec contains the specification for a HostEndpoint\n resource.\n properties:\n expectedIPs:\n description:\n \"The expected IP addresses (IPv4 and IPv6) of the endpoint.\n If \\\"InterfaceName\\\" is not present, Calico will look for an interface\n matching any of the IPs in the list and apply policy to that. Note:\n \\tWhen using the selector match criteria in an ingress or egress\n security Policy \\tor Profile, Calico converts the selector into\n a set of IP addresses. For host \\tendpoints, the ExpectedIPs field\n is used for that purpose. (If only the interface \\tname is specified,\n Calico does not learn the IPs of the interface for use in match\n \\tcriteria.)\"\n items:\n type: string\n type: array\n interfaceName:\n description:\n \"Either \\\"*\\\", or the name of a specific Linux interface\n to apply policy to; or empty. \\\"*\\\" indicates that this HostEndpoint\n governs all traffic to, from or through the default network namespace\n of the host named by the \\\"Node\\\" field; entering and leaving that\n namespace via any interface, including those from/to non-host-networked\n local workloads. \\n If InterfaceName is not \\\"*\\\", this HostEndpoint\n only governs traffic that enters or leaves the host through the\n specific interface named by InterfaceName, or - when InterfaceName\n is empty - through the specific interface that has one of the IPs\n in ExpectedIPs. Therefore, when InterfaceName is empty, at least\n one expected IP must be specified. Only external interfaces (such\n as eth0) are supported here; it isn't possible for a HostEndpoint\n to protect traffic through a specific local workload interface.\n \\n Note: Only some kinds of policy are implemented for \\\"*\\\" HostEndpoints;\n initially just pre-DNAT policy. Please check Calico documentation\n for the latest position.\"\n type: string\n node:\n description: The node name identifying the Calico node instance.\n type: string\n ports:\n description:\n Ports contains the endpoint's named ports, which may\n be referenced in security policy rules.\n items:\n properties:\n name:\n type: string\n port:\n type: integer\n protocol:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n required:\n - name\n - port\n - protocol\n type: object\n type: array\n profiles:\n description:\n A list of identifiers of security Profile objects that\n apply to this endpoint. Each profile is applied in the order that\n they appear in this list. Profile rules are applied after the selector-based\n security policy.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamblocks.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMBlock\n listKind: IPAMBlockList\n plural: ipamblocks\n singular: ipamblock\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n IPAMBlockSpec contains the specification for an IPAMBlock\n resource.\n properties:\n affinity:\n type: string\n allocations:\n items:\n type: integer\n # TODO: This nullable is manually added in. We should update controller-gen\n # to handle []*int properly itself.\n nullable: true\n type: array\n attributes:\n items:\n properties:\n handle_id:\n type: string\n secondary:\n additionalProperties:\n type: string\n type: object\n type: object\n type: array\n cidr:\n type: string\n deleted:\n type: boolean\n strictAffinity:\n type: boolean\n unallocated:\n items:\n type: integer\n type: array\n required:\n - allocations\n - attributes\n - cidr\n - strictAffinity\n - unallocated\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n IPAMConfigSpec contains the specification for an IPAMConfig\n resource.\n properties:\n autoAllocateBlocks:\n type: boolean\n maxBlocksPerHost:\n description:\n MaxBlocksPerHost, if non-zero, is the max number of blocks\n that can be affine to each host.\n type: integer\n strictAffinity:\n type: boolean\n required:\n - autoAllocateBlocks\n - strictAffinity\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamhandles.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMHandle\n listKind: IPAMHandleList\n plural: ipamhandles\n singular: ipamhandle\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n IPAMHandleSpec contains the specification for an IPAMHandle\n resource.\n properties:\n block:\n additionalProperties:\n type: integer\n type: object\n deleted:\n type: boolean\n handleID:\n type: string\n required:\n - block\n - handleID\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPPool\n listKind: IPPoolList\n plural: ippools\n singular: ippool\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description: IPPoolSpec contains the specification for an IPPool resource.\n properties:\n blockSize:\n description:\n The block size to use for IP address assignments from\n this pool. Defaults to 26 for IPv4 and 112 for IPv6.\n type: integer\n cidr:\n description: The pool CIDR.\n type: string\n disabled:\n description:\n When disabled is true, Calico IPAM will not assign addresses\n from this pool.\n type: boolean\n ipip:\n description:\n \"Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.\"\n properties:\n enabled:\n description:\n When enabled is true, ipip tunneling will be used\n to deliver packets to destinations within this pool.\n type: boolean\n mode:\n description:\n The IPIP mode. This can be one of \"always\" or \"cross-subnet\". A\n mode of \"always\" will also use IPIP tunneling for routing to\n destination IP addresses within this pool. A mode of \"cross-subnet\"\n will only use IPIP tunneling when the destination node is on\n a different subnet to the originating node. The default value\n (if not specified) is \"always\".\n type: string\n type: object\n ipipMode:\n description:\n Contains configuration for IPIP tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. IPIP tunneling\n is disabled).\n type: string\n nat-outgoing:\n description:\n \"Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.\"\n type: boolean\n natOutgoing:\n description:\n When nat-outgoing is true, packets sent from Calico networked\n containers in this pool to destinations outside of this pool will\n be masqueraded.\n type: boolean\n nodeSelector:\n description:\n Allows IPPool to allocate for a specific node by label\n selector.\n type: string\n vxlanMode:\n description:\n Contains configuration for VXLAN tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. VXLAN\n tunneling is disabled).\n type: string\n required:\n - cidr\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: kubecontrollersconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: KubeControllersConfiguration\n listKind: KubeControllersConfigurationList\n plural: kubecontrollersconfigurations\n singular: kubecontrollersconfiguration\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n KubeControllersConfigurationSpec contains the values of the\n Kubernetes controllers configuration.\n properties:\n controllers:\n description:\n Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description:\n Namespace enables and configures the namespace controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]\"\n type: string\n type: object\n node:\n description:\n Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description:\n HostEndpoint controls syncing nodes to host endpoints.\n Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description:\n \"AutoCreate enables automatic creation of\n host endpoints for every node. [Default: Disabled]\"\n type: string\n type: object\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]\"\n type: string\n syncLabels:\n description:\n \"SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]\"\n type: string\n type: object\n policy:\n description:\n Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]\"\n type: string\n type: object\n serviceAccount:\n description:\n ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]\"\n type: string\n type: object\n workloadEndpoint:\n description:\n WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]\"\n type: string\n type: object\n type: object\n etcdV3CompactionPeriod:\n description:\n \"EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]\"\n type: string\n healthChecks:\n description:\n \"HealthChecks enables or disables support for health\n checks [Default: Enabled]\"\n type: string\n logSeverityScreen:\n description:\n \"LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]\"\n type: string\n required:\n - controllers\n type: object\n status:\n description:\n KubeControllersConfigurationStatus represents the status\n of the configuration. It's useful for admins to be able to see the actual\n config that was applied, which can be modified by environment variables\n on the kube-controllers process.\n properties:\n environmentVars:\n additionalProperties:\n type: string\n description:\n EnvironmentVars contains the environment variables on\n the kube-controllers that influenced the RunningConfig.\n type: object\n runningConfig:\n description:\n RunningConfig contains the effective config that is running\n in the kube-controllers pod, after merging the API resource with\n any environment variables.\n properties:\n controllers:\n description:\n Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description:\n Namespace enables and configures the namespace\n controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]\"\n type: string\n type: object\n node:\n description:\n Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description:\n HostEndpoint controls syncing nodes to host\n endpoints. Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description:\n \"AutoCreate enables automatic creation\n of host endpoints for every node. [Default: Disabled]\"\n type: string\n type: object\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]\"\n type: string\n syncLabels:\n description:\n \"SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]\"\n type: string\n type: object\n policy:\n description:\n Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]\"\n type: string\n type: object\n serviceAccount:\n description:\n ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]\"\n type: string\n type: object\n workloadEndpoint:\n description:\n WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description:\n \"ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]\"\n type: string\n type: object\n type: object\n etcdV3CompactionPeriod:\n description:\n \"EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]\"\n type: string\n healthChecks:\n description:\n \"HealthChecks enables or disables support for health\n checks [Default: Enabled]\"\n type: string\n logSeverityScreen:\n description:\n \"LogSeverityScreen is the log severity above which\n logs are sent to the stdout. [Default: Info]\"\n type: string\n required:\n - controllers\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkPolicy\n listKind: NetworkPolicyList\n plural: networkpolicies\n singular: networkpolicy\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n properties:\n egress:\n description:\n The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description:\n \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with Not. All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description:\n Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description:\n \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and Selector are defined on the same rule, then only workload\n endpoints that are matched by both selectors will be selected\n by the rule. \\n For NetworkPolicy, an empty NamespaceSelector\n implies that the Selector is limited to selecting only\n workload endpoints in the same namespace as the NetworkPolicy.\n \\n For NetworkPolicy, `global()` NamespaceSelector implies\n that the Selector is limited to selecting only GlobalNetworkSet\n or HostEndpoint. \\n For GlobalNetworkPolicy, an empty\n NamespaceSelector implies the Selector applies to workload\n endpoints across all namespaces.\"\n type: string\n nets:\n description:\n Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description:\n NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description:\n NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description:\n NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description:\n \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description:\n \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label my_label. \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label my_label.\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description:\n ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description:\n Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description:\n Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n type: object\n http:\n description:\n HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description:\n Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description:\n \"Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR'd together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.\"\n items:\n description:\n \"HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix\"\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description:\n ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description:\n Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description:\n Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description:\n IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description:\n Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description:\n Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description:\n Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description:\n Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description:\n NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description:\n \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description:\n Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description:\n \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and Selector are defined on the same rule, then only workload\n endpoints that are matched by both selectors will be selected\n by the rule. \\n For NetworkPolicy, an empty NamespaceSelector\n implies that the Selector is limited to selecting only\n workload endpoints in the same namespace as the NetworkPolicy.\n \\n For NetworkPolicy, `global()` NamespaceSelector implies\n that the Selector is limited to selecting only GlobalNetworkSet\n or HostEndpoint. \\n For GlobalNetworkPolicy, an empty\n NamespaceSelector implies the Selector applies to workload\n endpoints across all namespaces.\"\n type: string\n nets:\n description:\n Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description:\n NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description:\n NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description:\n NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description:\n \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description:\n \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label my_label. \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label my_label.\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description:\n ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description:\n Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description:\n Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description:\n The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description:\n \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with Not. All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description:\n Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description:\n \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and Selector are defined on the same rule, then only workload\n endpoints that are matched by both selectors will be selected\n by the rule. \\n For NetworkPolicy, an empty NamespaceSelector\n implies that the Selector is limited to selecting only\n workload endpoints in the same namespace as the NetworkPolicy.\n \\n For NetworkPolicy, `global()` NamespaceSelector implies\n that the Selector is limited to selecting only GlobalNetworkSet\n or HostEndpoint. \\n For GlobalNetworkPolicy, an empty\n NamespaceSelector implies the Selector applies to workload\n endpoints across all namespaces.\"\n type: string\n nets:\n description:\n Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description:\n NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description:\n NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description:\n NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description:\n \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description:\n \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label my_label. \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label my_label.\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description:\n ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description:\n Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description:\n Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n type: object\n http:\n description:\n HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description:\n Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description:\n \"Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR'd together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.\"\n items:\n description:\n \"HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix\"\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description:\n ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description:\n Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description:\n Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description:\n IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description:\n Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description:\n Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description:\n Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description:\n Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description:\n NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description:\n \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description:\n Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description:\n \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and Selector are defined on the same rule, then only workload\n endpoints that are matched by both selectors will be selected\n by the rule. \\n For NetworkPolicy, an empty NamespaceSelector\n implies that the Selector is limited to selecting only\n workload endpoints in the same namespace as the NetworkPolicy.\n \\n For NetworkPolicy, `global()` NamespaceSelector implies\n that the Selector is limited to selecting only GlobalNetworkSet\n or HostEndpoint. \\n For GlobalNetworkPolicy, an empty\n NamespaceSelector implies the Selector applies to workload\n endpoints across all namespaces.\"\n type: string\n nets:\n description:\n Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description:\n NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description:\n NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description:\n NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description:\n \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description:\n \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label my_label. \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label my_label.\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description:\n ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description:\n Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description:\n Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n order:\n description:\n Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order. If the order is omitted, it may be\n considered to be \"infinite\" - i.e. the policy will be applied last. Policies\n with identical order will be applied in alphanumerical order based\n on the Policy \"Name\".\n type: number\n selector:\n description:\n \"The selector is an expression used to pick pick out\n the endpoints that the policy should be applied to. \\n Selector\n expressions follow this syntax: \\n \\tlabel == \\\"string_literal\\\"\n \\ -> comparison, e.g. my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\"\n \\ -> not equal; also matches if label is not present \\tlabel in\n { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is\n one of \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\",\n ... } -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\",\n \\\"c\\\" \\thas(label_name) -> True if that label is present \\t! expr\n -> negation of expr \\texpr && expr -> Short-circuit and \\texpr\n || expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description:\n ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n types:\n description:\n \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress are present in the policy. The default\n is: \\n - [ PolicyTypeIngress ], if there are no Egress rules (including\n the case where there are also no Ingress rules) \\n - [ PolicyTypeEgress\n ], if there are Egress rules but no Ingress rules \\n - [ PolicyTypeIngress,\n PolicyTypeEgress ], if there are both Ingress and Egress rules.\n \\n When the policy is read back again, Types will always be one\n of these values, never empty or nil.\"\n items:\n description:\n PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkSet\n listKind: NetworkSetList\n plural: networksets\n singular: networkset\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet.\n properties:\n apiVersion:\n description:\n \"APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\"\n type: string\n kind:\n description:\n \"Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\"\n type: string\n metadata:\n type: object\n spec:\n description:\n NetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n\n---\n---\n# Source: calico/templates/calico-kube-controllers-rbac.yaml\n\n# Include a clusterrole for the kube-controllers component,\n# and bind it to the calico-kube-controllers serviceaccount.\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-kube-controllers\nrules:\n # Nodes are watched to monitor for deletions.\n - apiGroups: [\"\"]\n resources:\n - nodes\n verbs:\n - watch\n - list\n - get\n # Pods are queried to check for existence.\n - apiGroups: [\"\"]\n resources:\n - pods\n verbs:\n - get\n # IPAM resources are manipulated when nodes are deleted.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ippools\n verbs:\n - list\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n verbs:\n - get\n - list\n - create\n - update\n - delete\n # kube-controllers manages hostendpoints.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - hostendpoints\n verbs:\n - get\n - list\n - create\n - update\n - delete\n # Needs access to update clusterinformations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - clusterinformations\n verbs:\n - get\n - create\n - update\n # KubeControllersConfiguration is where it gets its config\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - kubecontrollersconfigurations\n verbs:\n # read its own config\n - get\n # create a default if none exists\n - create\n # update status\n - update\n # watch for changes\n - watch\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-kube-controllers\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-kube-controllers\nsubjects:\n - kind: ServiceAccount\n name: calico-kube-controllers\n namespace: kube-system\n---\n---\n# Source: calico/templates/calico-node-rbac.yaml\n# Include a clusterrole for the calico-node DaemonSet,\n# and bind it to the calico-node serviceaccount.\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-node\nrules:\n # The CNI plugin needs to get pods, nodes, and namespaces.\n - apiGroups: [\"\"]\n resources:\n - pods\n - nodes\n - namespaces\n verbs:\n - get\n - apiGroups: [\"\"]\n resources:\n - endpoints\n - services\n verbs:\n # Used to discover service IPs for advertisement.\n - watch\n - list\n # Used to discover Typhas.\n - get\n # Pod CIDR auto-detection on kubeadm needs access to config maps.\n - apiGroups: [\"\"]\n resources:\n - configmaps\n verbs:\n - get\n - apiGroups: [\"\"]\n resources:\n - nodes/status\n verbs:\n # Needed for clearing NodeNetworkUnavailable flag.\n - patch\n # Calico stores some configuration information in node annotations.\n - update\n # Watch for changes to Kubernetes NetworkPolicies.\n - apiGroups: [\"networking.k8s.io\"]\n resources:\n - networkpolicies\n verbs:\n - watch\n - list\n # Used by Calico for policy information.\n - apiGroups: [\"\"]\n resources:\n - pods\n - namespaces\n - serviceaccounts\n verbs:\n - list\n - watch\n # The CNI plugin patches pods/status.\n - apiGroups: [\"\"]\n resources:\n - pods/status\n verbs:\n - patch\n # Calico monitors various CRDs for config.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - globalfelixconfigs\n - felixconfigurations\n - bgppeers\n - globalbgpconfigs\n - bgpconfigurations\n - ippools\n - ipamblocks\n - globalnetworkpolicies\n - globalnetworksets\n - networkpolicies\n - networksets\n - clusterinformations\n - hostendpoints\n - blockaffinities\n verbs:\n - get\n - list\n - watch\n # Calico must create and update some CRDs on startup.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ippools\n - felixconfigurations\n - clusterinformations\n verbs:\n - create\n - update\n # Calico stores some configuration information on the node.\n - apiGroups: [\"\"]\n resources:\n - nodes\n verbs:\n - get\n - list\n - watch\n # These permissions are only required for upgrade from v2.6, and can\n # be removed after upgrade or on fresh installations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - bgpconfigurations\n - bgppeers\n verbs:\n - create\n - update\n # These permissions are required for Calico CNI to perform IPAM allocations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n verbs:\n - get\n - list\n - create\n - update\n - delete\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ipamconfigs\n verbs:\n - get\n # Block affinities must also be watchable by confd for route aggregation.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n verbs:\n - watch\n # The Calico IPAM migration needs to get daemonsets. These permissions can be\n # removed if not upgrading from an installation using host-local IPAM.\n - apiGroups: [\"apps\"]\n resources:\n - daemonsets\n verbs:\n - get\n\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-node\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-node\nsubjects:\n - kind: ServiceAccount\n name: calico-node\n namespace: kube-system\n\n---\n# Source: calico/templates/calico-node.yaml\n# This manifest installs the calico-node container, as well\n# as the CNI plugins and network config on\n# each master and worker node in a Kubernetes cluster.\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: calico-node\n namespace: kube-system\n labels:\n k8s-app: calico-node\nspec:\n selector:\n matchLabels:\n k8s-app: calico-node\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n template:\n metadata:\n labels:\n k8s-app: calico-node\n annotations:\n # This, along with the CriticalAddonsOnly toleration below,\n # marks the pod as a critical add-on, ensuring it gets\n # priority scheduling and that its resources are reserved\n # if it ever gets evicted.\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n # Make sure calico-node gets scheduled on all nodes.\n - effect: NoSchedule\n operator: Exists\n # Mark the pod as a critical add-on for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - effect: NoExecute\n operator: Exists\n serviceAccountName: calico-node\n # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a \"force\n # deletion\": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.\n terminationGracePeriodSeconds: 0\n priorityClassName: system-node-critical\n initContainers:\n # This container performs upgrade from host-local IPAM to calico-ipam.\n # It can be deleted if this is a fresh installation, or if you have already\n # upgraded to use calico-ipam.\n - name: upgrade-ipam\n image: docker.io/calico/cni:v3.21.1\n command: [\"/opt/cni/bin/calico-ipam\", \"-upgrade\"]\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n - name: KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: CALICO_NETWORKING_BACKEND\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: calico_backend\n volumeMounts:\n - mountPath: /var/lib/cni/networks\n name: host-local-net-dir\n - mountPath: /host/opt/cni/bin\n name: cni-bin-dir\n securityContext:\n privileged: true\n # This container installs the CNI binaries\n # and CNI network config file on each node.\n - name: install-cni\n image: docker.io/calico/cni:v3.21.1\n command: [\"/opt/cni/bin/install\"]\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: \"10-calico.conflist\"\n # The CNI network config to install on each node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: cni_network_config\n # Set the hostname based on the k8s node name.\n - name: KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Prevents the container from sleeping forever.\n - name: SLEEP\n value: \"false\"\n - name: CNI_NET_DIR\n value: \"/var/snap/microk8s/current/args/cni-network\"\n volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: cni-net-dir\n securityContext:\n privileged: true\n # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes\n # to communicate with Felix over the Policy Sync API.\n - name: flexvol-driver\n image: docker.io/calico/pod2daemon-flexvol:v3.21.1\n volumeMounts:\n - name: flexvol-driver-host\n mountPath: /host/driver\n securityContext:\n privileged: true\n containers:\n # Runs calico-node container on each Kubernetes node. This\n # container programs network policy and routes on each\n # host.\n - name: calico-node\n image: docker.io/calico/node:v3.21.1\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # Use Kubernetes API as the backing datastore.\n - name: DATASTORE_TYPE\n value: \"kubernetes\"\n # Wait for the datastore.\n - name: WAIT_FOR_DATASTORE\n value: \"true\"\n # Set based on the k8s node name.\n - name: NODENAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n # Choose the backend to use.\n - name: CALICO_NETWORKING_BACKEND\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: calico_backend\n # Cluster type to identify the deployment type\n - name: CLUSTER_TYPE\n value: \"k8s,bgp\"\n # Auto-detect the BGP IP address.\n - name: IP\n value: \"autodetect\"\n - name: IP_AUTODETECTION_METHOD\n value: \"can-reach=192.168.1.43\"\n # Enable IPIP\n #- name: CALICO_IPV4POOL_IPIP\n - name: CALICO_IPV4POOL_VXLAN\n value: \"Always\"\n # Set MTU for tunnel device used if ipip is enabled\n - name: FELIX_IPINIPMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Set MTU for the VXLAN tunnel device.\n - name: FELIX_VXLANMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Set MTU for the Wireguard tunnel device.\n - name: FELIX_WIREGUARDMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # The default IPv4 pool to create on startup if none exists. Pod IPs will be\n # chosen from this range. Changing this value after installation will have\n # no effect. This should fall within `--cluster-cidr`.\n - name: CALICO_IPV4POOL_CIDR\n value: \"10.1.0.0/16\"\n # Disable file logging so `kubectl logs` works.\n - name: CALICO_DISABLE_FILE_LOGGING\n value: \"true\"\n # Set Felix endpoint to host default action to ACCEPT.\n - name: FELIX_DEFAULTENDPOINTTOHOSTACTION\n value: \"ACCEPT\"\n # Disable IPv6 on Kubernetes.\n - name: FELIX_IPV6SUPPORT\n value: \"false\"\n # Set Felix logging to \"error\"\n - name: FELIX_LOGSEVERITYSCREEN\n value: \"error\"\n - name: FELIX_HEALTHENABLED\n value: \"true\"\n securityContext:\n privileged: true\n resources:\n requests:\n cpu: 250m\n livenessProbe:\n exec:\n command:\n - /bin/calico-node\n - -felix-live\n # - -bird-live\n periodSeconds: 10\n initialDelaySeconds: 10\n failureThreshold: 6\n readinessProbe:\n exec:\n command:\n - /bin/calico-node\n - -felix-ready\n # - -bird-ready\n periodSeconds: 10\n volumeMounts:\n - mountPath: /lib/modules\n name: lib-modules\n readOnly: true\n - mountPath: /run/xtables.lock\n name: xtables-lock\n readOnly: false\n - mountPath: /var/run/calico\n name: var-run-calico\n readOnly: false\n - mountPath: /var/lib/calico\n name: var-lib-calico\n readOnly: false\n - name: policysync\n mountPath: /var/run/nodeagent\n volumes:\n # Used by calico-node.\n - name: lib-modules\n hostPath:\n path: /lib/modules\n - name: var-run-calico\n hostPath:\n path: /var/snap/microk8s/current/var/run/calico\n - name: var-lib-calico\n hostPath:\n path: /var/snap/microk8s/current/var/lib/calico\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n # Used to install CNI.\n - name: cni-bin-dir\n hostPath:\n path: /var/snap/microk8s/current/opt/cni/bin\n - name: cni-net-dir\n hostPath:\n path: /var/snap/microk8s/current/args/cni-network\n # Mount in the directory for host-local IPAM allocations. This is\n # used when upgrading from host-local to calico-ipam, and can be removed\n # if not using the upgrade-ipam init container.\n - name: host-local-net-dir\n hostPath:\n path: /var/snap/microk8s/current/var/lib/cni/networks\n # Used to create per-pod Unix Domain Sockets\n - name: policysync\n hostPath:\n type: DirectoryOrCreate\n path: /var/snap/microk8s/current/var/run/nodeagent\n # Used to install Flex Volume Driver\n - name: flexvol-driver-host\n hostPath:\n type: DirectoryOrCreate\n path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-node\n namespace: kube-system\n\n---\n# Source: calico/templates/calico-kube-controllers.yaml\n# See https://github.com/projectcalico/kube-controllers\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\nspec:\n # The controllers can only have a single active instance.\n replicas: 1\n selector:\n matchLabels:\n k8s-app: calico-kube-controllers\n strategy:\n type: Recreate\n template:\n metadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\n annotations:\n scheduler.alpha.kubernetes.io/critical-pod: \"\"\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n tolerations:\n # Mark the pod as a critical add-on for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n serviceAccountName: calico-kube-controllers\n priorityClassName: system-cluster-critical\n containers:\n - name: calico-kube-controllers\n image: docker.io/calico/kube-controllers:v3.17.3\n env:\n # Choose which controllers to run.\n - name: ENABLED_CONTROLLERS\n value: node\n - name: DATASTORE_TYPE\n value: kubernetes\n readinessProbe:\n exec:\n command:\n - /usr/bin/check-status\n - -r\n\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n\n---\n# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict\n\napiVersion: policy/v1beta1\nkind: PodDisruptionBudget\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\nspec:\n maxUnavailable: 1\n selector:\n matchLabels:\n k8s-app: calico-kube-controllers\n\n---\n# Source: calico/templates/calico-etcd-secrets.yaml\n---\n# Source: calico/templates/calico-typha.yaml\n---\n# Source: calico/templates/configure-canal.yaml\n" }, { "path": "tests/unit/yamls/invalid.yaml", "content": "not a yaml" }, { "path": "tests/utils.py", "content": "import os.path\nimport datetime\nimport time\nimport yaml\nimport platform\nimport psutil\nfrom subprocess import check_output, CalledProcessError, check_call\n\n\narch_translate = {\"aarch64\": \"arm64\", \"x86_64\": \"amd64\"}\n\n\ndef get_arch():\n \"\"\"\n Returns the architecture we are running on\n \"\"\"\n return arch_translate[platform.machine()]\n\n\ndef run_until_success(cmd, timeout_insec=60, err_out=None):\n \"\"\"\n Run a command until it succeeds or times out.\n Args:\n cmd: Command to run\n timeout_insec: Time out in seconds\n err_out: If command fails and this is the output, return.\n\n Returns: The string output of the command\n\n \"\"\"\n deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec)\n while True:\n try:\n output = check_output(cmd.split()).strip().decode(\"utf8\")\n return output.replace(\"\\\\n\", \"\\n\")\n except CalledProcessError as err:\n output = err.output.strip().decode(\"utf8\").replace(\"\\\\n\", \"\\n\")\n print(output)\n if output == err_out:\n return output\n if datetime.datetime.now() > deadline:\n raise\n print(\"Retrying {}\".format(cmd))\n time.sleep(3)\n\n\ndef kubectl(cmd, timeout_insec=300, err_out=None):\n \"\"\"\n Do a kubectl \n Args:\n cmd: left part of kubectl command\n timeout_insec: timeout for this job\n err_out: If command fails and this is the output, return.\n\n Returns: the kubectl response in a string\n\n \"\"\"\n cmd = \"/snap/bin/microk8s.kubectl \" + cmd\n return run_until_success(cmd, timeout_insec, err_out)\n\n\ndef docker(cmd):\n \"\"\"\n Do a docker \n Args:\n cmd: left part of docker command\n\n Returns: the docker response in a string\n\n \"\"\"\n docker_bin = \"/usr/bin/docker\"\n if os.path.isfile(\"/snap/bin/microk8s.docker\"):\n docker_bin = \"/snap/bin/microk8s.docker\"\n cmd = docker_bin + \" \" + cmd\n return run_until_success(cmd)\n\n\ndef kubectl_get(target, timeout_insec=300):\n \"\"\"\n Do a kubectl get and return the results in a yaml structure.\n Args:\n target: which resource we are getting\n timeout_insec: timeout for this job\n\n Returns: YAML structured response\n\n \"\"\"\n cmd = \"get -o yaml \" + target\n output = kubectl(cmd, timeout_insec)\n return yaml.safe_load(output)\n\n\ndef wait_for_pod_state(\n pod, namespace, desired_state, desired_reason=None, label=None, timeout_insec=600\n):\n \"\"\"\n Wait for a a pod state. If you do not specify a pod name and you set instead a label\n only the first pod will be checked.\n \"\"\"\n deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec)\n while True:\n if datetime.datetime.now() > deadline:\n all_res = kubectl(\"get all -A\")\n err_msg = f\"Pod {pod} not in {desired_state} after {timeout_insec} seconds, \"\n err_msg += f\"label: {label}, desired_reason: {desired_reason}.\\n\"\n print(err_msg)\n print(f\"kubectl get all -A\\n{all_res}\\n\")\n raise TimeoutError(err_msg)\n cmd = \"po {} -n {}\".format(pod, namespace)\n if label:\n cmd += \" -l {}\".format(label)\n data = kubectl_get(cmd, timeout_insec)\n if pod == \"\":\n if len(data[\"items\"]) > 0:\n status = data[\"items\"][0][\"status\"]\n else:\n status = []\n else:\n status = data[\"status\"]\n if \"containerStatuses\" in status:\n container_status = status[\"containerStatuses\"][0]\n state, details = list(container_status[\"state\"].items())[0]\n if desired_reason:\n reason = details.get(\"reason\")\n if state == desired_state and reason == desired_reason:\n break\n elif state == desired_state:\n break\n time.sleep(3)\n\n\ndef wait_for_installation(cluster_nodes=1, timeout_insec=360):\n \"\"\"\n Wait for kubernetes service to appear.\n \"\"\"\n while True:\n cmd = \"svc kubernetes\"\n data = kubectl_get(cmd, timeout_insec)\n service = data[\"metadata\"][\"name\"]\n if \"kubernetes\" in service:\n break\n else:\n time.sleep(3)\n\n while True:\n cmd = \"get no\"\n nodes = kubectl(cmd, timeout_insec)\n if nodes.count(\" Ready\") == cluster_nodes:\n break\n else:\n time.sleep(3)\n\n # Allow rest of the services to come up\n time.sleep(30)\n\n\ndef wait_for_namespace_termination(namespace, timeout_insec=360):\n \"\"\"\n Wait for the termination of the provided namespace.\n \"\"\"\n\n print(\"Waiting for namespace {} to be removed\".format(namespace))\n deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec)\n while True:\n try:\n cmd = \"/snap/bin/microk8s.kubectl get ns {}\".format(namespace)\n check_output(cmd.split()).strip().decode(\"utf8\")\n print(\"Waiting...\")\n except CalledProcessError:\n if datetime.datetime.now() > deadline:\n raise\n else:\n return\n time.sleep(10)\n\n\ndef microk8s_enable(addon, timeout_insec=300):\n \"\"\"\n Disable an addon\n\n Args:\n addon: name of the addon\n timeout_insec: seconds to keep retrying\n\n \"\"\"\n # NVidia pre-check so as to not wait for a timeout.\n if addon == \"gpu\":\n nv_out = run_until_success(\"lsmod\", timeout_insec=10)\n if \"nvidia\" not in nv_out:\n print(\"Not a cuda capable system. Will not test gpu addon\")\n raise CalledProcessError(1, \"Nothing to do for gpu\")\n\n cmd = \"/snap/bin/microk8s.enable {}\".format(addon)\n return run_until_success(cmd, timeout_insec)\n\n\ndef microk8s_disable(addon):\n \"\"\"\n Enable an addon\n\n Args:\n addon: name of the addon\n\n \"\"\"\n cmd = \"/snap/bin/microk8s.disable {}\".format(addon)\n return run_until_success(cmd, timeout_insec=300)\n\n\ndef microk8s_clustering_capable():\n \"\"\"\n Are we in a clustering capable microk8s?\n \"\"\"\n return os.path.isfile(\"/snap/bin/microk8s.join\")\n\n\ndef microk8s_reset(cluster_nodes=1):\n \"\"\"\n Call microk8s reset\n \"\"\"\n cmd = \"/snap/bin/microk8s.reset\"\n run_until_success(cmd, timeout_insec=300)\n wait_for_installation(cluster_nodes)\n\n\ndef update_yaml_with_arch(manifest_file):\n \"\"\"\n Updates any $ARCH entry with the architecture in the manifest\n\n \"\"\"\n arch = arch_translate[platform.machine()]\n with open(manifest_file) as f:\n s = f.read()\n\n with open(manifest_file, \"w\") as f:\n s = s.replace(\"$ARCH\", arch)\n f.write(s)\n\n\ndef is_container():\n \"\"\"\n Returns: True if the deployment is in a VM/container.\n\n \"\"\"\n try:\n if os.path.isdir(\"/run/systemd/system\"):\n container = check_output(\"sudo systemd-detect-virt --container\".split())\n print(\"Tests are running in {}\".format(container))\n return True\n except CalledProcessError:\n print(\"systemd-detect-virt did not detect a container\")\n\n if os.path.exists(\"/run/container_type\"):\n return True\n\n try:\n check_call(\"sudo grep -E (lxc|hypervisor) /proc/1/environ /proc/cpuinfo\".split())\n print(\"Tests are running in an undetectable container\")\n return True\n except CalledProcessError:\n print(\"no indication of a container in /proc\")\n\n return False\n\n\ndef is_strict():\n if \"STRICT\" in os.environ and os.environ[\"STRICT\"] == \"yes\":\n return True\n return False\n\n\ndef is_ipv6_configured():\n try:\n output = check_output([\"ip\", \"-6\", \"address\"])\n return b\"inet6\" in output\n except CalledProcessError:\n return False\n\n\ndef _get_process(name):\n return [p for p in psutil.process_iter() if name == p.name()]\n" }, { "path": "tests/validators.py", "content": "import time\nimport os\nimport re\nimport requests\nimport platform\nimport yaml\nimport subprocess\nfrom pathlib import Path\n\nfrom utils import (\n get_arch,\n kubectl,\n wait_for_pod_state,\n docker,\n update_yaml_with_arch,\n)\n\nTEMPLATES = Path(__file__).absolute().parent / \"templates\"\n\n\ndef validate_dns_dashboard():\n \"\"\"\n Validate the dashboard addon by trying to access the kubernetes dashboard.\n The dashboard will return an HTML indicating that it is up and running.\n \"\"\"\n service = \"kubernetes-dashboard:\"\n ns = \"kube-system\"\n app_names = [\"k8s-app=kubernetes-dashboard\", \"k8s-app=dashboard-metrics-scraper\"]\n\n output = kubectl(\"get ns\")\n if \"kubernetes-dashboard\" in output:\n # we are running a newer version of the dashboard introduced in 1.33\n service = \"kubernetes-dashboard-kong-proxy:443\"\n ns = \"kubernetes-dashboard\"\n components = [\"api\", \"auth\", \"metrics-scraper\", \"web\"]\n app_names = [f\"app.kubernetes.io/name=kubernetes-dashboard-{app}\" for app in components]\n app_names.append(\"app.kubernetes.io/name=kong\")\n\n for app_name in app_names:\n wait_for_pod_state(\"\", ns, \"running\", label=f\"{app_name}\")\n\n attempt = 30\n while attempt > 0:\n try:\n output = kubectl(f\"get --raw /api/v1/namespaces/{ns}/services/https:{service}/proxy/\")\n if \"Kubernetes Dashboard\" in output:\n break\n except subprocess.CalledProcessError:\n pass\n time.sleep(10)\n attempt -= 1\n\n assert attempt > 0\n\n\ndef validate_storage():\n \"\"\"\n Validate storage by creating a PVC.\n \"\"\"\n output = kubectl(\"describe deployment hostpath-provisioner -n kube-system\")\n if \"hostpath-provisioner-{}:1.0.0\".format(get_arch()) in output:\n # we are running with a hostpath-provisioner that is old and we need to patch it\n cmd = (\n \"set image deployment hostpath-provisioner\"\n \"-n kube-system\"\n \"hostpath-provisioner=cdkbot/hostpath-provisioner:1.1.0\"\n )\n kubectl(cmd)\n\n wait_for_pod_state(\"\", \"kube-system\", \"running\", label=\"k8s-app=hostpath-provisioner\")\n manifest = TEMPLATES / \"pvc.yaml\"\n kubectl(\"apply -f {}\".format(manifest))\n wait_for_pod_state(\"hostpath-test-pod\", \"default\", \"running\")\n\n attempt = 50\n while attempt >= 0:\n output = kubectl(\"get pvc\")\n if \"Bound\" in output:\n break\n time.sleep(2)\n attempt -= 1\n\n # Make sure the test pod writes data sto the storage\n found = False\n for root, dirs, files in os.walk(\"/var/snap/microk8s/common/default-storage\"):\n for file in files:\n if file == \"dates\":\n found = True\n assert found\n assert \"myclaim\" in output\n assert \"Bound\" in output\n kubectl(\"delete -f {}\".format(manifest))\n\n\ndef common_ingress():\n \"\"\"\n Perform the Ingress validations that are common for all\n the Ingress controllers.\n \"\"\"\n attempt = 50\n while attempt >= 0:\n output = kubectl(\"get ing\")\n if \"microbot.127.0.0.1.nip.io\" in output:\n break\n time.sleep(5)\n attempt -= 1\n assert \"microbot.127.0.0.1.nip.io\" in output\n\n service_ok = False\n attempt = 50\n while attempt >= 0:\n try:\n resp = requests.get(\"http://microbot.127.0.0.1.nip.io/\")\n if resp.status_code == 200 and \"microbot.png\" in resp.content.decode(\"utf-8\"):\n service_ok = True\n break\n except requests.RequestException:\n time.sleep(5)\n attempt -= 1\n\n assert service_ok\n\n\ndef validate_ingress():\n \"\"\"\n Validate ingress by creating a ingress rule.\n \"\"\"\n ds = kubectl(\"get ds -n ingress\")\n if \"nginx-ingress-microk8s-controller\" in ds:\n wait_for_pod_state(\"\", \"ingress\", \"running\", label=\"name=nginx-ingress-microk8s\")\n else:\n # Support migration to Traefik ingress controller\n wait_for_pod_state(\"\", \"ingress\", \"running\", label=\"app.kubernetes.io/name=traefik\")\n\n manifest = TEMPLATES / \"ingress.yaml\"\n update_yaml_with_arch(manifest)\n kubectl(\"apply -f {}\".format(manifest))\n wait_for_pod_state(\"\", \"default\", \"running\", label=\"app=microbot\")\n\n common_ingress()\n\n kubectl(\"delete -f {}\".format(manifest))\n\n\ndef validate_registry():\n \"\"\"\n Validate the private registry.\n \"\"\"\n\n wait_for_pod_state(\"\", \"container-registry\", \"running\", label=\"app=registry\")\n pvc_stdout = kubectl(\"get pvc registry-claim -n container-registry -o yaml\")\n pvc_yaml = yaml.safe_load(pvc_stdout)\n storage = pvc_yaml[\"spec\"][\"resources\"][\"requests\"][\"storage\"]\n assert re.match(\"(^[2-9][0-9]{1,}|^[1-9][0-9]{2,})(Gi$)\", storage)\n docker(\"pull busybox\")\n docker(\"tag busybox localhost:32000/my-busybox\")\n docker(\"push localhost:32000/my-busybox\")\n\n manifest = TEMPLATES / \"bbox-local.yaml\"\n kubectl(\"apply -f {}\".format(manifest))\n wait_for_pod_state(\"busybox\", \"default\", \"running\")\n output = kubectl(\"describe po busybox\")\n assert \"localhost:32000/my-busybox\" in output\n kubectl(\"delete -f {}\".format(manifest))\n\n\ndef validate_forward():\n \"\"\"\n Validate ports are forwarded\n \"\"\"\n manifest = TEMPLATES / \"nginx-pod.yaml\"\n kubectl(\"apply -f {}\".format(manifest))\n wait_for_pod_state(\"\", \"default\", \"running\", label=\"app=nginx\")\n os.system(\"killall kubectl\")\n os.system(\"/snap/bin/microk8s.kubectl port-forward pod/nginx 5123:80 &\")\n attempt = 10\n while attempt >= 0:\n try:\n resp = requests.get(\"http://localhost:5123\")\n if resp.status_code == 200:\n break\n except requests.RequestException:\n pass\n attempt -= 1\n time.sleep(2)\n\n assert resp.status_code == 200\n os.system(\"killall kubectl\")\n\n\ndef validate_metrics_server():\n \"\"\"\n Validate the metrics server works\n \"\"\"\n wait_for_pod_state(\"\", \"kube-system\", \"running\", label=\"k8s-app=metrics-server\")\n attempt = 30\n while attempt > 0:\n try:\n output = kubectl(\"get --raw /apis/metrics.k8s.io/v1beta1/pods\")\n if \"PodMetricsList\" in output:\n break\n except subprocess.CalledProcessError:\n pass\n time.sleep(10)\n attempt -= 1\n\n assert attempt > 0\n\n\ndef validate_metallb_config(ip_ranges=\"192.168.0.105\"):\n \"\"\"\n Validate Metallb\n \"\"\"\n if platform.machine() != \"x86_64\":\n print(\"Metallb tests are only relevant in x86 architectures\")\n return\n out = kubectl(\n \"get ipaddresspool -n metallb-system default-addresspool -o jsonpath='{.spec.addresses}\"\n )\n for ip_range in ip_ranges.split(\",\"):\n assert ip_range in out\n\n\ndef validate_dual_stack():\n # Deploy the test deployment and service\n manifest = TEMPLATES / \"dual-stack.yaml\"\n kubectl(\"apply -f {}\".format(manifest))\n\n wait_for_pod_state(\"\", \"default\", \"running\", label=\"run=nginxdualstack\")\n\n ipv6_endpoint = kubectl(\n \"get endpoints nginx6 \"\n \"-o jsonpath={.subsets[0].addresses[0].ip} \"\n \"--output=jsonpath=[{.subsets[0].addresses[0].ip}]\"\n )\n\n print(\"Pinging endpoint: http://{}/\".format(ipv6_endpoint))\n url = f\"http://{ipv6_endpoint}/\"\n attempt = 10\n service_ok = False\n while attempt >= 0:\n try:\n resp = requests.get(url)\n if \"Kubernetes IPv6 nginx\" in str(resp.content):\n print(resp.content)\n service_ok = True\n break\n except requests.RequestException:\n time.sleep(5)\n attempt -= 1\n\n assert service_ok\n kubectl(\"delete -f {}\".format(manifest))\n" }, { "path": "tests/verify-branches.py", "content": "import requests\nfrom subprocess import check_output\n\n\nclass TestMicrok8sBranches(object):\n def test_branches(self):\n \"\"\"Ensures LP builders push to correct snap tracks.\n\n We need to make sure the LP builders pointing to the master github branch are only pushing\n to the latest and current k8s stable snap tracks. An indication that this is not enforced is\n that we do not have a branch for the k8s release for the previous stable release. Let me\n clarify with an example.\n\n Assuming upstream stable k8s release is v1.12.x, there has to be a 1.11 github branch used\n by the respective LP builders for building the v1.11.y.\n \"\"\"\n upstream_version = self._upstream_release()\n assert upstream_version\n version_parts = upstream_version.split(\".\")\n major_minor_upstream_version = \"{}.{}\".format(version_parts[0][1:], version_parts[1])\n if version_parts[1] != \"0\":\n prev_major_minor_version = \"{}.{}\".format(\n version_parts[0][1:], int(version_parts[1]) - 1\n )\n else:\n major = int(version_parts[0][1:]) - 1\n minor = self._get_max_minor(major)\n prev_major_minor_version = \"{}.{}\".format(major, minor)\n print(\n \"Current stable is {}. Making sure we have a branch for {}\".format(\n major_minor_upstream_version, prev_major_minor_version\n )\n )\n cmd = \"git ls-remote --heads http://github.com/canonical/microk8s.git {}\".format(\n prev_major_minor_version\n )\n branch = check_output(cmd.split()).decode(\"utf-8\")\n assert prev_major_minor_version in branch\n\n def _upstream_release(self):\n \"\"\"Return the latest stable k8s in the release series\"\"\"\n release_url = \"https://dl.k8s.io/release/stable.txt\"\n r = requests.get(release_url)\n if r.status_code == 200:\n return r.content.decode().strip()\n else:\n None\n\n def _get_max_minor(self, major):\n \"\"\"Get the latest minor release of the provided major.\n For example if you use 1 as major you will get back X where X gives you latest 1.X release.\n \"\"\"\n minor = 0\n while self._upstream_release_exists(major, minor):\n minor += 1\n return minor - 1\n\n def _upstream_release_exists(self, major, minor):\n \"\"\"Return true if the major.minor release exists\"\"\"\n release_url = \"https://dl.k8s.io/release/stable-{}.{}.txt\".format(major, minor)\n r = requests.get(release_url)\n if r.status_code == 200:\n return True\n else:\n return False\n" }, { "path": "tox.ini", "content": "[tox]\nskipsdist=True\nskip_missing_interpreters = True\nenvlist = lint, scripts, wrappers\n\n[testenv]\nbasepython = python3\nenvdir = {toxinidir}/.tox_env\npassenv =\n MK8S_*\ndeps =\n black ==21.4b2\n click==7.1.2\n flake8\n flake8-colors\n pep8-naming\n codespell\n -r{toxinidir}/tests/requirements.txt\n\n[testenv:lint]\ncommands =\n flake8 --max-line-length=120 --ignore=C901,N801,N802,N803,N806,N816,W503,E203\n codespell --ignore-words-list=\"aks,ccompiler,NotIn\" --quiet-level=2 --skip=\"*.patch,*.spec,.tox_env,.git,*.nsi\"\n black --diff --check --exclude \"/(\\.eggs|\\.git|\\.tox|\\.venv|\\.build|dist|charmhelpers|mod)/\" .\n\n[testenv:scripts]\nsetenv = PYTHONPATH={toxinidir}/scripts\ncommands =\n pytest -s tests/unit/test_upgrade_calico_cni.py {posargs}\n\n[testenv:wrappers]\nsetenv = PYTHONPATH={toxinidir}/scripts/wrappers\ncommands =\n pytest -s tests/unit/ \\ \n --ignore=tests/unit/test_upgrade_calico_cni.py \\\n --ignore-glob=tests/unit/cluster/* {posargs}\n\n[testenv:cluster]\nsetenv = PYTHONPATH={toxinidir}/scripts/wrappers\ncommands =\n pytest -s tests/unit/cluster {posargs}\n\n[testenv:clustering]\ncommands = pytest -s tests/test-cluster.py\n\n[testenv:simple]\ncommands = pytest -rA -s tests/test-simple.py\n\n[flake8]\nexclude =\n .git,\n __pycache__,\n .tox,\n .tox_env,\nmax-complexity = 10\nimport-order-style = google\n" }, { "path": "upgrade-scripts/000-switch-to-calico/commit-master.sh", "content": "#!/bin/bash\nset -ex\n\necho \"Switching master to calico\"\n\nsource $SNAP/actions/common/utils.sh\n\nRESOURCES=\"$SNAP/upgrade-scripts/000-switch-to-calico/resources\"\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico\"\n\nmkdir -p \"$BACKUP_DIR\"\n\nmkdir -p \"$BACKUP_DIR/args/cni-network/\"\ncp \"$SNAP_DATA\"/args/cni-network/* \"$BACKUP_DIR/args/cni-network/\" 2>/dev/null || true\nfind \"$SNAP_DATA\"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \\;\nARCH=\"$($SNAP/bin/uname -m)\"\ncp \"$RESOURCES/calico.yaml\" \"$SNAP_DATA/args/cni-network/cni.yaml\"\n\necho \"Restarting services\"\ncp \"$SNAP_DATA\"/args/kube-apiserver \"$BACKUP_DIR/args\"\nrefresh_opt_in_config \"allow-privileged\" \"true\" kube-apiserver\n\n# Reconfigure kubelet/containerd to pick up the new CNI config and binary.\ncp \"$SNAP_DATA\"/args/kubelet \"$BACKUP_DIR/args\"\n\ncp \"$SNAP_DATA\"/args/containerd-template.toml \"$BACKUP_DIR/args\"\nif grep -qE \"bin_dir.*SNAP}\\/\" $SNAP_DATA/args/containerd-template.toml; then\n echo \"Restarting containerd\"\n \"${SNAP}/bin/sed\" -i 's;bin_dir = \"${SNAP}/opt;bin_dir = \"${SNAP_DATA}/opt;g' \"$SNAP_DATA/args/containerd-template.toml\"\n snapctl restart ${SNAP_NAME}.daemon-containerd\nfi\n\ncp \"$SNAP_DATA\"/args/kube-proxy \"$BACKUP_DIR/args\"\nrefresh_opt_in_config \"cluster-cidr\" \"10.1.0.0/16\" kube-proxy\n\necho \"Restarting kubelite\"\nsnapctl restart ${SNAP_NAME}.daemon-kubelite\n\nset_service_not_expected_to_start flanneld\nsnapctl stop ${SNAP_NAME}.daemon-flanneld\nremove_vxlan_interfaces\n\n# Allow for services to restart\nsleep 15\n${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30\n\nKUBECTL=\"$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config\"\n$KUBECTL apply -f \"$SNAP_DATA/args/cni-network/cni.yaml\"\n\necho \"Calico is enabled\"\n" }, { "path": "upgrade-scripts/000-switch-to-calico/commit-node.sh", "content": "#!/bin/bash\n\nset -ex\n\necho \"Switching master to calico\"\n\nsource $SNAP/actions/common/utils.sh\n\nRESOURCES=\"$SNAP/upgrade-scripts/000-switch-to-calico/resources\"\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico\"\n\nmkdir -p \"$BACKUP_DIR\"\n\nmkdir -p \"$BACKUP_DIR/args/cni-network/\"\ncp \"$SNAP_DATA\"/args/cni-network/* \"$BACKUP_DIR/args/cni-network/\" 2>/dev/null || true\nfind \"$SNAP_DATA\"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \\;\nARCH=\"$($SNAP/bin/uname -m)\"\nCALICO_MANIFEST=\"$RESOURCES/calico.yaml\"\nrun_with_sudo cp \"$CALICO_MANIFEST\" \"$SNAP_DATA/args/cni-network/cni.yaml\"\n\ncp \"$SNAP_DATA\"/args/kube-apiserver \"$BACKUP_DIR/args\"\nrefresh_opt_in_config \"allow-privileged\" \"true\" kube-apiserver\n\n# Reconfigure kubelet/containerd to pick up the new CNI config and binary.\ncp \"$SNAP_DATA\"/args/kubelet \"$BACKUP_DIR/args\"\n\ncp \"$SNAP_DATA\"/args/containerd-template.toml \"$BACKUP_DIR/args\"\nif grep -qE \"bin_dir.*SNAP}\\/\" $SNAP_DATA/args/containerd-template.toml; then\n echo \"Restarting containerd\"\n \"${SNAP}/bin/sed\" -i 's;bin_dir = \"${SNAP}/opt;bin_dir = \"${SNAP_DATA}/opt;g' \"$SNAP_DATA/args/containerd-template.toml\"\n snapctl restart ${SNAP_NAME}.daemon-containerd\nfi\n\ncp \"$SNAP_DATA\"/args/kube-proxy \"$BACKUP_DIR/args\"\nrefresh_opt_in_config \"cluster-cidr\" \"10.1.0.0/16\" kube-proxy\n\necho \"Restarting kubelite\"\nsnapctl restart ${SNAP_NAME}.daemon-kubelite\n\nset_service_not_expected_to_start flanneld\nsnapctl stop ${SNAP_NAME}.daemon-flanneld\nremove_vxlan_interfaces\n\necho \"Calico is enabled\"\n" }, { "path": "upgrade-scripts/000-switch-to-calico/description.txt", "content": "Removes the flannel cni and installs calico." }, { "path": "upgrade-scripts/000-switch-to-calico/prepare-master.sh", "content": "#!/usr/bin/env bash\n\necho \"Preparing master for calico\"\n" }, { "path": "upgrade-scripts/000-switch-to-calico/prepare-node.sh", "content": "#!/bin/bash\n\nset -e\n\necho \"Praparing node for calico\"\n" }, { "path": "upgrade-scripts/000-switch-to-calico/resources/calico.yaml", "content": "---\n# Source: calico/templates/calico-kube-controllers.yaml\n# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict\n\napiVersion: policy/v1\nkind: PodDisruptionBudget\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\nspec:\n maxUnavailable: 1\n selector:\n matchLabels:\n k8s-app: calico-kube-controllers\n---\n# Source: calico/templates/calico-kube-controllers.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n---\n# Source: calico/templates/calico-node.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-node\n namespace: kube-system\n---\n# Source: calico/templates/calico-node.yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: calico-cni-plugin\n namespace: kube-system\n---\n# Source: calico/templates/calico-config.yaml\n# This ConfigMap is used to configure a self-hosted Calico installation.\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-config\n namespace: kube-system\ndata:\n # Typha is disabled.\n typha_service_name: \"none\"\n # Configure the backend to use.\n # calico_backend: \"bird\"\n calico_backend: \"vxlan\"\n\n # Configure the MTU to use for workload interfaces and tunnels.\n # By default, MTU is auto-detected, and explicitly setting this field should not be required.\n # You can override auto-detection by providing a non-zero value.\n veth_mtu: \"0\"\n\n # The CNI network configuration to install on each node. The special\n # values in this config will be automatically populated.\n cni_network_config: |-\n {\n \"name\": \"k8s-pod-network\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": [\n {\n \"type\": \"calico\",\n \"log_level\": \"info\",\n \"log_file_path\": \"/var/snap/microk8s/common/var/log/calico/cni/cni.log\",\n \"datastore_type\": \"kubernetes\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n \"nodename_file\": \"/var/snap/microk8s/current/var/lib/calico/nodename\",\n \"mtu\": __CNI_MTU__,\n \"ipam\": {\n \"type\": \"calico-ipam\"\n },\n \"policy\": {\n \"type\": \"k8s\"\n },\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n }\n },\n {\n \"type\": \"portmap\",\n \"snat\": true,\n \"capabilities\": {\"portMappings\": true}\n },\n {\n \"type\": \"bandwidth\",\n \"capabilities\": {\"bandwidth\": true}\n }\n ]\n }\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: bgpconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPConfiguration\n listKind: BGPConfigurationList\n plural: bgpconfigurations\n singular: bgpconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: BGPConfiguration contains the configuration for any BGP routing.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPConfigurationSpec contains the values of the BGP configuration.\n properties:\n asNumber:\n description: 'ASNumber is the default AS number used by a node. [Default:\n 64512]'\n format: int32\n type: integer\n bindMode:\n description: BindMode indicates whether to listen for BGP connections\n on all addresses (None) or only on the node's canonical IP address\n Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen\n for BGP connections on all addresses.\n type: string\n communities:\n description: Communities is a list of BGP community values and their\n arbitrary names for tagging routes.\n items:\n description: Community contains standard or large community value\n and its name.\n properties:\n name:\n description: Name given to community value.\n type: string\n value:\n description: Value must be of format `aa:nn` or `aa:nn:mm`.\n For standard community use `aa:nn` format, where `aa` and\n `nn` are 16 bit number. For large community use `aa:nn:mm`\n format, where `aa`, `nn` and `mm` are 32 bit number. Where,\n `aa` is an AS Number, `nn` and `mm` are per-AS identifier.\n pattern: ^(\\d+):(\\d+)$|^(\\d+):(\\d+):(\\d+)$\n type: string\n type: object\n type: array\n ignoredInterfaces:\n description: IgnoredInterfaces indicates the network interfaces that\n needs to be excluded when reading device routes.\n items:\n type: string\n type: array\n listenPort:\n description: ListenPort is the port where BGP protocol should listen.\n Defaults to 179\n maximum: 65535\n minimum: 1\n type: integer\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: INFO]'\n type: string\n nodeMeshMaxRestartTime:\n description: Time to allow for software restart for node-to-mesh peerings. When\n specified, this is configured as the graceful restart timeout. When\n not specified, the BIRD default of 120s is used. This field can\n only be set on the default BGPConfiguration instance and requires\n that NodeMesh is enabled\n type: string\n nodeMeshPassword:\n description: Optional BGP password for full node-to-mesh peerings.\n This field can only be set on the default BGPConfiguration instance\n and requires that NodeMesh is enabled\n properties:\n secretKeyRef:\n description: Selects a key of a secret in the node pod's namespace.\n properties:\n key:\n description: The key of the secret to select from. Must be\n a valid secret key.\n type: string\n name:\n default: \"\"\n description: 'Name of the referent. This field is effectively\n required, but due to backwards compatibility is allowed\n to be empty. Instances of this type with an empty value\n here are almost certainly wrong. TODO: Add other useful\n fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Drop `kubebuilder:default` when controller-gen doesn''t\n need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.'\n type: string\n optional:\n description: Specify whether the Secret or its key must be\n defined\n type: boolean\n required:\n - key\n type: object\n type: object\n nodeToNodeMeshEnabled:\n description: 'NodeToNodeMeshEnabled sets whether full node to node\n BGP mesh is enabled. [Default: true]'\n type: boolean\n prefixAdvertisements:\n description: PrefixAdvertisements contains per-prefix advertisement\n configuration.\n items:\n description: PrefixAdvertisement configures advertisement properties\n for the specified CIDR.\n properties:\n cidr:\n description: CIDR for which properties should be advertised.\n type: string\n communities:\n description: Communities can be list of either community names\n already defined in `Specs.Communities` or community value\n of format `aa:nn` or `aa:nn:mm`. For standard community use\n `aa:nn` format, where `aa` and `nn` are 16 bit number. For\n large community use `aa:nn:mm` format, where `aa`, `nn` and\n `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and\n `mm` are per-AS identifier.\n items:\n type: string\n type: array\n type: object\n type: array\n serviceClusterIPs:\n description: ServiceClusterIPs are the CIDR blocks from which service\n cluster IPs are allocated. If specified, Calico will advertise these\n blocks, as well as any cluster IPs within them.\n items:\n description: ServiceClusterIPBlock represents a single allowed ClusterIP\n CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n serviceExternalIPs:\n description: ServiceExternalIPs are the CIDR blocks for Kubernetes\n Service External IPs. Kubernetes Service ExternalIPs will only be\n advertised if they are within one of these blocks.\n items:\n description: ServiceExternalIPBlock represents a single allowed\n External IP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n serviceLoadBalancerIPs:\n description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes\n Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress\n IPs will only be advertised if they are within one of these blocks.\n items:\n description: ServiceLoadBalancerIPBlock represents a single allowed\n LoadBalancer IP CIDR block.\n properties:\n cidr:\n type: string\n type: object\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: bgpfilters.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPFilter\n listKind: BGPFilterList\n plural: bgpfilters\n singular: bgpfilter\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPFilterSpec contains the IPv4 and IPv6 filter rules of\n the BGP Filter.\n properties:\n exportV4:\n description: The ordered set of IPv4 BGPFilter rules acting on exporting\n routes to a peer.\n items:\n description: BGPFilterRuleV4 defines a BGP filter rule consisting\n a single IPv4 CIDR block and a filter action for this CIDR.\n properties:\n action:\n type: string\n cidr:\n type: string\n interface:\n type: string\n matchOperator:\n type: string\n prefixLength:\n properties:\n max:\n format: int32\n maximum: 32\n minimum: 0\n type: integer\n min:\n format: int32\n maximum: 32\n minimum: 0\n type: integer\n type: object\n source:\n type: string\n required:\n - action\n type: object\n type: array\n exportV6:\n description: The ordered set of IPv6 BGPFilter rules acting on exporting\n routes to a peer.\n items:\n description: BGPFilterRuleV6 defines a BGP filter rule consisting\n a single IPv6 CIDR block and a filter action for this CIDR.\n properties:\n action:\n type: string\n cidr:\n type: string\n interface:\n type: string\n matchOperator:\n type: string\n prefixLength:\n properties:\n max:\n format: int32\n maximum: 128\n minimum: 0\n type: integer\n min:\n format: int32\n maximum: 128\n minimum: 0\n type: integer\n type: object\n source:\n type: string\n required:\n - action\n type: object\n type: array\n importV4:\n description: The ordered set of IPv4 BGPFilter rules acting on importing\n routes from a peer.\n items:\n description: BGPFilterRuleV4 defines a BGP filter rule consisting\n a single IPv4 CIDR block and a filter action for this CIDR.\n properties:\n action:\n type: string\n cidr:\n type: string\n interface:\n type: string\n matchOperator:\n type: string\n prefixLength:\n properties:\n max:\n format: int32\n maximum: 32\n minimum: 0\n type: integer\n min:\n format: int32\n maximum: 32\n minimum: 0\n type: integer\n type: object\n source:\n type: string\n required:\n - action\n type: object\n type: array\n importV6:\n description: The ordered set of IPv6 BGPFilter rules acting on importing\n routes from a peer.\n items:\n description: BGPFilterRuleV6 defines a BGP filter rule consisting\n a single IPv6 CIDR block and a filter action for this CIDR.\n properties:\n action:\n type: string\n cidr:\n type: string\n interface:\n type: string\n matchOperator:\n type: string\n prefixLength:\n properties:\n max:\n format: int32\n maximum: 128\n minimum: 0\n type: integer\n min:\n format: int32\n maximum: 128\n minimum: 0\n type: integer\n type: object\n source:\n type: string\n required:\n - action\n type: object\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: bgppeers.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BGPPeer\n listKind: BGPPeerList\n plural: bgppeers\n singular: bgppeer\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BGPPeerSpec contains the specification for a BGPPeer resource.\n properties:\n asNumber:\n description: The AS Number of the peer.\n format: int32\n type: integer\n filters:\n description: The ordered set of BGPFilters applied on this BGP peer.\n items:\n type: string\n type: array\n keepOriginalNextHop:\n description: Option to keep the original nexthop field when routes\n are sent to a BGP Peer. Setting \"true\" configures the selected BGP\n Peers node to use the \"next hop keep;\" instead of \"next hop self;\"(default)\n in the specific branch of the Node on \"bird.cfg\".\n type: boolean\n maxRestartTime:\n description: Time to allow for software restart. When specified,\n this is configured as the graceful restart timeout. When not specified,\n the BIRD default of 120s is used.\n type: string\n node:\n description: The node name identifying the Calico node instance that\n is targeted by this peer. If this is not set, and no nodeSelector\n is specified, then this BGP peer selects all nodes in the cluster.\n type: string\n nodeSelector:\n description: Selector for the nodes that should have this peering. When\n this is set, the Node field must be empty.\n type: string\n numAllowedLocalASNumbers:\n description: Maximum number of local AS numbers that are allowed in\n the AS path for received routes. This removes BGP loop prevention\n and should only be used if absolutely necessary.\n format: int32\n type: integer\n password:\n description: Optional BGP password for the peerings generated by this\n BGPPeer resource.\n properties:\n secretKeyRef:\n description: Selects a key of a secret in the node pod's namespace.\n properties:\n key:\n description: The key of the secret to select from. Must be\n a valid secret key.\n type: string\n name:\n default: \"\"\n description: 'Name of the referent. This field is effectively\n required, but due to backwards compatibility is allowed\n to be empty. Instances of this type with an empty value\n here are almost certainly wrong. TODO: Add other useful\n fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n TODO: Drop `kubebuilder:default` when controller-gen doesn''t\n need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.'\n type: string\n optional:\n description: Specify whether the Secret or its key must be\n defined\n type: boolean\n required:\n - key\n type: object\n type: object\n peerIP:\n description: The IP address of the peer followed by an optional port\n number to peer with. If port number is given, format should be `[]:port`\n or `:` for IPv4. If optional port number is not set,\n and this peer IP and ASNumber belongs to a calico/node with ListenPort\n set in BGPConfiguration, then we use that port to peer.\n type: string\n peerSelector:\n description: Selector for the remote nodes to peer with. When this\n is set, the PeerIP and ASNumber fields must be empty. For each\n peering between the local node and selected remote nodes, we configure\n an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified,\n and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The\n remote AS number comes from the remote node's NodeBGPSpec.ASNumber,\n or the global default if that is not set.\n type: string\n reachableBy:\n description: Add an exact, i.e. /32, static route toward peer IP in\n order to prevent route flapping. ReachableBy contains the address\n of the gateway which peer can be reached by.\n type: string\n sourceAddress:\n description: Specifies whether and how to configure a source address\n for the peerings generated by this BGPPeer resource. Default value\n \"UseNodeIP\" means to configure the node IP as the source address. \"None\"\n means not to configure a source address.\n type: string\n ttlSecurity:\n description: TTLSecurity enables the generalized TTL security mechanism\n (GTSM) which protects against spoofed packets by ignoring received\n packets with a smaller than expected TTL value. The provided value\n is the number of hops (edges) between the peers.\n type: integer\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: blockaffinities.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: BlockAffinity\n listKind: BlockAffinityList\n plural: blockaffinities\n singular: blockaffinity\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: BlockAffinitySpec contains the specification for a BlockAffinity\n resource.\n properties:\n cidr:\n type: string\n deleted:\n description: Deleted indicates that this block affinity is being deleted.\n This field is a string for compatibility with older releases that\n mistakenly treat this field as a string.\n type: string\n node:\n type: string\n state:\n type: string\n required:\n - cidr\n - deleted\n - node\n - state\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: caliconodestatuses.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: CalicoNodeStatus\n listKind: CalicoNodeStatusList\n plural: caliconodestatuses\n singular: caliconodestatus\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus\n resource.\n properties:\n classes:\n description: Classes declares the types of information to monitor\n for this calico/node, and allows for selective status reporting\n about certain subsets of information.\n items:\n type: string\n type: array\n node:\n description: The node name identifies the Calico node instance for\n node status.\n type: string\n updatePeriodSeconds:\n description: UpdatePeriodSeconds is the period at which CalicoNodeStatus\n should be updated. Set to 0 to disable CalicoNodeStatus refresh.\n Maximum update period is one day.\n format: int32\n type: integer\n type: object\n status:\n description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus.\n No validation needed for status since it is updated by Calico.\n properties:\n agent:\n description: Agent holds agent status on the node.\n properties:\n birdV4:\n description: BIRDV4 represents the latest observed status of bird4.\n properties:\n lastBootTime:\n description: LastBootTime holds the value of lastBootTime\n from bird.ctl output.\n type: string\n lastReconfigurationTime:\n description: LastReconfigurationTime holds the value of lastReconfigTime\n from bird.ctl output.\n type: string\n routerID:\n description: Router ID used by bird.\n type: string\n state:\n description: The state of the BGP Daemon.\n type: string\n version:\n description: Version of the BGP daemon\n type: string\n type: object\n birdV6:\n description: BIRDV6 represents the latest observed status of bird6.\n properties:\n lastBootTime:\n description: LastBootTime holds the value of lastBootTime\n from bird.ctl output.\n type: string\n lastReconfigurationTime:\n description: LastReconfigurationTime holds the value of lastReconfigTime\n from bird.ctl output.\n type: string\n routerID:\n description: Router ID used by bird.\n type: string\n state:\n description: The state of the BGP Daemon.\n type: string\n version:\n description: Version of the BGP daemon\n type: string\n type: object\n type: object\n bgp:\n description: BGP holds node BGP status.\n properties:\n numberEstablishedV4:\n description: The total number of IPv4 established bgp sessions.\n type: integer\n numberEstablishedV6:\n description: The total number of IPv6 established bgp sessions.\n type: integer\n numberNotEstablishedV4:\n description: The total number of IPv4 non-established bgp sessions.\n type: integer\n numberNotEstablishedV6:\n description: The total number of IPv6 non-established bgp sessions.\n type: integer\n peersV4:\n description: PeersV4 represents IPv4 BGP peers status on the node.\n items:\n description: CalicoNodePeer contains the status of BGP peers\n on the node.\n properties:\n peerIP:\n description: IP address of the peer whose condition we are\n reporting.\n type: string\n since:\n description: Since the state or reason last changed.\n type: string\n state:\n description: State is the BGP session state.\n type: string\n type:\n description: Type indicates whether this peer is configured\n via the node-to-node mesh, or via en explicit global or\n per-node BGPPeer object.\n type: string\n type: object\n type: array\n peersV6:\n description: PeersV6 represents IPv6 BGP peers status on the node.\n items:\n description: CalicoNodePeer contains the status of BGP peers\n on the node.\n properties:\n peerIP:\n description: IP address of the peer whose condition we are\n reporting.\n type: string\n since:\n description: Since the state or reason last changed.\n type: string\n state:\n description: State is the BGP session state.\n type: string\n type:\n description: Type indicates whether this peer is configured\n via the node-to-node mesh, or via en explicit global or\n per-node BGPPeer object.\n type: string\n type: object\n type: array\n required:\n - numberEstablishedV4\n - numberEstablishedV6\n - numberNotEstablishedV4\n - numberNotEstablishedV6\n type: object\n lastUpdated:\n description: LastUpdated is a timestamp representing the server time\n when CalicoNodeStatus object last updated. It is represented in\n RFC3339 form and is in UTC.\n format: date-time\n nullable: true\n type: string\n routes:\n description: Routes reports routes known to the Calico BGP daemon\n on the node.\n properties:\n routesV4:\n description: RoutesV4 represents IPv4 routes on the node.\n items:\n description: CalicoNodeRoute contains the status of BGP routes\n on the node.\n properties:\n destination:\n description: Destination of the route.\n type: string\n gateway:\n description: Gateway for the destination.\n type: string\n interface:\n description: Interface for the destination\n type: string\n learnedFrom:\n description: LearnedFrom contains information regarding\n where this route originated.\n properties:\n peerIP:\n description: If sourceType is NodeMesh or BGPPeer, IP\n address of the router that sent us this route.\n type: string\n sourceType:\n description: Type of the source where a route is learned\n from.\n type: string\n type: object\n type:\n description: Type indicates if the route is being used for\n forwarding or not.\n type: string\n type: object\n type: array\n routesV6:\n description: RoutesV6 represents IPv6 routes on the node.\n items:\n description: CalicoNodeRoute contains the status of BGP routes\n on the node.\n properties:\n destination:\n description: Destination of the route.\n type: string\n gateway:\n description: Gateway for the destination.\n type: string\n interface:\n description: Interface for the destination\n type: string\n learnedFrom:\n description: LearnedFrom contains information regarding\n where this route originated.\n properties:\n peerIP:\n description: If sourceType is NodeMesh or BGPPeer, IP\n address of the router that sent us this route.\n type: string\n sourceType:\n description: Type of the source where a route is learned\n from.\n type: string\n type: object\n type:\n description: Type indicates if the route is being used for\n forwarding or not.\n type: string\n type: object\n type: array\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: clusterinformations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: ClusterInformation\n listKind: ClusterInformationList\n plural: clusterinformations\n singular: clusterinformation\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: ClusterInformation contains the cluster specific information.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: ClusterInformationSpec contains the values of describing\n the cluster.\n properties:\n calicoVersion:\n description: CalicoVersion is the version of Calico that the cluster\n is running\n type: string\n clusterGUID:\n description: ClusterGUID is the GUID of the cluster\n type: string\n clusterType:\n description: ClusterType describes the type of the cluster\n type: string\n datastoreReady:\n description: DatastoreReady is used during significant datastore migrations\n to signal to components such as Felix that it should wait before\n accessing the datastore.\n type: boolean\n variant:\n description: Variant declares which variant of Calico should be active.\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: felixconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: FelixConfiguration\n listKind: FelixConfigurationList\n plural: felixconfigurations\n singular: felixconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: Felix Configuration contains the configuration for Felix.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: FelixConfigurationSpec contains the values of the Felix configuration.\n properties:\n allowIPIPPacketsFromWorkloads:\n description: 'AllowIPIPPacketsFromWorkloads controls whether Felix\n will add a rule to drop IPIP encapsulated traffic from workloads\n [Default: false]'\n type: boolean\n allowVXLANPacketsFromWorkloads:\n description: 'AllowVXLANPacketsFromWorkloads controls whether Felix\n will add a rule to drop VXLAN encapsulated traffic from workloads\n [Default: false]'\n type: boolean\n awsSrcDstCheck:\n description: 'Set source-destination-check on AWS EC2 instances. Accepted\n value must be one of \"DoNothing\", \"Enable\" or \"Disable\". [Default:\n DoNothing]'\n enum:\n - DoNothing\n - Enable\n - Disable\n type: string\n bpfCTLBLogFilter:\n description: 'BPFCTLBLogFilter specifies, what is logged by connect\n time load balancer when BPFLogLevel is debug. Currently has to be\n specified as ''all'' when BPFLogFilters is set to see CTLB logs.\n [Default: unset - means logs are emitted when BPFLogLevel id debug\n and BPFLogFilters not set.]'\n type: string\n bpfConnectTimeLoadBalancing:\n description: 'BPFConnectTimeLoadBalancing when in BPF mode, controls\n whether Felix installs the connect-time load balancer. The connect-time\n load balancer is required for the host to be able to reach Kubernetes\n services and it improves the performance of pod-to-service connections.When\n set to TCP, connect time load balancing is available only for services\n with TCP ports. [Default: TCP]'\n enum:\n - TCP\n - Enabled\n - Disabled\n type: string\n bpfConnectTimeLoadBalancingEnabled:\n description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode,\n controls whether Felix installs the connection-time load balancer. The\n connect-time load balancer is required for the host to be able to\n reach Kubernetes services and it improves the performance of pod-to-service\n connections. The only reason to disable it is for debugging purposes.\n This will be deprecated. Use BPFConnectTimeLoadBalancing [Default:\n true]'\n type: boolean\n bpfDSROptoutCIDRs:\n description: BPFDSROptoutCIDRs is a list of CIDRs which are excluded\n from DSR. That is, clients in those CIDRs will accesses nodeports\n as if BPFExternalServiceMode was set to Tunnel.\n items:\n type: string\n type: array\n bpfDataIfacePattern:\n description: BPFDataIfacePattern is a regular expression that controls\n which interfaces Felix should attach BPF programs to in order to\n catch traffic to/from the network. This needs to match the interfaces\n that Calico workload traffic flows over as well as any interfaces\n that handle incoming traffic to nodeports and services from outside\n the cluster. It should not match the workload interfaces (usually\n named cali...).\n type: string\n bpfDisableGROForIfaces:\n description: BPFDisableGROForIfaces is a regular expression that controls\n which interfaces Felix should disable the Generic Receive Offload\n [GRO] option. It should not match the workload interfaces (usually\n named cali...).\n type: string\n bpfDisableUnprivileged:\n description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled\n sysctl to disable unprivileged use of BPF. This ensures that unprivileged\n users cannot access Calico''s BPF maps and cannot insert their own\n BPF programs to interfere with Calico''s. [Default: true]'\n type: boolean\n bpfEnabled:\n description: 'BPFEnabled, if enabled Felix will use the BPF dataplane.\n [Default: false]'\n type: boolean\n bpfEnforceRPF:\n description: 'BPFEnforceRPF enforce strict RPF on all host interfaces\n with BPF programs regardless of what is the per-interfaces or global\n setting. Possible values are Disabled, Strict or Loose. [Default:\n Loose]'\n pattern: ^(?i)(Disabled|Strict|Loose)?$\n type: string\n bpfExcludeCIDRsFromNAT:\n description: BPFExcludeCIDRsFromNAT is a list of CIDRs that are to\n be excluded from NAT resolution so that host can handle them. A\n typical usecase is node local DNS cache.\n items:\n type: string\n type: array\n bpfExtToServiceConnmark:\n description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit\n mark that is set on connections from an external client to a local\n service. This mark allows us to control how packets of that connection\n are routed within the host and how is routing interpreted by RPF\n check. [Default: 0]'\n type: integer\n bpfExternalServiceMode:\n description: 'BPFExternalServiceMode in BPF mode, controls how connections\n from outside the cluster to services (node ports and cluster IPs)\n are forwarded to remote workloads. If set to \"Tunnel\" then both\n request and response traffic is tunneled to the remote node. If\n set to \"DSR\", the request traffic is tunneled but the response traffic\n is sent directly from the remote node. In \"DSR\" mode, the remote\n node appears to use the IP of the ingress node; this requires a\n permissive L2 network. [Default: Tunnel]'\n pattern: ^(?i)(Tunnel|DSR)?$\n type: string\n bpfForceTrackPacketsFromIfaces:\n description: 'BPFForceTrackPacketsFromIfaces in BPF mode, forces traffic\n from these interfaces to skip Calico''s iptables NOTRACK rule, allowing\n traffic from those interfaces to be tracked by Linux conntrack. Should\n only be used for interfaces that are not used for the Calico fabric. For\n example, a docker bridge device for non-Calico-networked containers.\n [Default: docker+]'\n items:\n type: string\n type: array\n bpfHostConntrackBypass:\n description: 'BPFHostConntrackBypass Controls whether to bypass Linux\n conntrack in BPF mode for workloads and services. [Default: true\n - bypass Linux conntrack]'\n type: boolean\n bpfHostNetworkedNATWithoutCTLB:\n description: 'BPFHostNetworkedNATWithoutCTLB when in BPF mode, controls\n whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing\n determines the CTLB behavior. [Default: Enabled]'\n enum:\n - Enabled\n - Disabled\n type: string\n bpfKubeProxyEndpointSlicesEnabled:\n description: BPFKubeProxyEndpointSlicesEnabled is deprecated and has\n no effect. BPF kube-proxy always accepts endpoint slices. This option\n will be removed in the next release.\n type: boolean\n bpfKubeProxyIptablesCleanupEnabled:\n description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF\n mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s\n iptables chains. Should only be enabled if kube-proxy is not running. [Default:\n true]'\n type: boolean\n bpfKubeProxyMinSyncPeriod:\n description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the\n minimum time between updates to the dataplane for Felix''s embedded\n kube-proxy. Lower values give reduced set-up latency. Higher values\n reduce Felix CPU usage by batching up more work. [Default: 1s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n bpfL3IfacePattern:\n description: BPFL3IfacePattern is a regular expression that allows\n to list tunnel devices like wireguard or vxlan (i.e., L3 devices)\n in addition to BPFDataIfacePattern. That is, tunnel interfaces not\n created by Calico, that Calico workload traffic flows over as well\n as any interfaces that handle incoming traffic to nodeports and\n services from outside the cluster.\n type: string\n bpfLogFilters:\n additionalProperties:\n type: string\n description: \"BPFLogFilters is a map of key=values where the value\n is a pcap filter expression and the key is an interface name with\n 'all' denoting all interfaces, 'weps' all workload endpoints and\n 'heps' all host endpoints. \\n When specified as an env var, it accepts\n a comma-separated list of key=values. [Default: unset - means all\n debug logs are emitted]\"\n type: object\n bpfLogLevel:\n description: 'BPFLogLevel controls the log level of the BPF programs\n when in BPF dataplane mode. One of \"Off\", \"Info\", or \"Debug\". The\n logs are emitted to the BPF trace pipe, accessible with the command\n `tc exec bpf debug`. [Default: Off].'\n pattern: ^(?i)(Off|Info|Debug)?$\n type: string\n bpfMapSizeConntrack:\n description: 'BPFMapSizeConntrack sets the size for the conntrack\n map. This map must be large enough to hold an entry for each active\n connection. Warning: changing the size of the conntrack map can\n cause disruption.'\n type: integer\n bpfMapSizeIPSets:\n description: BPFMapSizeIPSets sets the size for ipsets map. The IP\n sets map must be large enough to hold an entry for each endpoint\n matched by every selector in the source/destination matches in network\n policy. Selectors such as \"all()\" can result in large numbers of\n entries (one entry per endpoint in that case).\n type: integer\n bpfMapSizeIfState:\n description: BPFMapSizeIfState sets the size for ifstate map. The\n ifstate map must be large enough to hold an entry for each device\n (host + workloads) on a host.\n type: integer\n bpfMapSizeNATAffinity:\n type: integer\n bpfMapSizeNATBackend:\n description: BPFMapSizeNATBackend sets the size for nat back end map.\n This is the total number of endpoints. This is mostly more than\n the size of the number of services.\n type: integer\n bpfMapSizeNATFrontend:\n description: BPFMapSizeNATFrontend sets the size for nat front end\n map. FrontendMap should be large enough to hold an entry for each\n nodeport, external IP and each port in each service.\n type: integer\n bpfMapSizeRoute:\n description: BPFMapSizeRoute sets the size for the routes map. The\n routes map should be large enough to hold one entry per workload\n and a handful of entries per host (enough to cover its own IPs and\n tunnel IPs).\n type: integer\n bpfPSNATPorts:\n anyOf:\n - type: integer\n - type: string\n description: 'BPFPSNATPorts sets the range from which we randomly\n pick a port if there is a source port collision. This should be\n within the ephemeral range as defined by RFC 6056 (1024–65535) and\n preferably outside the ephemeral ranges used by common operating\n systems. Linux uses 32768–60999, while others mostly use the IANA\n defined range 49152–65535. It is not necessarily a problem if this\n range overlaps with the operating systems. Both ends of the range\n are inclusive. [Default: 20000:29999]'\n pattern: ^.*\n x-kubernetes-int-or-string: true\n bpfPolicyDebugEnabled:\n description: BPFPolicyDebugEnabled when true, Felix records detailed\n information about the BPF policy programs, which can be examined\n with the calico-bpf command-line tool.\n type: boolean\n bpfRedirectToPeer:\n description: 'BPFRedirectToPeer controls which whether it is allowed\n to forward straight to the peer side of the workload devices. It\n is allowed for any host L2 devices by default (L2Only), but it breaks\n TCP dump on the host side of workload device as it bypasses it on\n ingress. Value of Enabled also allows redirection from L3 host devices\n like IPIP tunnel or Wireguard directly to the peer side of the workload''s\n device. This makes redirection faster, however, it breaks tools\n like tcpdump on the peer side. Use Enabled with caution. [Default:\n L2Only]'\n type: string\n chainInsertMode:\n description: 'ChainInsertMode controls whether Felix hooks the kernel''s\n top-level iptables chains by inserting a rule at the top of the\n chain or by appending a rule at the bottom. insert is the safe default\n since it prevents Calico''s rules from being bypassed. If you switch\n to append mode, be sure that the other rules in the chains signal\n acceptance by falling through to the Calico rules, otherwise the\n Calico policy will be bypassed. [Default: insert]'\n pattern: ^(?i)(insert|append)?$\n type: string\n dataplaneDriver:\n description: DataplaneDriver filename of the external dataplane driver\n to use. Only used if UseInternalDataplaneDriver is set to false.\n type: string\n dataplaneWatchdogTimeout:\n description: \"DataplaneWatchdogTimeout is the readiness/liveness timeout\n used for Felix's (internal) dataplane driver. Increase this value\n if you experience spurious non-ready or non-live events when Felix\n is under heavy load. Decrease the value to get felix to report non-live\n or non-ready more quickly. [Default: 90s] \\n Deprecated: replaced\n by the generic HealthTimeoutOverrides.\"\n type: string\n debugDisableLogDropping:\n type: boolean\n debugHost:\n description: DebugHost is the host IP or hostname to bind the debug\n port to. Only used if DebugPort is set. [Default:localhost]\n type: string\n debugMemoryProfilePath:\n type: string\n debugPort:\n description: DebugPort if set, enables Felix's debug HTTP port, which\n allows memory and CPU profiles to be retrieved. The debug port\n is not secure, it should not be exposed to the internet.\n type: integer\n debugSimulateCalcGraphHangAfter:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n debugSimulateDataplaneApplyDelay:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n debugSimulateDataplaneHangAfter:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n defaultEndpointToHostAction:\n description: 'DefaultEndpointToHostAction controls what happens to\n traffic that goes from a workload endpoint to the host itself (after\n the traffic hits the endpoint egress policy). By default Calico\n blocks traffic from workload endpoints to the host itself with an\n iptables \"DROP\" action. If you want to allow some or all traffic\n from endpoint to host, set this parameter to RETURN or ACCEPT. Use\n RETURN if you have your own rules in the iptables \"INPUT\" chain;\n Calico will insert its rules at the top of that chain, then \"RETURN\"\n packets to the \"INPUT\" chain once it has completed processing workload\n endpoint egress policy. Use ACCEPT to unconditionally accept packets\n from workloads after processing workload endpoint egress policy.\n [Default: Drop]'\n pattern: ^(?i)(Drop|Accept|Return)?$\n type: string\n deviceRouteProtocol:\n description: This defines the route protocol added to programmed device\n routes, by default this will be RTPROT_BOOT when left blank.\n type: integer\n deviceRouteSourceAddress:\n description: This is the IPv4 source address to use on programmed\n device routes. By default the source address is left blank, leaving\n the kernel to choose the source address used.\n type: string\n deviceRouteSourceAddressIPv6:\n description: This is the IPv6 source address to use on programmed\n device routes. By default the source address is left blank, leaving\n the kernel to choose the source address used.\n type: string\n disableConntrackInvalidCheck:\n type: boolean\n endpointReportingDelay:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n endpointReportingEnabled:\n type: boolean\n endpointStatusPathPrefix:\n description: \"EndpointStatusPathPrefix is the path to the directory\n where endpoint status will be written. Endpoint status file reporting\n is disabled if field is left empty. \\n Chosen directory should match\n the directory used by the CNI for PodStartupDelay. [Default: \\\"\\\"]\"\n type: string\n externalNodesList:\n description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes\n which may source tunnel traffic and have the tunneled traffic be\n accepted at calico nodes.\n items:\n type: string\n type: array\n failsafeInboundHostPorts:\n description: 'FailsafeInboundHostPorts is a list of PortProto struct\n objects including UDP/TCP/SCTP ports and CIDRs that Felix will allow\n incoming traffic to host endpoints on irrespective of the security\n policy. This is useful to avoid accidentally cutting off a host\n with incorrect configuration. For backwards compatibility, if the\n protocol is not specified, it defaults to \"tcp\". If a CIDR is not\n specified, it will allow traffic from all addresses. To disable\n all inbound host ports, use the value \"[]\". The default value allows\n ssh access, DHCP, BGP, etcd and the Kubernetes API. [Default: tcp:22,\n udp:68, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666,\n tcp:6667 ]'\n items:\n description: ProtoPort is combination of protocol, port, and CIDR.\n Protocol and port must be specified.\n properties:\n net:\n type: string\n port:\n type: integer\n protocol:\n type: string\n required:\n - port\n - protocol\n type: object\n type: array\n failsafeOutboundHostPorts:\n description: 'FailsafeOutboundHostPorts is a list of List of PortProto\n struct objects including UDP/TCP/SCTP ports and CIDRs that Felix\n will allow outgoing traffic from host endpoints to irrespective\n of the security policy. This is useful to avoid accidentally cutting\n off a host with incorrect configuration. For backwards compatibility,\n if the protocol is not specified, it defaults to \"tcp\". If a CIDR\n is not specified, it will allow traffic from all addresses. To disable\n all outbound host ports, use the value \"[]\". The default value opens\n etcd''s standard ports to ensure that Felix does not get cut off\n from etcd as well as allowing DHCP, DNS, BGP and the Kubernetes\n API. [Default: udp:53, udp:67, tcp:179, tcp:2379, tcp:2380, tcp:5473,\n tcp:6443, tcp:6666, tcp:6667 ]'\n items:\n description: ProtoPort is combination of protocol, port, and CIDR.\n Protocol and port must be specified.\n properties:\n net:\n type: string\n port:\n type: integer\n protocol:\n type: string\n required:\n - port\n - protocol\n type: object\n type: array\n featureDetectOverride:\n description: FeatureDetectOverride is used to override feature detection\n based on auto-detected platform capabilities. Values are specified\n in a comma separated list with no spaces, example; \"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\". \"true\"\n or \"false\" will force the feature, empty or omitted values are auto-detected.\n pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$\n type: string\n featureGates:\n description: FeatureGates is used to enable or disable tech-preview\n Calico features. Values are specified in a comma separated list\n with no spaces, example; \"BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false\".\n This is used to enable features that are not fully production ready.\n pattern: ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$\n type: string\n floatingIPs:\n description: FloatingIPs configures whether or not Felix will program\n non-OpenStack floating IP addresses. (OpenStack-derived floating\n IPs are always programmed, regardless of this setting.)\n enum:\n - Enabled\n - Disabled\n type: string\n genericXDPEnabled:\n description: 'GenericXDPEnabled enables Generic XDP so network cards\n that don''t support XDP offload or driver modes can use XDP. This\n is not recommended since it doesn''t provide better performance\n than iptables. [Default: false]'\n type: boolean\n goGCThreshold:\n description: \"GoGCThreshold Sets the Go runtime's garbage collection\n threshold. I.e. the percentage that the heap is allowed to grow\n before garbage collection is triggered. In general, doubling the\n value halves the CPU time spent doing GC, but it also doubles peak\n GC memory overhead. A special value of -1 can be used to disable\n GC entirely; this should only be used in conjunction with the GoMemoryLimitMB\n setting. \\n This setting is overridden by the GOGC environment variable.\n \\n [Default: 40]\"\n type: integer\n goMaxProcs:\n description: \"GoMaxProcs sets the maximum number of CPUs that the\n Go runtime will use concurrently. A value of -1 means \\\"use the\n system default\\\"; typically the number of real CPUs on the system.\n \\n this setting is overridden by the GOMAXPROCS environment variable.\n \\n [Default: -1]\"\n type: integer\n goMemoryLimitMB:\n description: \"GoMemoryLimitMB sets a (soft) memory limit for the Go\n runtime in MB. The Go runtime will try to keep its memory usage\n under the limit by triggering GC as needed. To avoid thrashing,\n it will exceed the limit if GC starts to take more than 50% of the\n process's CPU time. A value of -1 disables the memory limit. \\n\n Note that the memory limit, if used, must be considerably less than\n any hard resource limit set at the container or pod level. This\n is because felix is not the only process that must run in the container\n or pod. \\n This setting is overridden by the GOMEMLIMIT environment\n variable. \\n [Default: -1]\"\n type: integer\n healthEnabled:\n type: boolean\n healthHost:\n type: string\n healthPort:\n type: integer\n healthTimeoutOverrides:\n description: HealthTimeoutOverrides allows the internal watchdog timeouts\n of individual subcomponents to be overridden. This is useful for\n working around \"false positive\" liveness timeouts that can occur\n in particularly stressful workloads or if CPU is constrained. For\n a list of active subcomponents, see Felix's logs.\n items:\n properties:\n name:\n type: string\n timeout:\n type: string\n required:\n - name\n - timeout\n type: object\n type: array\n interfaceExclude:\n description: 'InterfaceExclude is a comma-separated list of interfaces\n that Felix should exclude when monitoring for host endpoints. The\n default value ensures that Felix ignores Kubernetes'' IPVS dummy\n interface, which is used internally by kube-proxy. If you want to\n exclude multiple interface names using a single value, the list\n supports regular expressions. For regular expressions you must wrap\n the value with ''/''. For example having values ''/^kube/,veth1''\n will exclude all interfaces that begin with ''kube'' and also the\n interface ''veth1''. [Default: kube-ipvs0]'\n type: string\n interfacePrefix:\n description: 'InterfacePrefix is the interface name prefix that identifies\n workload endpoints and so distinguishes them from host endpoint\n interfaces. Note: in environments other than bare metal, the orchestrators\n configure this appropriately. For example our Kubernetes and Docker\n integrations set the ''cali'' value, and our OpenStack integration\n sets the ''tap'' value. [Default: cali]'\n type: string\n interfaceRefreshInterval:\n description: InterfaceRefreshInterval is the period at which Felix\n rescans local interfaces to verify their state. The rescan can be\n disabled by setting the interval to 0.\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n ipForwarding:\n description: 'IPForwarding controls whether Felix sets the host sysctls\n to enable IP forwarding. IP forwarding is required when using Calico\n for workload networking. This should only be disabled on hosts\n where Calico is used for host protection. [Default: Enabled]'\n enum:\n - Enabled\n - Disabled\n type: string\n ipipEnabled:\n description: 'IPIPEnabled overrides whether Felix should configure\n an IPIP interface on the host. Optional as Felix determines this\n based on the existing IP pools. [Default: nil (unset)]'\n type: boolean\n ipipMTU:\n description: 'IPIPMTU is the MTU to set on the tunnel device. See\n Configuring MTU [Default: 1440]'\n type: integer\n ipsetsRefreshInterval:\n description: 'IpsetsRefreshInterval is the period at which Felix re-checks\n all iptables state to ensure that no other process has accidentally\n broken Calico''s rules. Set to 0 to disable iptables refresh. [Default:\n 90s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n iptablesBackend:\n description: IptablesBackend specifies which backend of iptables will\n be used. The default is Auto.\n pattern: ^(?i)(Auto|FelixConfiguration|FelixConfigurationList|Legacy|NFT)?$\n type: string\n iptablesFilterAllowAction:\n pattern: ^(?i)(Accept|Return)?$\n type: string\n iptablesFilterDenyAction:\n description: IptablesFilterDenyAction controls what happens to traffic\n that is denied by network policy. By default Calico blocks traffic\n with an iptables \"DROP\" action. If you want to use \"REJECT\" action\n instead you can configure it in here.\n pattern: ^(?i)(Drop|Reject)?$\n type: string\n iptablesLockFilePath:\n description: 'IptablesLockFilePath is the location of the iptables\n lock file. You may need to change this if the lock file is not in\n its standard location (for example if you have mapped it into Felix''s\n container at a different path). [Default: /run/xtables.lock]'\n type: string\n iptablesLockProbeInterval:\n description: 'IptablesLockProbeInterval is the time that Felix will\n wait between attempts to acquire the iptables lock if it is not\n available. Lower values make Felix more responsive when the lock\n is contended, but use more CPU. [Default: 50ms]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n iptablesLockTimeout:\n description: 'IptablesLockTimeout is the time that Felix will wait\n for the iptables lock, or 0, to disable. To use this feature, Felix\n must share the iptables lock file with all other processes that\n also take the lock. When running Felix inside a container, this\n requires the /run directory of the host to be mounted into the calico/node\n or calico/felix container. [Default: 0s disabled]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n iptablesMangleAllowAction:\n pattern: ^(?i)(Accept|Return)?$\n type: string\n iptablesMarkMask:\n description: 'IptablesMarkMask is the mask that Felix selects its\n IPTables Mark bits from. Should be a 32 bit hexadecimal number with\n at least 8 bits set, none of which clash with any other mark bits\n in use on the system. [Default: 0xff000000]'\n format: int32\n type: integer\n iptablesNATOutgoingInterfaceFilter:\n type: string\n iptablesPostWriteCheckInterval:\n description: 'IptablesPostWriteCheckInterval is the period after Felix\n has done a write to the dataplane that it schedules an extra read\n back in order to check the write was not clobbered by another process.\n This should only occur if another application on the system doesn''t\n respect the iptables lock. [Default: 1s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n iptablesRefreshInterval:\n description: 'IptablesRefreshInterval is the period at which Felix\n re-checks the IP sets in the dataplane to ensure that no other process\n has accidentally broken Calico''s rules. Set to 0 to disable IP\n sets refresh. Note: the default for this value is lower than the\n other refresh intervals as a workaround for a Linux kernel bug that\n was fixed in kernel version 4.11. If you are using v4.11 or greater\n you may want to set this to, a higher value to reduce Felix CPU\n usage. [Default: 10s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n ipv6Support:\n description: IPv6Support controls whether Felix enables support for\n IPv6 (if supported by the in-use dataplane).\n type: boolean\n kubeNodePortRanges:\n description: 'KubeNodePortRanges holds list of port ranges used for\n service node ports. Only used if felix detects kube-proxy running\n in ipvs mode. Felix uses these ranges to separate host and workload\n traffic. [Default: 30000:32767].'\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n logDebugFilenameRegex:\n description: LogDebugFilenameRegex controls which source code files\n have their Debug log output included in the logs. Only logs from\n files with names that match the given regular expression are included. The\n filter only applies to Debug level logs.\n type: string\n logFilePath:\n description: 'LogFilePath is the full path to the Felix log. Set to\n none to disable file logging. [Default: /var/log/calico/felix.log]'\n type: string\n logPrefix:\n description: 'LogPrefix is the log prefix that Felix uses when rendering\n LOG rules. [Default: calico-packet]'\n type: string\n logSeverityFile:\n description: 'LogSeverityFile is the log severity above which logs\n are sent to the log file. [Default: Info]'\n pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$\n type: string\n logSeveritySys:\n description: 'LogSeveritySys is the log severity above which logs\n are sent to the syslog. Set to None for no logging to syslog. [Default:\n Info]'\n pattern: ^(?i)(Debug|Info|Warning|Error|Fatal)?$\n type: string\n maxIpsetSize:\n description: MaxIpsetSize is the maximum number of IP addresses that\n can be stored in an IP set. Not applicable if using the nftables\n backend.\n type: integer\n metadataAddr:\n description: 'MetadataAddr is the IP address or domain name of the\n server that can answer VM queries for cloud-init metadata. In OpenStack,\n this corresponds to the machine running nova-api (or in Ubuntu,\n nova-api-metadata). A value of none (case-insensitive) means that\n Felix should not set up any NAT rule for the metadata path. [Default:\n 127.0.0.1]'\n type: string\n metadataPort:\n description: 'MetadataPort is the port of the metadata server. This,\n combined with global.MetadataAddr (if not ''None''), is used to\n set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort.\n In most cases this should not need to be changed [Default: 8775].'\n type: integer\n mtuIfacePattern:\n description: MTUIfacePattern is a regular expression that controls\n which interfaces Felix should scan in order to calculate the host's\n MTU. This should not match workload interfaces (usually named cali...).\n type: string\n natOutgoingAddress:\n description: NATOutgoingAddress specifies an address to use when performing\n source NAT for traffic in a natOutgoing pool that is leaving the\n network. By default the address used is an address on the interface\n the traffic is leaving on (ie it uses the iptables MASQUERADE target)\n type: string\n natPortRange:\n anyOf:\n - type: integer\n - type: string\n description: NATPortRange specifies the range of ports that is used\n for port mapping when doing outgoing NAT. When unset the default\n behavior of the network stack is used.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n netlinkTimeout:\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n nftablesFilterAllowAction:\n pattern: ^(?i)(Accept|Return)?$\n type: string\n nftablesFilterDenyAction:\n description: FilterDenyAction controls what happens to traffic that\n is denied by network policy. By default Calico blocks traffic with\n a \"drop\" action. If you want to use a \"reject\" action instead you\n can configure it here.\n pattern: ^(?i)(Drop|Reject)?$\n type: string\n nftablesMangleAllowAction:\n pattern: ^(?i)(Accept|Return)?$\n type: string\n nftablesMarkMask:\n description: 'MarkMask is the mask that Felix selects its nftables\n Mark bits from. Should be a 32 bit hexadecimal number with at least\n 8 bits set, none of which clash with any other mark bits in use\n on the system. [Default: 0xffff0000]'\n format: int32\n type: integer\n nftablesMode:\n description: 'NFTablesMode configures nftables support in Felix. [Default:\n Disabled]'\n type: string\n nftablesRefreshInterval:\n description: 'NftablesRefreshInterval controls the interval at which\n Felix periodically refreshes the nftables rules. [Default: 90s]'\n type: string\n openstackRegion:\n description: 'OpenstackRegion is the name of the region that a particular\n Felix belongs to. In a multi-region Calico/OpenStack deployment,\n this must be configured somehow for each Felix (here in the datamodel,\n or in felix.cfg or the environment on each compute node), and must\n match the [calico] openstack_region value configured in neutron.conf\n on each node. [Default: Empty]'\n type: string\n policySyncPathPrefix:\n description: 'PolicySyncPathPrefix is used to by Felix to communicate\n policy changes to external services, like Application layer policy.\n [Default: Empty]'\n type: string\n prometheusGoMetricsEnabled:\n description: 'PrometheusGoMetricsEnabled disables Go runtime metrics\n collection, which the Prometheus client does by default, when set\n to false. This reduces the number of metrics reported, reducing\n Prometheus load. [Default: true]'\n type: boolean\n prometheusMetricsEnabled:\n description: 'PrometheusMetricsEnabled enables the Prometheus metrics\n server in Felix if set to true. [Default: false]'\n type: boolean\n prometheusMetricsHost:\n description: 'PrometheusMetricsHost is the host that the Prometheus\n metrics server should bind to. [Default: empty]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. [Default: 9091]'\n type: integer\n prometheusProcessMetricsEnabled:\n description: 'PrometheusProcessMetricsEnabled disables process metrics\n collection, which the Prometheus client does by default, when set\n to false. This reduces the number of metrics reported, reducing\n Prometheus load. [Default: true]'\n type: boolean\n prometheusWireGuardMetricsEnabled:\n description: 'PrometheusWireGuardMetricsEnabled disables wireguard\n metrics collection, which the Prometheus client does by default,\n when set to false. This reduces the number of metrics reported,\n reducing Prometheus load. [Default: true]'\n type: boolean\n removeExternalRoutes:\n description: Whether or not to remove device routes that have not\n been programmed by Felix. Disabling this will allow external applications\n to also add device routes. This is enabled by default which means\n we will remove externally added routes.\n type: boolean\n reportingInterval:\n description: 'ReportingInterval is the interval at which Felix reports\n its status into the datastore or 0 to disable. Must be non-zero\n in OpenStack deployments. [Default: 30s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n reportingTTL:\n description: 'ReportingTTL is the time-to-live setting for process-wide\n status reports. [Default: 90s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n routeRefreshInterval:\n description: 'RouteRefreshInterval is the period at which Felix re-checks\n the routes in the dataplane to ensure that no other process has\n accidentally broken Calico''s rules. Set to 0 to disable route refresh.\n [Default: 90s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n routeSource:\n description: 'RouteSource configures where Felix gets its routing\n information. - WorkloadIPs: use workload endpoints to construct\n routes. - CalicoIPAM: the default - use IPAM data to construct routes.'\n pattern: ^(?i)(WorkloadIPs|CalicoIPAM)?$\n type: string\n routeSyncDisabled:\n description: RouteSyncDisabled will disable all operations performed\n on the route table. Set to true to run in network-policy mode only.\n type: boolean\n routeTableRange:\n description: Deprecated in favor of RouteTableRanges. Calico programs\n additional Linux route tables for various purposes. RouteTableRange\n specifies the indices of the route tables that Calico should use.\n properties:\n max:\n type: integer\n min:\n type: integer\n required:\n - max\n - min\n type: object\n routeTableRanges:\n description: Calico programs additional Linux route tables for various\n purposes. RouteTableRanges specifies a set of table index ranges\n that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`.\n items:\n properties:\n max:\n type: integer\n min:\n type: integer\n required:\n - max\n - min\n type: object\n type: array\n serviceLoopPrevention:\n description: 'When service IP advertisement is enabled, prevent routing\n loops to service IPs that are not in use, by dropping or rejecting\n packets that do not get DNAT''d by kube-proxy. Unless set to \"Disabled\",\n in which case such routing loops continue to be allowed. [Default:\n Drop]'\n pattern: ^(?i)(Drop|Reject|Disabled)?$\n type: string\n sidecarAccelerationEnabled:\n description: 'SidecarAccelerationEnabled enables experimental sidecar\n acceleration [Default: false]'\n type: boolean\n usageReportingEnabled:\n description: 'UsageReportingEnabled reports anonymous Calico version\n number and cluster size to projectcalico.org. Logs warnings returned\n by the usage server. For example, if a significant security vulnerability\n has been discovered in the version of Calico being used. [Default:\n true]'\n type: boolean\n usageReportingInitialDelay:\n description: 'UsageReportingInitialDelay controls the minimum delay\n before Felix makes a report. [Default: 300s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n usageReportingInterval:\n description: 'UsageReportingInterval controls the interval at which\n Felix makes reports. [Default: 86400s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n useInternalDataplaneDriver:\n description: UseInternalDataplaneDriver, if true, Felix will use its\n internal dataplane programming logic. If false, it will launch\n an external dataplane driver and communicate with it over protobuf.\n type: boolean\n vxlanEnabled:\n description: 'VXLANEnabled overrides whether Felix should create the\n VXLAN tunnel device for IPv4 VXLAN networking. Optional as Felix\n determines this based on the existing IP pools. [Default: nil (unset)]'\n type: boolean\n vxlanMTU:\n description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel\n device. See Configuring MTU [Default: 1410]'\n type: integer\n vxlanMTUV6:\n description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel\n device. See Configuring MTU [Default: 1390]'\n type: integer\n vxlanPort:\n type: integer\n vxlanVNI:\n type: integer\n windowsManageFirewallRules:\n description: 'WindowsManageFirewallRules configures whether or not\n Felix will program Windows Firewall rules. (to allow inbound access\n to its own metrics ports) [Default: Disabled]'\n enum:\n - Enabled\n - Disabled\n type: string\n wireguardEnabled:\n description: 'WireguardEnabled controls whether Wireguard is enabled\n for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network).\n [Default: false]'\n type: boolean\n wireguardEnabledV6:\n description: 'WireguardEnabledV6 controls whether Wireguard is enabled\n for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network).\n [Default: false]'\n type: boolean\n wireguardHostEncryptionEnabled:\n description: 'WireguardHostEncryptionEnabled controls whether Wireguard\n host-to-host encryption is enabled. [Default: false]'\n type: boolean\n wireguardInterfaceName:\n description: 'WireguardInterfaceName specifies the name to use for\n the IPv4 Wireguard interface. [Default: wireguard.cali]'\n type: string\n wireguardInterfaceNameV6:\n description: 'WireguardInterfaceNameV6 specifies the name to use for\n the IPv6 Wireguard interface. [Default: wg-v6.cali]'\n type: string\n wireguardKeepAlive:\n description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive\n option. Set 0 to disable. [Default: 0]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n wireguardListeningPort:\n description: 'WireguardListeningPort controls the listening port used\n by IPv4 Wireguard. [Default: 51820]'\n type: integer\n wireguardListeningPortV6:\n description: 'WireguardListeningPortV6 controls the listening port\n used by IPv6 Wireguard. [Default: 51821]'\n type: integer\n wireguardMTU:\n description: 'WireguardMTU controls the MTU on the IPv4 Wireguard\n interface. See Configuring MTU [Default: 1440]'\n type: integer\n wireguardMTUV6:\n description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard\n interface. See Configuring MTU [Default: 1420]'\n type: integer\n wireguardRoutingRulePriority:\n description: 'WireguardRoutingRulePriority controls the priority value\n to use for the Wireguard routing rule. [Default: 99]'\n type: integer\n wireguardThreadingEnabled:\n description: 'WireguardThreadingEnabled controls whether Wireguard\n has NAPI threading enabled. [Default: false]'\n type: boolean\n workloadSourceSpoofing:\n description: WorkloadSourceSpoofing controls whether pods can use\n the allowedSourcePrefixes annotation to send traffic with a source\n IP address that is not theirs. This is disabled by default. When\n set to \"Any\", pods can request any prefix.\n pattern: ^(?i)(Disabled|Any)?$\n type: string\n xdpEnabled:\n description: 'XDPEnabled enables XDP acceleration for suitable untracked\n incoming deny rules. [Default: true]'\n type: boolean\n xdpRefreshInterval:\n description: 'XDPRefreshInterval is the period at which Felix re-checks\n all XDP state to ensure that no other process has accidentally broken\n Calico''s BPF maps or attached programs. Set to 0 to disable XDP\n refresh. [Default: 90s]'\n pattern: ^([0-9]+(\\\\.[0-9]+)?(ms|s|m|h))*$\n type: string\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: globalnetworkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: GlobalNetworkPolicy\n listKind: GlobalNetworkPolicyList\n plural: globalnetworkpolicies\n singular: globalnetworkpolicy\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n properties:\n applyOnForward:\n description: ApplyOnForward indicates to apply the rules in this policy\n on forward traffic.\n type: boolean\n doNotTrack:\n description: DoNotTrack indicates whether packets matched by the rules\n in this policy should go through the data plane's connection tracking,\n such as Linux conntrack. If True, the rules in this policy are\n applied before any data plane connection tracking, and packets allowed\n by this policy are marked as not to be tracked.\n type: boolean\n egress:\n description: The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description: The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n namespaceSelector:\n description: NamespaceSelector is an optional field for an expression\n used to select a pod based on namespaces.\n type: string\n order:\n description: Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order within the same tier. If the order\n is omitted, it may be considered to be \"infinite\" - i.e. the policy\n will be applied last. Policies with identical order will be applied\n in alphanumerical order based on the Policy \"Name\" within the tier.\n type: number\n performanceHints:\n description: \"PerformanceHints contains a list of hints to Calico's\n policy engine to help process the policy more efficiently. Hints\n never change the enforcement behaviour of the policy. \\n Currently,\n the only available hint is \\\"AssumeNeededOnEveryNode\\\". When that\n hint is set on a policy, Felix will act as if the policy matches\n a local endpoint even if it does not. This is useful for \\\"preloading\\\"\n any large static policies that are known to be used on every node.\n If the policy is _not_ used on a particular node then the work done\n to preload the policy (and to maintain it) is wasted.\"\n items:\n type: string\n type: array\n preDNAT:\n description: PreDNAT indicates to apply the rules in this policy before\n any DNAT.\n type: boolean\n selector:\n description: \"The selector is an expression used to pick out the endpoints\n that the policy should be applied to. \\n Selector expressions follow\n this syntax: \\n \\tlabel == \\\"string_literal\\\" -> comparison, e.g.\n my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\" -> not\n equal; also matches if label is not present \\tlabel in { \\\"a\\\",\n \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is one of\n \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... }\n \\ -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\", \\\"c\\\"\n \\thas(label_name) -> True if that label is present \\t! expr ->\n negation of expr \\texpr && expr -> Short-circuit and \\texpr ||\n expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description: ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n tier:\n description: The name of the tier that this policy belongs to. If\n this is omitted, the default tier (name is \"default\") is assumed. The\n specified tier must exist in order to create security policies within\n the tier, the \"default\" tier is created automatically if it does\n not exist, this means for deployments requiring only a single Tier,\n the tier name may be omitted on all policy management requests.\n type: string\n types:\n description: \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress rules are present in the policy. The\n default is: \\n - [ PolicyTypeIngress ], if there are no Egress rules\n (including the case where there are also no Ingress rules) \\n\n - [ PolicyTypeEgress ], if there are Egress rules but no Ingress\n rules \\n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are\n both Ingress and Egress rules. \\n When the policy is read back again,\n Types will always be one of these values, never empty or nil.\"\n items:\n description: PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: globalnetworksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: GlobalNetworkSet\n listKind: GlobalNetworkSetList\n plural: globalnetworksets\n singular: globalnetworkset\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs\n that share labels to allow rules to refer to them via selectors. The labels\n of GlobalNetworkSet are not namespaced.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: GlobalNetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: hostendpoints.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: HostEndpoint\n listKind: HostEndpointList\n plural: hostendpoints\n singular: hostendpoint\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: HostEndpointSpec contains the specification for a HostEndpoint\n resource.\n properties:\n expectedIPs:\n description: \"The expected IP addresses (IPv4 and IPv6) of the endpoint.\n If \\\"InterfaceName\\\" is not present, Calico will look for an interface\n matching any of the IPs in the list and apply policy to that. Note:\n \\tWhen using the selector match criteria in an ingress or egress\n security Policy \\tor Profile, Calico converts the selector into\n a set of IP addresses. For host \\tendpoints, the ExpectedIPs field\n is used for that purpose. (If only the interface \\tname is specified,\n Calico does not learn the IPs of the interface for use in match\n \\tcriteria.)\"\n items:\n type: string\n type: array\n interfaceName:\n description: \"Either \\\"*\\\", or the name of a specific Linux interface\n to apply policy to; or empty. \\\"*\\\" indicates that this HostEndpoint\n governs all traffic to, from or through the default network namespace\n of the host named by the \\\"Node\\\" field; entering and leaving that\n namespace via any interface, including those from/to non-host-networked\n local workloads. \\n If InterfaceName is not \\\"*\\\", this HostEndpoint\n only governs traffic that enters or leaves the host through the\n specific interface named by InterfaceName, or - when InterfaceName\n is empty - through the specific interface that has one of the IPs\n in ExpectedIPs. Therefore, when InterfaceName is empty, at least\n one expected IP must be specified. Only external interfaces (such\n as \\\"eth0\\\") are supported here; it isn't possible for a HostEndpoint\n to protect traffic through a specific local workload interface.\n \\n Note: Only some kinds of policy are implemented for \\\"*\\\" HostEndpoints;\n initially just pre-DNAT policy. Please check Calico documentation\n for the latest position.\"\n type: string\n node:\n description: The node name identifying the Calico node instance.\n type: string\n ports:\n description: Ports contains the endpoint's named ports, which may\n be referenced in security policy rules.\n items:\n properties:\n name:\n type: string\n port:\n type: integer\n protocol:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n required:\n - name\n - port\n - protocol\n type: object\n type: array\n profiles:\n description: A list of identifiers of security Profile objects that\n apply to this endpoint. Each profile is applied in the order that\n they appear in this list. Profile rules are applied after the selector-based\n security policy.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamblocks.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMBlock\n listKind: IPAMBlockList\n plural: ipamblocks\n singular: ipamblock\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMBlockSpec contains the specification for an IPAMBlock\n resource.\n properties:\n affinity:\n description: Affinity of the block, if this block has one. If set,\n it will be of the form \"host:\". If not set, this block\n is not affine to a host.\n type: string\n allocations:\n description: Array of allocations in-use within this block. nil entries\n mean the allocation is free. For non-nil entries at index i, the\n index is the ordinal of the allocation within this block and the\n value is the index of the associated attributes in the Attributes\n array.\n items:\n type: integer\n # TODO: This nullable is manually added in. We should update controller-gen\n # to handle []*int properly itself.\n nullable: true\n type: array\n attributes:\n description: Attributes is an array of arbitrary metadata associated\n with allocations in the block. To find attributes for a given allocation,\n use the value of the allocation's entry in the Allocations array\n as the index of the element in this array.\n items:\n properties:\n handle_id:\n type: string\n secondary:\n additionalProperties:\n type: string\n type: object\n type: object\n type: array\n cidr:\n description: The block's CIDR.\n type: string\n deleted:\n description: Deleted is an internal boolean used to workaround a limitation\n in the Kubernetes API whereby deletion will not return a conflict\n error if the block has been updated. It should not be set manually.\n type: boolean\n sequenceNumber:\n default: 0\n description: We store a sequence number that is updated each time\n the block is written. Each allocation will also store the sequence\n number of the block at the time of its creation. When releasing\n an IP, passing the sequence number associated with the allocation\n allows us to protect against a race condition and ensure the IP\n hasn't been released and re-allocated since the release request.\n format: int64\n type: integer\n sequenceNumberForAllocation:\n additionalProperties:\n format: int64\n type: integer\n description: Map of allocated ordinal within the block to sequence\n number of the block at the time of allocation. Kubernetes does not\n allow numerical keys for maps, so the key is cast to a string.\n type: object\n strictAffinity:\n description: StrictAffinity on the IPAMBlock is deprecated and no\n longer used by the code. Use IPAMConfig StrictAffinity instead.\n type: boolean\n unallocated:\n description: Unallocated is an ordered list of allocations which are\n free in the block.\n items:\n type: integer\n type: array\n required:\n - allocations\n - attributes\n - cidr\n - strictAffinity\n - unallocated\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMConfigSpec contains the specification for an IPAMConfig\n resource.\n properties:\n autoAllocateBlocks:\n type: boolean\n maxBlocksPerHost:\n description: MaxBlocksPerHost, if non-zero, is the max number of blocks\n that can be affine to each host.\n maximum: 2147483647\n minimum: 0\n type: integer\n strictAffinity:\n type: boolean\n required:\n - autoAllocateBlocks\n - strictAffinity\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamhandles.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPAMHandle\n listKind: IPAMHandleList\n plural: ipamhandles\n singular: ipamhandle\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPAMHandleSpec contains the specification for an IPAMHandle\n resource.\n properties:\n block:\n additionalProperties:\n type: integer\n type: object\n deleted:\n type: boolean\n handleID:\n type: string\n required:\n - block\n - handleID\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ippools.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPPool\n listKind: IPPoolList\n plural: ippools\n singular: ippool\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPPoolSpec contains the specification for an IPPool resource.\n properties:\n allowedUses:\n description: AllowedUse controls what the IP pool will be used for. If\n not specified or empty, defaults to [\"Tunnel\", \"Workload\"] for back-compatibility\n items:\n type: string\n type: array\n blockSize:\n description: The block size to use for IP address assignments from\n this pool. Defaults to 26 for IPv4 and 122 for IPv6.\n type: integer\n cidr:\n description: The pool CIDR.\n type: string\n disableBGPExport:\n description: 'Disable exporting routes from this IP Pool''s CIDR over\n BGP. [Default: false]'\n type: boolean\n disabled:\n description: When disabled is true, Calico IPAM will not assign addresses\n from this pool.\n type: boolean\n ipip:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n properties:\n enabled:\n description: When enabled is true, ipip tunneling will be used\n to deliver packets to destinations within this pool.\n type: boolean\n mode:\n description: The IPIP mode. This can be one of \"always\" or \"cross-subnet\". A\n mode of \"always\" will also use IPIP tunneling for routing to\n destination IP addresses within this pool. A mode of \"cross-subnet\"\n will only use IPIP tunneling when the destination node is on\n a different subnet to the originating node. The default value\n (if not specified) is \"always\".\n type: string\n type: object\n ipipMode:\n description: Contains configuration for IPIP tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. IPIP tunneling\n is disabled).\n type: string\n nat-outgoing:\n description: 'Deprecated: this field is only used for APIv1 backwards\n compatibility. Setting this field is not allowed, this field is\n for internal use only.'\n type: boolean\n natOutgoing:\n description: When natOutgoing is true, packets sent from Calico networked\n containers in this pool to destinations outside of this pool will\n be masqueraded.\n type: boolean\n nodeSelector:\n description: Allows IPPool to allocate for a specific node by label\n selector.\n type: string\n vxlanMode:\n description: Contains configuration for VXLAN tunneling for this pool.\n If not specified, then this is defaulted to \"Never\" (i.e. VXLAN\n tunneling is disabled).\n type: string\n required:\n - cidr\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: ipreservations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: IPReservation\n listKind: IPReservationList\n plural: ipreservations\n singular: ipreservation\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: IPReservationSpec contains the specification for an IPReservation\n resource.\n properties:\n reservedCIDRs:\n description: ReservedCIDRs is a list of CIDRs and/or IP addresses\n that Calico IPAM will exclude from new allocations.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: kubecontrollersconfigurations.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: KubeControllersConfiguration\n listKind: KubeControllersConfigurationList\n plural: kubecontrollersconfigurations\n singular: kubecontrollersconfiguration\n preserveUnknownFields: false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: KubeControllersConfigurationSpec contains the values of the\n Kubernetes controllers configuration.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description: Namespace enables and configures the namespace controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host endpoints.\n Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation of\n host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the controller\n to determine if an IP address has been leaked. Set to 0\n to disable IP garbage collection. [Default: 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform reconciliation\n with the Calico datastore. [Default: 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which logs\n are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default: 9094]'\n type: integer\n required:\n - controllers\n type: object\n status:\n description: KubeControllersConfigurationStatus represents the status\n of the configuration. It's useful for admins to be able to see the actual\n config that was applied, which can be modified by environment variables\n on the kube-controllers process.\n properties:\n environmentVars:\n additionalProperties:\n type: string\n description: EnvironmentVars contains the environment variables on\n the kube-controllers that influenced the RunningConfig.\n type: object\n runningConfig:\n description: RunningConfig contains the effective config that is running\n in the kube-controllers pod, after merging the API resource with\n any environment variables.\n properties:\n controllers:\n description: Controllers enables and configures individual Kubernetes\n controllers\n properties:\n namespace:\n description: Namespace enables and configures the namespace\n controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n node:\n description: Node enables and configures the node controller.\n Enabled by default, set to nil to disable.\n properties:\n hostEndpoint:\n description: HostEndpoint controls syncing nodes to host\n endpoints. Disabled by default, set to nil to disable.\n properties:\n autoCreate:\n description: 'AutoCreate enables automatic creation\n of host endpoints for every node. [Default: Disabled]'\n type: string\n type: object\n leakGracePeriod:\n description: 'LeakGracePeriod is the period used by the\n controller to determine if an IP address has been leaked.\n Set to 0 to disable IP garbage collection. [Default:\n 15m]'\n type: string\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n syncLabels:\n description: 'SyncLabels controls whether to copy Kubernetes\n node labels to Calico nodes. [Default: Enabled]'\n type: string\n type: object\n policy:\n description: Policy enables and configures the policy controller.\n Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n serviceAccount:\n description: ServiceAccount enables and configures the service\n account controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n workloadEndpoint:\n description: WorkloadEndpoint enables and configures the workload\n endpoint controller. Enabled by default, set to nil to disable.\n properties:\n reconcilerPeriod:\n description: 'ReconcilerPeriod is the period to perform\n reconciliation with the Calico datastore. [Default:\n 5m]'\n type: string\n type: object\n type: object\n debugProfilePort:\n description: DebugProfilePort configures the port to serve memory\n and cpu profiles on. If not specified, profiling is disabled.\n format: int32\n type: integer\n etcdV3CompactionPeriod:\n description: 'EtcdV3CompactionPeriod is the period between etcdv3\n compaction requests. Set to 0 to disable. [Default: 10m]'\n type: string\n healthChecks:\n description: 'HealthChecks enables or disables support for health\n checks [Default: Enabled]'\n type: string\n logSeverityScreen:\n description: 'LogSeverityScreen is the log severity above which\n logs are sent to the stdout. [Default: Info]'\n type: string\n prometheusMetricsPort:\n description: 'PrometheusMetricsPort is the TCP port that the Prometheus\n metrics server should bind to. Set to 0 to disable. [Default:\n 9094]'\n type: integer\n required:\n - controllers\n type: object\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networkpolicies.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkPolicy\n listKind: NetworkPolicyList\n plural: networkpolicies\n singular: networkpolicy\n preserveUnknownFields: false\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n properties:\n egress:\n description: The ordered set of egress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n ingress:\n description: The ordered set of ingress rules. Each rule contains\n a set of packet match criteria and a corresponding action to apply.\n items:\n description: \"A Rule encapsulates a set of match criteria and an\n action. Both selector-based security Policy and security Profiles\n reference rules - separated out as a list of rules for both ingress\n and egress packet matching. \\n Each positive match criteria has\n a negated version, prefixed with \\\"Not\\\". All the match criteria\n within a rule must be satisfied for a packet to match. A single\n rule can contain the positive and negative version of a match\n and both must be satisfied for the rule to match.\"\n properties:\n action:\n type: string\n destination:\n description: Destination contains the match criteria that apply\n to destination entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n http:\n description: HTTP contains match criteria that apply to HTTP\n requests.\n properties:\n methods:\n description: Methods is an optional field that restricts\n the rule to apply only to HTTP requests that use one of\n the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple\n methods are OR'd together.\n items:\n type: string\n type: array\n paths:\n description: 'Paths is an optional field that restricts\n the rule to apply to HTTP requests that use one of the\n listed HTTP Paths. Multiple paths are OR''d together.\n e.g: - exact: /foo - prefix: /bar NOTE: Each entry may\n ONLY specify either a `exact` or a `prefix` match. The\n validator will check for it.'\n items:\n description: 'HTTPPath specifies an HTTP path to match.\n It may be either of the form: exact: : which matches\n the path exactly or prefix: : which matches\n the path prefix'\n properties:\n exact:\n type: string\n prefix:\n type: string\n type: object\n type: array\n type: object\n icmp:\n description: ICMP is an optional field that restricts the rule\n to apply to a specific type and code of ICMP traffic. This\n should only be specified if the Protocol field is set to \"ICMP\"\n or \"ICMPv6\".\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n ipVersion:\n description: IPVersion is an optional field that restricts the\n rule to only match a specific IP version.\n type: integer\n metadata:\n description: Metadata contains additional information for this\n rule\n properties:\n annotations:\n additionalProperties:\n type: string\n description: Annotations is a set of key value pairs that\n give extra information about the rule\n type: object\n type: object\n notICMP:\n description: NotICMP is the negated version of the ICMP field.\n properties:\n code:\n description: Match on a specific ICMP code. If specified,\n the Type value must also be specified. This is a technical\n limitation imposed by the kernel's iptables firewall,\n which Calico uses to enforce the rule.\n type: integer\n type:\n description: Match on a specific ICMP type. For example\n a value of 8 refers to ICMP Echo Request (i.e. pings).\n type: integer\n type: object\n notProtocol:\n anyOf:\n - type: integer\n - type: string\n description: NotProtocol is the negated version of the Protocol\n field.\n pattern: ^.*\n x-kubernetes-int-or-string: true\n protocol:\n anyOf:\n - type: integer\n - type: string\n description: \"Protocol is an optional field that restricts the\n rule to only apply to traffic of a specific IP protocol. Required\n if any of the EntityRules contain Ports (because ports only\n apply to certain protocols). \\n Must be one of these string\n values: \\\"TCP\\\", \\\"UDP\\\", \\\"ICMP\\\", \\\"ICMPv6\\\", \\\"SCTP\\\",\n \\\"UDPLite\\\" or an integer in the range 1-255.\"\n pattern: ^.*\n x-kubernetes-int-or-string: true\n source:\n description: Source contains the match criteria that apply to\n source entity.\n properties:\n namespaceSelector:\n description: \"NamespaceSelector is an optional field that\n contains a selector expression. Only traffic that originates\n from (or terminates at) endpoints within the selected\n namespaces will be matched. When both NamespaceSelector\n and another selector are defined on the same rule, then\n only workload endpoints that are matched by both selectors\n will be selected by the rule. \\n For NetworkPolicy, an\n empty NamespaceSelector implies that the Selector is limited\n to selecting only workload endpoints in the same namespace\n as the NetworkPolicy. \\n For NetworkPolicy, `global()`\n NamespaceSelector implies that the Selector is limited\n to selecting only GlobalNetworkSet or HostEndpoint. \\n\n For GlobalNetworkPolicy, an empty NamespaceSelector implies\n the Selector applies to workload endpoints across all\n namespaces.\"\n type: string\n nets:\n description: Nets is an optional field that restricts the\n rule to only apply to traffic that originates from (or\n terminates at) IP addresses in any of the given subnets.\n items:\n type: string\n type: array\n notNets:\n description: NotNets is the negated version of the Nets\n field.\n items:\n type: string\n type: array\n notPorts:\n description: NotPorts is the negated version of the Ports\n field. Since only some protocols have ports, if any ports\n are specified it requires the Protocol match in the Rule\n to be set to \"TCP\" or \"UDP\".\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n notSelector:\n description: NotSelector is the negated version of the Selector\n field. See Selector field for subtleties with negated\n selectors.\n type: string\n ports:\n description: \"Ports is an optional field that restricts\n the rule to only apply to traffic that has a source (destination)\n port that matches one of these ranges/values. This value\n is a list of integers or strings that represent ranges\n of ports. \\n Since only some protocols have ports, if\n any ports are specified it requires the Protocol match\n in the Rule to be set to \\\"TCP\\\" or \\\"UDP\\\".\"\n items:\n anyOf:\n - type: integer\n - type: string\n pattern: ^.*\n x-kubernetes-int-or-string: true\n type: array\n selector:\n description: \"Selector is an optional field that contains\n a selector expression (see Policy for sample syntax).\n \\ Only traffic that originates from (terminates at) endpoints\n matching the selector will be matched. \\n Note that: in\n addition to the negated version of the Selector (see NotSelector\n below), the selector expression syntax itself supports\n negation. The two types of negation are subtly different.\n One negates the set of matched endpoints, the other negates\n the whole match: \\n \\tSelector = \\\"!has(my_label)\\\" matches\n packets that are from other Calico-controlled \\tendpoints\n that do not have the label \\\"my_label\\\". \\n \\tNotSelector\n = \\\"has(my_label)\\\" matches packets that are not from\n Calico-controlled \\tendpoints that do have the label \\\"my_label\\\".\n \\n The effect is that the latter will accept packets from\n non-Calico sources whereas the former is limited to packets\n from Calico-controlled endpoints.\"\n type: string\n serviceAccounts:\n description: ServiceAccounts is an optional field that restricts\n the rule to only apply to traffic that originates from\n (or terminates at) a pod running as a matching service\n account.\n properties:\n names:\n description: Names is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account whose name is in the list.\n items:\n type: string\n type: array\n selector:\n description: Selector is an optional field that restricts\n the rule to only apply to traffic that originates\n from (or terminates at) a pod running as a service\n account that matches the given label selector. If\n both Names and Selector are specified then they are\n AND'ed.\n type: string\n type: object\n services:\n description: \"Services is an optional field that contains\n options for matching Kubernetes Services. If specified,\n only traffic that originates from or terminates at endpoints\n within the selected service(s) will be matched, and only\n to/from each endpoint's port. \\n Services cannot be specified\n on the same rule as Selector, NotSelector, NamespaceSelector,\n Nets, NotNets or ServiceAccounts. \\n Ports and NotPorts\n can only be specified with Services on ingress rules.\"\n properties:\n name:\n description: Name specifies the name of a Kubernetes\n Service to match.\n type: string\n namespace:\n description: Namespace specifies the namespace of the\n given Service. If left empty, the rule will match\n within this policy's namespace.\n type: string\n type: object\n type: object\n required:\n - action\n type: object\n type: array\n order:\n description: Order is an optional field that specifies the order in\n which the policy is applied. Policies with higher \"order\" are applied\n after those with lower order within the same tier. If the order\n is omitted, it may be considered to be \"infinite\" - i.e. the policy\n will be applied last. Policies with identical order will be applied\n in alphanumerical order based on the Policy \"Name\" within the tier.\n type: number\n performanceHints:\n description: \"PerformanceHints contains a list of hints to Calico's\n policy engine to help process the policy more efficiently. Hints\n never change the enforcement behaviour of the policy. \\n Currently,\n the only available hint is \\\"AssumeNeededOnEveryNode\\\". When that\n hint is set on a policy, Felix will act as if the policy matches\n a local endpoint even if it does not. This is useful for \\\"preloading\\\"\n any large static policies that are known to be used on every node.\n If the policy is _not_ used on a particular node then the work done\n to preload the policy (and to maintain it) is wasted.\"\n items:\n type: string\n type: array\n selector:\n description: \"The selector is an expression used to pick out the endpoints\n that the policy should be applied to. \\n Selector expressions follow\n this syntax: \\n \\tlabel == \\\"string_literal\\\" -> comparison, e.g.\n my_label == \\\"foo bar\\\" \\tlabel != \\\"string_literal\\\" -> not\n equal; also matches if label is not present \\tlabel in { \\\"a\\\",\n \\\"b\\\", \\\"c\\\", ... } -> true if the value of label X is one of\n \\\"a\\\", \\\"b\\\", \\\"c\\\" \\tlabel not in { \\\"a\\\", \\\"b\\\", \\\"c\\\", ... }\n \\ -> true if the value of label X is not one of \\\"a\\\", \\\"b\\\", \\\"c\\\"\n \\thas(label_name) -> True if that label is present \\t! expr ->\n negation of expr \\texpr && expr -> Short-circuit and \\texpr ||\n expr -> Short-circuit or \\t( expr ) -> parens for grouping \\tall()\n or the empty selector -> matches all endpoints. \\n Label names are\n allowed to contain alphanumerics, -, _ and /. String literals are\n more permissive but they do not support escape characters. \\n Examples\n (with made-up labels): \\n \\ttype == \\\"webserver\\\" && deployment\n == \\\"prod\\\" \\ttype in {\\\"frontend\\\", \\\"backend\\\"} \\tdeployment !=\n \\\"dev\\\" \\t! has(label_name)\"\n type: string\n serviceAccountSelector:\n description: ServiceAccountSelector is an optional field for an expression\n used to select a pod based on service accounts.\n type: string\n tier:\n description: The name of the tier that this policy belongs to. If\n this is omitted, the default tier (name is \"default\") is assumed. The\n specified tier must exist in order to create security policies within\n the tier, the \"default\" tier is created automatically if it does\n not exist, this means for deployments requiring only a single Tier,\n the tier name may be omitted on all policy management requests.\n type: string\n types:\n description: \"Types indicates whether this policy applies to ingress,\n or to egress, or to both. When not explicitly specified (and so\n the value on creation is empty or nil), Calico defaults Types according\n to what Ingress and Egress are present in the policy. The default\n is: \\n - [ PolicyTypeIngress ], if there are no Egress rules (including\n the case where there are also no Ingress rules) \\n - [ PolicyTypeEgress\n ], if there are Egress rules but no Ingress rules \\n - [ PolicyTypeIngress,\n PolicyTypeEgress ], if there are both Ingress and Egress rules.\n \\n When the policy is read back again, Types will always be one\n of these values, never empty or nil.\"\n items:\n description: PolicyType enumerates the possible values of the PolicySpec\n Types field.\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: networksets.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: NetworkSet\n listKind: NetworkSetList\n plural: networksets\n singular: networkset\n preserveUnknownFields: false\n scope: Namespaced\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet.\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: NetworkSetSpec contains the specification for a NetworkSet\n resource.\n properties:\n nets:\n description: The list of IP networks that belong to this set.\n items:\n type: string\n type: array\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n controller-gen.kubebuilder.io/version: (devel)\n creationTimestamp: null\n name: tiers.crd.projectcalico.org\nspec:\n group: crd.projectcalico.org\n names:\n kind: Tier\n listKind: TierList\n plural: tiers\n singular: tier\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n properties:\n apiVersion:\n description: 'APIVersion defines the versioned schema of this representation\n of an object. Servers should convert recognized schemas to the latest\n internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n type: string\n kind:\n description: 'Kind is a string value representing the REST resource this\n object represents. Servers may infer this from the endpoint the client\n submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n type: string\n metadata:\n type: object\n spec:\n description: TierSpec contains the specification for a security policy\n tier resource.\n properties:\n defaultAction:\n description: 'DefaultAction specifies the action applied to workloads\n selected by a policy in the tier, but not rule matched the workload''s\n traffic. [Default: Deny]'\n enum:\n - Pass\n - Deny\n type: string\n order:\n description: Order is an optional field that specifies the order in\n which the tier is applied. Tiers with higher \"order\" are applied\n after those with lower order. If the order is omitted, it may be\n considered to be \"infinite\" - i.e. the tier will be applied last. Tiers\n with identical order will be applied in alphanumerical order based\n on the Tier \"Name\".\n type: number\n type: object\n type: object\n served: true\n storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: []\n storedVersions: []\n---\n# Source: calico/templates/kdd-crds.yaml\napiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n annotations:\n api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30\n policy.networking.k8s.io/bundle-version: v0.1.1\n policy.networking.k8s.io/channel: experimental\n creationTimestamp: null\n name: adminnetworkpolicies.policy.networking.k8s.io\nspec:\n group: policy.networking.k8s.io\n names:\n kind: AdminNetworkPolicy\n listKind: AdminNetworkPolicyList\n plural: adminnetworkpolicies\n shortNames:\n - anp\n singular: adminnetworkpolicy\n scope: Cluster\n versions:\n - additionalPrinterColumns:\n - jsonPath: .spec.priority\n name: Priority\n type: string\n - jsonPath: .metadata.creationTimestamp\n name: Age\n type: date\n name: v1alpha1\n schema:\n openAPIV3Schema:\n description: |-\n AdminNetworkPolicy is a cluster level resource that is part of the\n AdminNetworkPolicy API.\n properties:\n apiVersion:\n description: |-\n APIVersion defines the versioned schema of this representation of an object.\n Servers should convert recognized schemas to the latest internal value, and\n may reject unrecognized values.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources\n type: string\n kind:\n description: |-\n Kind is a string value representing the REST resource this object represents.\n Servers may infer this from the endpoint the client submits requests to.\n Cannot be updated.\n In CamelCase.\n More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds\n type: string\n metadata:\n type: object\n spec:\n description: Specification of the desired behavior of AdminNetworkPolicy.\n properties:\n egress:\n description: |-\n Egress is the list of Egress rules to be applied to the selected pods.\n A total of 100 rules will be allowed in each ANP instance.\n The relative precedence of egress rules within a single ANP object (all of\n which share the priority) will be determined by the order in which the rule\n is written. Thus, a rule that appears at the top of the egress rules\n would take the highest precedence.\n ANPs with no egress rules do not affect egress traffic.\n\n\n Support: Core\n items:\n description: |-\n AdminNetworkPolicyEgressRule describes an action to take on a particular\n set of traffic originating from pods selected by a AdminNetworkPolicy's\n Subject field.\n \n properties:\n action:\n description: |-\n Action specifies the effect this rule will have on matching traffic.\n Currently the following actions are supported:\n Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy)\n Deny: denies the selected traffic\n Pass: instructs the selected traffic to skip any remaining ANP rules, and\n then pass execution to any NetworkPolicies that select the pod.\n If the pod is not selected by any NetworkPolicies then execution\n is passed to any BaselineAdminNetworkPolicies that select the pod.\n\n\n Support: Core\n enum:\n - Allow\n - Deny\n - Pass\n type: string\n name:\n description: |-\n Name is an identifier for this rule, that may be no more than 100 characters\n in length. This field should be used by the implementation to help\n improve observability, readability and error-reporting for any applied\n AdminNetworkPolicies.\n\n\n Support: Core\n maxLength: 100\n type: string\n ports:\n description: |-\n Ports allows for matching traffic based on port and protocols.\n This field is a list of destination ports for the outgoing egress traffic.\n If Ports is not set then the rule does not filter traffic via port.\n\n\n Support: Core\n items:\n description: |-\n AdminNetworkPolicyPort describes how to select network ports on pod(s).\n Exactly one field must be set.\n maxProperties: 1\n minProperties: 1\n properties:\n namedPort:\n description: |-\n NamedPort selects a port on a pod(s) based on name.\n\n\n Support: Extended\n\n\n \n type: string\n portNumber:\n description: |-\n Port selects a port on a pod(s) based on number.\n\n\n Support: Core\n properties:\n port:\n description: |-\n Number defines a network port value.\n\n\n Support: Core\n format: int32\n maximum: 65535\n minimum: 1\n type: integer\n protocol:\n default: TCP\n description: |-\n Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must\n match. If not specified, this field defaults to TCP.\n\n\n Support: Core\n type: string\n required:\n - port\n - protocol\n type: object\n portRange:\n description: |-\n PortRange selects a port range on a pod(s) based on provided start and end\n values.\n\n\n Support: Core\n properties:\n end:\n description: |-\n End defines a network port that is the end of a port range, the End value\n must be greater than Start.\n\n\n Support: Core\n format: int32\n maximum: 65535\n minimum: 1\n type: integer\n protocol:\n default: TCP\n description: |-\n Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must\n match. If not specified, this field defaults to TCP.\n\n\n Support: Core\n type: string\n start:\n description: |-\n Start defines a network port that is the start of a port range, the Start\n value must be less than End.\n\n\n Support: Core\n format: int32\n maximum: 65535\n minimum: 1\n type: integer\n required:\n - end\n - start\n type: object\n type: object\n maxItems: 100\n type: array\n to:\n description: |-\n To is the List of destinations whose traffic this rule applies to.\n If any AdminNetworkPolicyEgressPeer matches the destination of outgoing\n traffic then the specified action is applied.\n This field must be defined and contain at least one item.\n\n\n Support: Core\n items:\n description: |-\n AdminNetworkPolicyEgressPeer defines a peer to allow traffic to.\n Exactly one of the selector pointers must be set for a given peer. If a\n consumer observes none of its fields are set, they must assume an unknown\n option has been specified and fail closed.\n maxProperties: 1\n minProperties: 1\n properties:\n namespaces:\n description: |-\n Namespaces defines a way to select all pods within a set of Namespaces.\n Note that host-networked pods are not included in this type of peer.\n\n\n Support: Core\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n networks:\n description: |-\n Networks defines a way to select peers via CIDR blocks.\n This is intended for representing entities that live outside the cluster,\n which can't be selected by pods, namespaces and nodes peers, but note\n that cluster-internal traffic will be checked against the rule as\n well. So if you Allow or Deny traffic to `\"0.0.0.0/0\"`, that will allow\n or deny all IPv4 pod-to-pod traffic as well. If you don't want that,\n add a rule that Passes all pod traffic before the Networks rule.\n\n\n Each item in Networks should be provided in the CIDR format and should be\n IPv4 or IPv6, for example \"10.0.0.0/8\" or \"fd00::/8\".\n\n\n Networks can have upto 25 CIDRs specified.\n\n\n Support: Extended\n\n\n \n items:\n description: |-\n CIDR is an IP address range in CIDR notation (for example, \"10.0.0.0/8\" or \"fd00::/8\").\n This string must be validated by implementations using net.ParseCIDR\n TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available.\n maxLength: 43\n type: string\n x-kubernetes-validations:\n - message: CIDR must be either an IPv4 or IPv6 address.\n IPv4 address embedded in IPv6 addresses are not\n supported\n rule: self.contains(':') != self.contains('.')\n maxItems: 25\n minItems: 1\n type: array\n x-kubernetes-list-type: set\n nodes:\n description: |-\n Nodes defines a way to select a set of nodes in\n the cluster. This field follows standard label selector\n semantics; if present but empty, it selects all Nodes.\n\n\n Support: Extended\n\n\n \n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n pods:\n description: |-\n Pods defines a way to select a set of pods in\n a set of namespaces. Note that host-networked pods\n are not included in this type of peer.\n\n\n Support: Core\n properties:\n namespaceSelector:\n description: |-\n NamespaceSelector follows standard label selector semantics; if empty,\n it selects all Namespaces.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label\n selector requirements. The requirements are\n ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the\n selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n podSelector:\n description: |-\n PodSelector is used to explicitly select pods within a namespace; if empty,\n it selects all Pods.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label\n selector requirements. The requirements are\n ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the\n selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n required:\n - namespaceSelector\n - podSelector\n type: object\n type: object\n maxItems: 100\n minItems: 1\n type: array\n required:\n - action\n - to\n type: object\n x-kubernetes-validations:\n - message: networks/nodes peer cannot be set with namedPorts since\n there are no namedPorts for networks/nodes\n rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes))\n && has(self.ports) && self.ports.exists(port, has(port.namedPort)))'\n maxItems: 100\n type: array\n ingress:\n description: |-\n Ingress is the list of Ingress rules to be applied to the selected pods.\n A total of 100 rules will be allowed in each ANP instance.\n The relative precedence of ingress rules within a single ANP object (all of\n which share the priority) will be determined by the order in which the rule\n is written. Thus, a rule that appears at the top of the ingress rules\n would take the highest precedence.\n ANPs with no ingress rules do not affect ingress traffic.\n\n\n Support: Core\n items:\n description: |-\n AdminNetworkPolicyIngressRule describes an action to take on a particular\n set of traffic destined for pods selected by an AdminNetworkPolicy's\n Subject field.\n properties:\n action:\n description: |-\n Action specifies the effect this rule will have on matching traffic.\n Currently the following actions are supported:\n Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy)\n Deny: denies the selected traffic\n Pass: instructs the selected traffic to skip any remaining ANP rules, and\n then pass execution to any NetworkPolicies that select the pod.\n If the pod is not selected by any NetworkPolicies then execution\n is passed to any BaselineAdminNetworkPolicies that select the pod.\n\n\n Support: Core\n enum:\n - Allow\n - Deny\n - Pass\n type: string\n from:\n description: |-\n From is the list of sources whose traffic this rule applies to.\n If any AdminNetworkPolicyIngressPeer matches the source of incoming\n traffic then the specified action is applied.\n This field must be defined and contain at least one item.\n\n\n Support: Core\n items:\n description: |-\n AdminNetworkPolicyIngressPeer defines an in-cluster peer to allow traffic from.\n Exactly one of the selector pointers must be set for a given peer. If a\n consumer observes none of its fields are set, they must assume an unknown\n option has been specified and fail closed.\n maxProperties: 1\n minProperties: 1\n properties:\n namespaces:\n description: |-\n Namespaces defines a way to select all pods within a set of Namespaces.\n Note that host-networked pods are not included in this type of peer.\n\n\n Support: Core\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n pods:\n description: |-\n Pods defines a way to select a set of pods in\n a set of namespaces. Note that host-networked pods\n are not included in this type of peer.\n\n\n Support: Core\n properties:\n namespaceSelector:\n description: |-\n NamespaceSelector follows standard label selector semantics; if empty,\n it selects all Namespaces.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label\n selector requirements. The requirements are\n ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the\n selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n podSelector:\n description: |-\n PodSelector is used to explicitly select pods within a namespace; if empty,\n it selects all Pods.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label\n selector requirements. The requirements are\n ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the\n selector applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n required:\n - namespaceSelector\n - podSelector\n type: object\n type: object\n maxItems: 100\n minItems: 1\n type: array\n name:\n description: |-\n Name is an identifier for this rule, that may be no more than 100 characters\n in length. This field should be used by the implementation to help\n improve observability, readability and error-reporting for any applied\n AdminNetworkPolicies.\n\n\n Support: Core\n maxLength: 100\n type: string\n ports:\n description: |-\n Ports allows for matching traffic based on port and protocols.\n This field is a list of ports which should be matched on\n the pods selected for this policy i.e the subject of the policy.\n So it matches on the destination port for the ingress traffic.\n If Ports is not set then the rule does not filter traffic via port.\n\n\n Support: Core\n items:\n description: |-\n AdminNetworkPolicyPort describes how to select network ports on pod(s).\n Exactly one field must be set.\n maxProperties: 1\n minProperties: 1\n properties:\n namedPort:\n description: |-\n NamedPort selects a port on a pod(s) based on name.\n\n\n Support: Extended\n\n\n \n type: string\n portNumber:\n description: |-\n Port selects a port on a pod(s) based on number.\n\n\n Support: Core\n properties:\n port:\n description: |-\n Number defines a network port value.\n\n\n Support: Core\n format: int32\n maximum: 65535\n minimum: 1\n type: integer\n protocol:\n default: TCP\n description: |-\n Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must\n match. If not specified, this field defaults to TCP.\n\n\n Support: Core\n type: string\n required:\n - port\n - protocol\n type: object\n portRange:\n description: |-\n PortRange selects a port range on a pod(s) based on provided start and end\n values.\n\n\n Support: Core\n properties:\n end:\n description: |-\n End defines a network port that is the end of a port range, the End value\n must be greater than Start.\n\n\n Support: Core\n format: int32\n maximum: 65535\n minimum: 1\n type: integer\n protocol:\n default: TCP\n description: |-\n Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must\n match. If not specified, this field defaults to TCP.\n\n\n Support: Core\n type: string\n start:\n description: |-\n Start defines a network port that is the start of a port range, the Start\n value must be less than End.\n\n\n Support: Core\n format: int32\n maximum: 65535\n minimum: 1\n type: integer\n required:\n - end\n - start\n type: object\n type: object\n maxItems: 100\n type: array\n required:\n - action\n - from\n type: object\n maxItems: 100\n type: array\n priority:\n description: |-\n Priority is a value from 0 to 1000. Rules with lower priority values have\n higher precedence, and are checked before rules with higher priority values.\n All AdminNetworkPolicy rules have higher precedence than NetworkPolicy or\n BaselineAdminNetworkPolicy rules\n The behavior is undefined if two ANP objects have same priority.\n\n\n Support: Core\n format: int32\n maximum: 1000\n minimum: 0\n type: integer\n subject:\n description: |-\n Subject defines the pods to which this AdminNetworkPolicy applies.\n Note that host-networked pods are not included in subject selection.\n\n\n Support: Core\n maxProperties: 1\n minProperties: 1\n properties:\n namespaces:\n description: Namespaces is used to select pods via namespace selectors.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n pods:\n description: Pods is used to select pods via namespace AND pod\n selectors.\n properties:\n namespaceSelector:\n description: |-\n NamespaceSelector follows standard label selector semantics; if empty,\n it selects all Namespaces.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n podSelector:\n description: |-\n PodSelector is used to explicitly select pods within a namespace; if empty,\n it selects all Pods.\n properties:\n matchExpressions:\n description: matchExpressions is a list of label selector\n requirements. The requirements are ANDed.\n items:\n description: |-\n A label selector requirement is a selector that contains values, a key, and an operator that\n relates the key and values.\n properties:\n key:\n description: key is the label key that the selector\n applies to.\n type: string\n operator:\n description: |-\n operator represents a key's relationship to a set of values.\n Valid operators are In, NotIn, Exists and DoesNotExist.\n type: string\n values:\n description: |-\n values is an array of string values. If the operator is In or NotIn,\n the values array must be non-empty. If the operator is Exists or DoesNotExist,\n the values array must be empty. This array is replaced during a strategic\n merge patch.\n items:\n type: string\n type: array\n required:\n - key\n - operator\n type: object\n type: array\n matchLabels:\n additionalProperties:\n type: string\n description: |-\n matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\n map is equivalent to an element of matchExpressions, whose key field is \"key\", the\n operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.\n type: object\n type: object\n x-kubernetes-map-type: atomic\n required:\n - namespaceSelector\n - podSelector\n type: object\n type: object\n required:\n - priority\n - subject\n type: object\n status:\n description: Status is the status to be reported by the implementation.\n properties:\n conditions:\n items:\n description: \"Condition contains details for one aspect of the current\n state of this API Resource.\\n---\\nThis struct is intended for\n direct use as an array at the field path .status.conditions. For\n example,\\n\\n\\n\\ttype FooStatus struct{\\n\\t // Represents the\n observations of a foo's current state.\\n\\t // Known .status.conditions.type\n are: \\\"Available\\\", \\\"Progressing\\\", and \\\"Degraded\\\"\\n\\t //\n +patchMergeKey=type\\n\\t // +patchStrategy=merge\\n\\t // +listType=map\\n\\t\n \\ // +listMapKey=type\\n\\t Conditions []metav1.Condition `json:\\\"conditions,omitempty\\\"\n patchStrategy:\\\"merge\\\" patchMergeKey:\\\"type\\\" protobuf:\\\"bytes,1,rep,name=conditions\\\"`\\n\\n\\n\\t\n \\ // other fields\\n\\t}\"\n properties:\n lastTransitionTime:\n description: |-\n lastTransitionTime is the last time the condition transitioned from one status to another.\n This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.\n format: date-time\n type: string\n message:\n description: |-\n message is a human readable message indicating details about the transition.\n This may be an empty string.\n maxLength: 32768\n type: string\n observedGeneration:\n description: |-\n observedGeneration represents the .metadata.generation that the condition was set based upon.\n For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\n with respect to the current state of the instance.\n format: int64\n minimum: 0\n type: integer\n reason:\n description: |-\n reason contains a programmatic identifier indicating the reason for the condition's last transition.\n Producers of specific condition types may define expected values and meanings for this field,\n and whether the values are considered a guaranteed API.\n The value should be a CamelCase string.\n This field may not be empty.\n maxLength: 1024\n minLength: 1\n pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$\n type: string\n status:\n description: status of the condition, one of True, False, Unknown.\n enum:\n - \"True\"\n - \"False\"\n - Unknown\n type: string\n type:\n description: |-\n type of condition in CamelCase or in foo.example.com/CamelCase.\n ---\n Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be\n useful (see .node.status.conditions), the ability to deconflict is important.\n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)\n maxLength: 316\n pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$\n type: string\n required:\n - lastTransitionTime\n - message\n - reason\n - status\n - type\n type: object\n type: array\n x-kubernetes-list-map-keys:\n - type\n x-kubernetes-list-type: map\n required:\n - conditions\n type: object\n required:\n - metadata\n - spec\n type: object\n served: true\n storage: true\n subresources:\n status: {}\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: null\n storedVersions: null\n---\n# Source: calico/templates/calico-kube-controllers-rbac.yaml\n# Include a clusterrole for the kube-controllers component,\n# and bind it to the calico-kube-controllers serviceaccount.\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-kube-controllers\nrules:\n # Nodes are watched to monitor for deletions.\n - apiGroups: [\"\"]\n resources:\n - nodes\n verbs:\n - watch\n - list\n - get\n # Pods are watched to check for existence as part of IPAM controller.\n - apiGroups: [\"\"]\n resources:\n - pods\n verbs:\n - get\n - list\n - watch\n # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ipreservations\n verbs:\n - list\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n - tiers\n verbs:\n - get\n - list\n - create\n - update\n - delete\n - watch\n # Pools are watched to maintain a mapping of blocks to IP pools.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ippools\n verbs:\n - list\n - watch\n # kube-controllers manages hostendpoints.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - hostendpoints\n verbs:\n - get\n - list\n - create\n - update\n - delete\n # Needs access to update clusterinformations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - clusterinformations\n verbs:\n - get\n - list\n - create\n - update\n - watch\n # KubeControllersConfiguration is where it gets its config\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - kubecontrollersconfigurations\n verbs:\n # read its own config\n - get\n - list\n # create a default if none exists\n - create\n # update status\n - update\n # watch for changes\n - watch\n---\n# Source: calico/templates/calico-node-rbac.yaml\n# Include a clusterrole for the calico-node DaemonSet,\n# and bind it to the calico-node serviceaccount.\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-node\nrules:\n # Used for creating service account tokens to be used by the CNI plugin\n - apiGroups: [\"\"]\n resources:\n - serviceaccounts/token\n resourceNames:\n - calico-cni-plugin\n verbs:\n - create\n # The CNI plugin needs to get pods, nodes, and namespaces.\n - apiGroups: [\"\"]\n resources:\n - pods\n - nodes\n - namespaces\n verbs:\n - get\n # EndpointSlices are used for Service-based network policy rule\n # enforcement.\n - apiGroups: [\"discovery.k8s.io\"]\n resources:\n - endpointslices\n verbs:\n - watch\n - list\n - apiGroups: [\"\"]\n resources:\n - endpoints\n - services\n verbs:\n # Used to discover service IPs for advertisement.\n - watch\n - list\n # Used to discover Typhas.\n - get\n # Pod CIDR auto-detection on kubeadm needs access to config maps.\n - apiGroups: [\"\"]\n resources:\n - configmaps\n verbs:\n - get\n - apiGroups: [\"\"]\n resources:\n - nodes/status\n verbs:\n # Needed for clearing NodeNetworkUnavailable flag.\n - patch\n # Calico stores some configuration information in node annotations.\n - update\n # Watch for changes to Kubernetes NetworkPolicies.\n - apiGroups: [\"networking.k8s.io\"]\n resources:\n - networkpolicies\n verbs:\n - watch\n - list\n # Watch for changes to Kubernetes AdminNetworkPolicies.\n - apiGroups: [\"policy.networking.k8s.io\"]\n resources:\n - adminnetworkpolicies\n verbs:\n - watch\n - list\n # Used by Calico for policy information.\n - apiGroups: [\"\"]\n resources:\n - pods\n - namespaces\n - serviceaccounts\n verbs:\n - list\n - watch\n # The CNI plugin patches pods/status.\n - apiGroups: [\"\"]\n resources:\n - pods/status\n verbs:\n - patch\n # Calico monitors various CRDs for config.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - globalfelixconfigs\n - felixconfigurations\n - bgppeers\n - bgpfilters\n - globalbgpconfigs\n - bgpconfigurations\n - ippools\n - ipreservations\n - ipamblocks\n - globalnetworkpolicies\n - globalnetworksets\n - networkpolicies\n - networksets\n - clusterinformations\n - hostendpoints\n - blockaffinities\n - caliconodestatuses\n - tiers\n verbs:\n - get\n - list\n - watch\n # Calico creates some tiers on startup.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - tiers\n verbs:\n - create\n # Calico must create and update some CRDs on startup.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ippools\n - felixconfigurations\n - clusterinformations\n verbs:\n - create\n - update\n # Calico must update some CRDs.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - caliconodestatuses\n verbs:\n - update\n # Calico stores some configuration information on the node.\n - apiGroups: [\"\"]\n resources:\n - nodes\n verbs:\n - get\n - list\n - watch\n # These permissions are only required for upgrade from v2.6, and can\n # be removed after upgrade or on fresh installations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - bgpconfigurations\n - bgppeers\n verbs:\n - create\n - update\n # These permissions are required for Calico CNI to perform IPAM allocations.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n verbs:\n - get\n - list\n - create\n - update\n - delete\n # The CNI plugin and calico/node need to be able to create a default\n # IPAMConfiguration\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - ipamconfigs\n verbs:\n - get\n - create\n # Block affinities must also be watchable by confd for route aggregation.\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n verbs:\n - watch\n # The Calico IPAM migration needs to get daemonsets. These permissions can be\n # removed if not upgrading from an installation using host-local IPAM.\n - apiGroups: [\"apps\"]\n resources:\n - daemonsets\n verbs:\n - get\n---\n# Source: calico/templates/calico-node-rbac.yaml\n# CNI cluster role\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-cni-plugin\nrules:\n - apiGroups: [\"\"]\n resources:\n - pods\n - nodes\n - namespaces\n verbs:\n - get\n - apiGroups: [\"\"]\n resources:\n - pods/status\n verbs:\n - patch\n - apiGroups: [\"crd.projectcalico.org\"]\n resources:\n - blockaffinities\n - ipamblocks\n - ipamhandles\n - clusterinformations\n - ippools\n - ipreservations\n - ipamconfigs\n verbs:\n - get\n - list\n - create\n - update\n - delete\n---\n# Source: calico/templates/calico-kube-controllers-rbac.yaml\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: calico-kube-controllers\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-kube-controllers\nsubjects:\n- kind: ServiceAccount\n name: calico-kube-controllers\n namespace: kube-system\n---\n# Source: calico/templates/calico-node-rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-node\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-node\nsubjects:\n- kind: ServiceAccount\n name: calico-node\n namespace: kube-system\n---\n# Source: calico/templates/calico-node-rbac.yaml\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: calico-cni-plugin\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: calico-cni-plugin\nsubjects:\n- kind: ServiceAccount\n name: calico-cni-plugin\n namespace: kube-system\n---\n# Source: calico/templates/calico-node.yaml\n# This manifest installs the calico-node container, as well\n# as the CNI plugins and network config on\n# each master and worker node in a Kubernetes cluster.\nkind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: calico-node\n namespace: kube-system\n labels:\n k8s-app: calico-node\nspec:\n selector:\n matchLabels:\n k8s-app: calico-node\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n template:\n metadata:\n labels:\n k8s-app: calico-node\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n # Make sure calico-node gets scheduled on all nodes.\n - effect: NoSchedule\n operator: Exists\n # Mark the pod as a critical add-on for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - effect: NoExecute\n operator: Exists\n serviceAccountName: calico-node\n securityContext:\n seccompProfile:\n type: RuntimeDefault\n # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a \"force\n # deletion\": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.\n terminationGracePeriodSeconds: 0\n priorityClassName: system-node-critical\n initContainers:\n # This container performs upgrade from host-local IPAM to calico-ipam.\n # It can be deleted if this is a fresh installation, or if you have already\n # upgraded to use calico-ipam.\n - name: upgrade-ipam\n image: docker.io/calico/cni:v3.29.3\n imagePullPolicy: IfNotPresent\n command: [\"/opt/cni/bin/calico-ipam\", \"-upgrade\"]\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n - name: KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n - name: CALICO_NETWORKING_BACKEND\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: calico_backend\n volumeMounts:\n - mountPath: /var/lib/cni/networks\n name: host-local-net-dir\n - mountPath: /host/opt/cni/bin\n name: cni-bin-dir\n securityContext:\n privileged: true\n # This container installs the CNI binaries\n # and CNI network config file on each node.\n - name: install-cni\n image: docker.io/calico/cni:v3.29.3\n imagePullPolicy: IfNotPresent\n command: [\"/opt/cni/bin/install\"]\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # Name of the CNI config file to create.\n - name: CNI_CONF_NAME\n value: \"10-calico.conflist\"\n # The CNI network config to install on each node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: cni_network_config\n # Set the hostname based on the k8s node name.\n - name: KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Prevents the container from sleeping forever.\n - name: SLEEP\n value: \"false\"\n - name: CNI_NET_DIR\n value: \"/var/snap/microk8s/current/args/cni-network\"\n volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: cni-net-dir\n securityContext:\n privileged: true\n # # This init container mounts the necessary filesystems needed by the BPF data plane\n # # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed\n # # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode.\n # - name: \"mount-bpffs\"\n # image: docker.io/calico/node:v3.29.3\n # imagePullPolicy: IfNotPresent\n # command: [\"calico-node\", \"-init\", \"-best-effort\"]\n # volumeMounts:\n # - mountPath: /sys/fs\n # name: sys-fs\n # # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host\n # # so that it outlives the init container.\n # mountPropagation: Bidirectional\n # - mountPath: /var/run/calico\n # name: var-run-calico\n # # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host\n # # so that it outlives the init container.\n # mountPropagation: Bidirectional\n # # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary,\n # # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly.\n # - mountPath: /nodeproc\n # name: nodeproc\n # readOnly: true\n # securityContext:\n # privileged: true\n containers:\n # Runs calico-node container on each Kubernetes node. This\n # container programs network policy and routes on each\n # host.\n - name: calico-node\n image: docker.io/calico/node:v3.29.3\n imagePullPolicy: IfNotPresent\n envFrom:\n - configMapRef:\n # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.\n name: kubernetes-services-endpoint\n optional: true\n env:\n # Use Kubernetes API as the backing datastore.\n - name: DATASTORE_TYPE\n value: \"kubernetes\"\n # Wait for the datastore.\n - name: WAIT_FOR_DATASTORE\n value: \"true\"\n # Set based on the k8s node name.\n - name: NODENAME\n valueFrom:\n fieldRef:\n fieldPath: spec.nodeName\n # Choose the backend to use.\n - name: CALICO_NETWORKING_BACKEND\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: calico_backend\n # Cluster type to identify the deployment type\n - name: CLUSTER_TYPE\n value: \"k8s,bgp\"\n # Auto-detect the BGP IP address.\n - name: IP\n value: \"autodetect\"\n - name: IP_AUTODETECTION_METHOD\n value: \"first-found\"\n # Enable IPIP\n # - name: CALICO_IPV4POOL_IPIP\n # value: \"Always\"\n # Enable or Disable VXLAN on the default IP pool.\n - name: CALICO_IPV4POOL_VXLAN\n value: \"Always\"\n # Enable or Disable VXLAN on the default IPv6 IP pool.\n - name: CALICO_IPV6POOL_VXLAN\n value: \"Never\"\n # Set MTU for tunnel device used if ipip is enabled\n - name: FELIX_IPINIPMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Set MTU for the VXLAN tunnel device.\n - name: FELIX_VXLANMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # Set MTU for the Wireguard tunnel device.\n - name: FELIX_WIREGUARDMTU\n valueFrom:\n configMapKeyRef:\n name: calico-config\n key: veth_mtu\n # The default IPv4 pool to create on startup if none exists. Pod IPs will be\n # chosen from this range. Changing this value after installation will have\n # no effect. This should fall within `--cluster-cidr`.\n - name: CALICO_IPV4POOL_CIDR\n value: \"10.1.0.0/16\"\n # Disable file logging so `kubectl logs` works.\n - name: CALICO_DISABLE_FILE_LOGGING\n value: \"true\"\n # Set Felix endpoint to host default action to ACCEPT.\n - name: FELIX_DEFAULTENDPOINTTOHOSTACTION\n value: \"ACCEPT\"\n # Disable IPv6 on Kubernetes.\n - name: FELIX_IPV6SUPPORT\n value: \"false\"\n - name: FELIX_HEALTHENABLED\n value: \"true\"\n - name: FELIX_FEATUREDETECTOVERRIDE\n value: ChecksumOffloadBroken=true\n securityContext:\n privileged: true\n resources:\n requests:\n cpu: 250m\n lifecycle:\n preStop:\n exec:\n command:\n - /bin/calico-node\n - -shutdown\n livenessProbe:\n exec:\n command:\n - /bin/calico-node\n - -felix-live\n # - -bird-live\n periodSeconds: 10\n initialDelaySeconds: 10\n failureThreshold: 6\n timeoutSeconds: 10\n readinessProbe:\n exec:\n command:\n - /bin/calico-node\n - -felix-ready\n # - -bird-ready\n periodSeconds: 10\n timeoutSeconds: 10\n volumeMounts:\n # For maintaining CNI plugin API credentials.\n - mountPath: /host/etc/cni/net.d\n name: cni-net-dir\n readOnly: false\n - mountPath: /lib/modules\n name: lib-modules\n readOnly: true\n - mountPath: /run/xtables.lock\n name: xtables-lock\n readOnly: false\n - mountPath: /var/run/calico\n name: var-run-calico\n readOnly: false\n - mountPath: /var/lib/calico\n name: var-lib-calico\n readOnly: false\n - name: policysync\n mountPath: /var/run/nodeagent\n # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the\n # parent directory.\n # - name: bpffs\n # mountPath: /sys/fs/bpf\n - name: cni-log-dir\n mountPath: /var/log/calico/cni\n readOnly: true\n volumes:\n # Used by calico-node.\n - name: lib-modules\n hostPath:\n path: /lib/modules\n - name: var-run-calico\n hostPath:\n path: /var/snap/microk8s/current/var/run/calico\n - name: var-lib-calico\n hostPath:\n path: /var/snap/microk8s/current/var/lib/calico\n - name: xtables-lock\n hostPath:\n path: /run/xtables.lock\n type: FileOrCreate\n - name: sys-fs\n hostPath:\n path: /sys/fs/\n type: DirectoryOrCreate\n # - name: bpffs\n # hostPath:\n # path: /sys/fs/bpf\n # type: Directory\n # # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs.\n # - name: nodeproc\n # hostPath:\n # path: /proc\n # Used to install CNI.\n - name: cni-bin-dir\n hostPath:\n path: /var/snap/microk8s/current/opt/cni/bin\n type: DirectoryOrCreate\n - name: cni-net-dir\n hostPath:\n path: /var/snap/microk8s/current/args/cni-network\n # Used to access CNI logs.\n - name: cni-log-dir\n hostPath:\n path: /var/snap/microk8s/common/var/log/calico/cni\n # Mount in the directory for host-local IPAM allocations. This is\n # used when upgrading from host-local to calico-ipam, and can be removed\n # if not using the upgrade-ipam init container.\n - name: host-local-net-dir\n hostPath:\n path: /var/snap/microk8s/current/var/lib/cni/networks\n # Used to create per-pod Unix Domain Sockets\n - name: policysync\n hostPath:\n type: DirectoryOrCreate\n path: /var/snap/microk8s/current/var/run/nodeagent\n---\n# Source: calico/templates/calico-kube-controllers.yaml\n# See https://github.com/projectcalico/kube-controllers\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\nspec:\n # The controllers can only have a single active instance.\n replicas: 1\n selector:\n matchLabels:\n k8s-app: calico-kube-controllers\n strategy:\n type: Recreate\n template:\n metadata:\n name: calico-kube-controllers\n namespace: kube-system\n labels:\n k8s-app: calico-kube-controllers\n spec:\n nodeSelector:\n kubernetes.io/os: linux\n tolerations:\n # Mark the pod as a critical add-on for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n - key: node-role.kubernetes.io/control-plane\n effect: NoSchedule\n serviceAccountName: calico-kube-controllers\n securityContext:\n seccompProfile:\n type: RuntimeDefault\n priorityClassName: system-cluster-critical\n containers:\n - name: calico-kube-controllers\n image: docker.io/calico/kube-controllers:v3.29.3\n imagePullPolicy: IfNotPresent\n env:\n # Choose which controllers to run.\n - name: ENABLED_CONTROLLERS\n value: node\n - name: DATASTORE_TYPE\n value: kubernetes\n livenessProbe:\n exec:\n command:\n - /usr/bin/check-status\n - -l\n periodSeconds: 10\n initialDelaySeconds: 10\n failureThreshold: 6\n timeoutSeconds: 10\n readinessProbe:\n exec:\n command:\n - /usr/bin/check-status\n - -r\n periodSeconds: 10\n securityContext:\n runAsNonRoot: true\n" }, { "path": "upgrade-scripts/000-switch-to-calico/rollback-master.sh", "content": "#!/bin/bash\nset -ex\n\necho \"Rolling back calico upgrade on master\"\n\nsource $SNAP/actions/common/utils.sh\n\n\nif [ -e \"$SNAP_DATA/args/cni-network/cni.yaml\" ]; then\n KUBECTL=\"$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config\"\n $KUBECTL delete -f \"$SNAP_DATA/args/cni-network/cni.yaml\"\nfi\n\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico\"\n\nif [ -e \"$BACKUP_DIR/args/cni-network/flannel.conflist\" ]; then\n find \"$SNAP_DATA\"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \\;\n cp -rf \"$BACKUP_DIR\"/args/cni-network/* \"$SNAP_DATA/args/cni-network/\"\nfi\n\necho \"Restarting kubelet\"\nif [ -e \"$BACKUP_DIR/args/kubelet\" ]; then\n cp \"$BACKUP_DIR\"/args/kubelet \"$SNAP_DATA/args/\"\nfi\n\necho \"Restarting kube-proxy\"\nif [ -e \"$BACKUP_DIR/args/kube-proxy\" ]; then\n cp \"$BACKUP_DIR\"/args/kube-proxy \"$SNAP_DATA/args/\"\nfi\n\necho \"Restarting kube-apiserver\"\nif [ -e \"$BACKUP_DIR/args/kube-apiserver\" ]; then\n cp \"$BACKUP_DIR\"/args/kube-apiserver \"$SNAP_DATA/args/\"\nfi\n\nsnapctl restart ${SNAP_NAME}.daemon-kubelite\n\n${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30\n\necho \"Restarting flannel\"\nset_service_expected_to_start flanneld\nremove_vxlan_interfaces\nsnapctl start ${SNAP_NAME}.daemon-flanneld\n\necho \"Restarting kubelet\"\nif grep -qE \"bin_dir.*SNAP_DATA}\\/\" $SNAP_DATA/args/containerd-template.toml; then\n echo \"Restarting containerd\"\n \"${SNAP}/bin/sed\" -i 's;bin_dir = \"${SNAP_DATA}/opt;bin_dir = \"${SNAP}/opt;g' \"$SNAP_DATA/args/containerd-template.toml\"\n snapctl restart ${SNAP_NAME}.daemon-containerd\nfi\n\necho \"Calico rolledback\"\n" }, { "path": "upgrade-scripts/000-switch-to-calico/rollback-node.sh", "content": "#!/bin/bash\nset -ex\n\necho \"Rolling back calico upgrade on a node\"\n\nsource $SNAP/actions/common/utils.sh\n\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico\"\n\nif [ -e \"$BACKUP_DIR/args/cni-network/flannel.conflist\" ]; then\n find \"$SNAP_DATA\"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \\;\n cp -rf \"$BACKUP_DIR\"/args/cni-network/* \"$SNAP_DATA/args/cni-network/\"\nfi\n\necho \"Restarting kubelet\"\nif [ -e \"$BACKUP_DIR/args/kubelet\" ]; then\n cp \"$BACKUP_DIR\"/args/kubelet \"$SNAP_DATA/args/\"\nfi\n\necho \"Restarting kube-proxy\"\nif [ -e \"$BACKUP_DIR/args/kube-proxy\" ]; then\n cp \"$BACKUP_DIR\"/args/kube-proxy \"$SNAP_DATA/args/\"\nfi\n\necho \"Restarting kube-apiserver\"\nif [ -e \"$BACKUP_DIR/args/kube-apiserver\" ]; then\n cp \"$BACKUP_DIR\"/args/kube-apiserver \"$SNAP_DATA/args/\"\nfi\n\nsnapctl restart ${SNAP_NAME}.daemon-kubelite\n\necho \"Restarting flannel\"\nset_service_expected_to_start flanneld\nremove_vxlan_interfaces\nsnapctl start ${SNAP_NAME}.daemon-flanneld\n\necho \"Restarting kubelet\"\nif grep -qE \"bin_dir.*SNAP_DATA}\\/\" $SNAP_DATA/args/containerd-template.toml; then\n echo \"Restarting containerd\"\n \"${SNAP}/bin/sed\" -i 's;bin_dir = \"${SNAP_DATA}/opt;bin_dir = \"${SNAP}/opt;g' \"$SNAP_DATA/args/containerd-template.toml\"\n snapctl restart ${SNAP_NAME}.daemon-containerd\nfi\n\necho \"Calico rolledback\"\n" }, { "path": "upgrade-scripts/001-switch-to-dqlite/commit-master.sh", "content": "#!/bin/bash\nset -ex\n\necho \"Switching master to dqlite\"\n\nsource $SNAP/actions/common/utils.sh\n\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite\"\nDB_DIR=\"$BACKUP_DIR/db\"\n\nmkdir -p \"$BACKUP_DIR/args/\"\n\necho \"Configuring services\"\n${SNAP}/microk8s-stop.wrapper\n\ncp \"$SNAP_DATA\"/args/kube-apiserver \"$BACKUP_DIR/args\"\n# Configure the API sever to talk to the external dqlite\nif [ -e ${SNAP}/default-args/k8s-dqlite ] &&\n ! [ -e ${SNAP_DATA}/args/k8s-dqlite ]\nthen\n echo \"Reconfiguring the API server for dqlite\"\n cp ${SNAP}/default-args/k8s-dqlite ${SNAP_DATA}/args/k8s-dqlite\nfi\n\nskip_opt_in_config storage-backend kube-apiserver\nskip_opt_in_config storage-dir kube-apiserver\n\nskip_opt_in_config \"etcd-servers\" kube-apiserver\nskip_opt_in_config \"etcd-cafile\" kube-apiserver\nskip_opt_in_config \"etcd-certfile\" kube-apiserver\nskip_opt_in_config \"etcd-keyfile\" kube-apiserver\n\nrefresh_opt_in_config etcd-servers unix://\\${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379 kube-apiserver\n\ncp \"$SNAP_DATA\"/args/etcd \"$BACKUP_DIR/args\"\ncat < \"$SNAP_DATA\"/args/etcd\n--data-dir=\\${SNAP_COMMON}/var/run/etcd\n--advertise-client-urls=http://127.0.0.1:12379\n--listen-client-urls=http://0.0.0.0:12379\n--enable-v2=true\nEOT\n\nif ! [ -e \"${SNAP_DATA}/var/kubernetes/backend/cluster.key\" ]\nthen\n init_cluster\nfi\n\nsnapctl restart ${SNAP_NAME}.daemon-k8s-dqlite\nsnapctl restart ${SNAP_NAME}.daemon-kubelite\n\nrun_etcd=\"$(is_service_expected_to_start etcd)\"\nif [ \"${run_etcd}\" == \"1\" ]\nthen\n snapctl start microk8s.daemon-etcd\n # TODO do some proper wait here\n sleep 15\n\n rm -rf \"$DB_DIR\"\n $SNAP/bin/k8s-dqlite migrator --mode backup --endpoint \"http://127.0.0.1:12379\" --db-dir \"$DB_DIR\" --debug\n chmod 600 \"$DB_DIR\"\n # Wait up to two minutes for the apiserver to come up.\n # TODO: this polling is not good enough. We should find a new way to ensure the apiserver is up.\n timeout=\"120\"\n start_timer=\"$(date +%s)\"\n while ! (is_apiserver_ready)\n do\n sleep 5\n now=\"$(date +%s)\"\n if [[ \"$now\" > \"$(($start_timer + $timeout))\" ]] ; then\n break\n fi\n done\n\n # if the API server came up try to load the CNI manifest\n now=\"$(date +%s)\"\n if [[ \"$now\" < \"$(($start_timer + $timeout))\" ]] ; then\n if (is_apiserver_ready)\n then\n $SNAP/bin/k8s-dqlite migrator --mode restore --endpoint \"unix://${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379\" --db-dir \"$DB_DIR\" --debug\n fi\n fi\n\n sleep 10\n set_service_not_expected_to_start etcd\n snapctl stop microk8s.daemon-etcd\n set_service_expected_to_start k8s-dqlite\nfi\n\nrm -rf ${SNAP_DATA}/var/lock/cni-loaded\n${SNAP}/microk8s-start.wrapper\n${SNAP}/microk8s-status.wrapper --wait-ready --timeout 120\n\necho \"Dqlite is enabled\"\n" }, { "path": "upgrade-scripts/001-switch-to-dqlite/commit-node.sh", "content": "#!/bin/bash\n\nset -ex\n\necho \"Switching node to dqlite\"\n\nsource $SNAP/actions/common/utils.sh\n\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite\"\n\nmkdir -p \"$BACKUP_DIR/args/\"\n\necho \"Configuring services\"\ncp \"$SNAP_DATA\"/args/kube-apiserver \"$BACKUP_DIR/args\"\n\nif [ -e ${SNAP}/default-args/k8s-dqlite ]\nthen\n echo \"Configuring the API server for dqlite\"\n cp ${SNAP}/default-args/k8s-dqlite ${SNAP_DATA}/args/k8s-dqlite\n refresh_opt_in_config etcd-servers unix://\\${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379 kube-apiserver\n skip_opt_in_config storage-backend kube-apiserver\n skip_opt_in_config storage-dir kube-apiserver\nelse\n refresh_opt_in_config \"storage-backend\" \"dqlite\" kube-apiserver\n refresh_opt_in_config \"storage-dir\" \"\\${SNAP_DATA}/var/kubernetes/backend/\" kube-apiserver\n skip_opt_in_config \"etcd-servers\" kube-apiserver\n skip_opt_in_config \"etcd-cafile\" kube-apiserver\n skip_opt_in_config \"etcd-certfile\" kube-apiserver\n skip_opt_in_config \"etcd-keyfile\" kube-apiserver\nfi\n\nif ! [ -e \"${SNAP_DATA}/var/kubernetes/backend/cluster.key\" ]\nthen\n init_cluster\nfi\n\nset_service_not_expected_to_start etcd\nset_service_expected_to_start k8s-dqlite\n\n${SNAP}/microk8s-stop.wrapper\nsleep 5\n${SNAP}/microk8s-start.wrapper\n\necho \"Dqlite is enabled on the node\"\n" }, { "path": "upgrade-scripts/001-switch-to-dqlite/description.txt", "content": "Migrates from etcd to dqlite" }, { "path": "upgrade-scripts/001-switch-to-dqlite/prepare-master.sh", "content": "#!/usr/bin/env bash\n\necho \"Master ready for dqlite\"\n\n" }, { "path": "upgrade-scripts/001-switch-to-dqlite/prepare-node.sh", "content": "#!/bin/bash\n\necho \"Nothing to do to praparing node for dqlite\"\n" }, { "path": "upgrade-scripts/001-switch-to-dqlite/rollback-master.sh", "content": "#!/bin/bash\nset -ex\n\necho \"Rolling back dqlite upgrade on master\"\n\nsource $SNAP/actions/common/utils.sh\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite\"\n\necho \"Restarting etcd\"\nset_service_expected_to_start etcd\nif [ -e \"$BACKUP_DIR/args/etcd\" ]; then\n cp \"$BACKUP_DIR\"/args/etcd \"$SNAP_DATA/args/\"\n snapctl restart ${SNAP_NAME}.daemon-etcd\nfi\n\necho \"Restarting kube-apiserver\"\nif [ -e \"$BACKUP_DIR/args/kube-apiserver\" ]; then\n cp \"$BACKUP_DIR\"/args/kube-apiserver \"$SNAP_DATA/args/\"\nfi\n\nsnapctl restart ${SNAP_NAME}.daemon-kubelite\n\n${SNAP}/microk8s-start.wrapper\n${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30\n\necho \"Dqlite rolled back\"\n" }, { "path": "upgrade-scripts/001-switch-to-dqlite/rollback-node.sh", "content": "#!/bin/bash\nset -ex\n\necho \"Rolling back dqlite upgrade on master\"\n\nsource $SNAP/actions/common/utils.sh\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite\"\n\necho \"Restarting etcd\"\nset_service_expected_to_start etcd\nif [ -e \"$BACKUP_DIR/args/etcd\" ]; then\n cp \"$BACKUP_DIR\"/args/etcd \"$SNAP_DATA/args/\"\n snapctl restart ${SNAP_NAME}.daemon-etcd\nfi\n\necho \"Restarting kube-apiserver\"\nif [ -e \"$BACKUP_DIR/args/kube-apiserver\" ]; then\n cp \"$BACKUP_DIR\"/args/kube-apiserver \"$SNAP_DATA/args/\"\nfi\n\nsnapctl restart ${SNAP_NAME}.daemon-kubelite\n\n${SNAP}/microk8s-start.wrapper\n\necho \"Dqlite rolled back\"\n" }, { "path": "upgrade-scripts/002-switch-to-flannel-etcd/commit-master.sh", "content": "#!/bin/bash\nset -ex\n\necho \"Switching master to flannel-etcd\"\n\nsource $SNAP/actions/common/utils.sh\n\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/002-switch-to-flannel-etcd\"\nDB_DIR=\"$BACKUP_DIR/db\"\n\nmkdir -p \"$BACKUP_DIR/args/\"\n\nif [ -e \"$SNAP_DATA/args/cni-network/cni.yaml\" ]; then\n KUBECTL=\"$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config\"\n $KUBECTL delete -f \"$SNAP_DATA/args/cni-network/cni.yaml\" || true\n sleep 10\nfi\n\necho \"Configuring services\"\n${SNAP}/microk8s-stop.wrapper\n\ncp \"$SNAP_DATA\"/args/kube-apiserver \"$BACKUP_DIR/args\"\n\n\"${SNAP}/bin/sed\" -i '/--storage-backend/d' \"$SNAP_DATA/args/kube-apiserver\"\n\"${SNAP}/bin/sed\" -i '/--storage-dir/d' \"$SNAP_DATA/args/kube-apiserver\"\n\"${SNAP}/bin/sed\" -i '/--etcd-servers/d' \"$SNAP_DATA/args/kube-apiserver\"\n\necho \"--etcd-servers=https://127.0.0.1:12379\" >> \"$SNAP_DATA/args/kube-apiserver\"\necho \"--etcd-cafile=\\${SNAP_DATA}/certs/ca.crt\" >> \"$SNAP_DATA/args/kube-apiserver\"\necho \"--etcd-certfile=\\${SNAP_DATA}/certs/server.crt\" >> \"$SNAP_DATA/args/kube-apiserver\"\necho \"--etcd-keyfile=\\${SNAP_DATA}/certs/server.key\" >> \"$SNAP_DATA/args/kube-apiserver\"\n\ncp \"$SNAP_DATA\"/args/etcd \"$BACKUP_DIR/args\"\nrm -rf ${SNAP_COMMON}/var/run/etcd/*\ncp \"$SNAP\"/default-args/etcd \"$SNAP_DATA\"/args/\nchmod 660 \"$SNAP_DATA\"/args/etcd\n\ncp -r \"$SNAP_DATA\"/args/cni-network \"$BACKUP_DIR/args/\"\nfind \"$SNAP_DATA\"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \\;\ncp --no-preserve=mode,ownership \"$SNAP\"/default-args/cni-network/* \"$SNAP_DATA\"/args/cni-network/\nchmod -R 770 \"$SNAP_DATA\"/args/cni-network\n\ngroup=$(get_microk8s_group)\nif getent group ${group} >/dev/null 2>&1\nthen\n chgrp ${group} -R ${SNAP_DATA}/args/ || true\nfi\n\nset_service_expected_to_start etcd\nset_service_expected_to_start flanneld\nset_service_not_expected_to_start k8s-dqlite\n\n${SNAP}/microk8s-start.wrapper\n${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30\n\necho \"Switch to etcd and flanneld completed\"\n" }, { "path": "upgrade-scripts/002-switch-to-flannel-etcd/commit-node.sh", "content": "#!/bin/bash\n\nset -ex\n\necho \"This will transition only the master node\"\nexit 1\n" }, { "path": "upgrade-scripts/002-switch-to-flannel-etcd/description.txt", "content": "Migrates to a flannel and etcd setup" }, { "path": "upgrade-scripts/002-switch-to-flannel-etcd/prepare-master.sh", "content": "#!/usr/bin/env bash\n\necho \"Master ready for flannel-etcd transition\"\n" }, { "path": "upgrade-scripts/002-switch-to-flannel-etcd/prepare-node.sh", "content": "#!/bin/bash\n\necho \"This will transition only the master node\"\nexit 1\n" }, { "path": "upgrade-scripts/002-switch-to-flannel-etcd/rollback-master.sh", "content": "#!/bin/bash\nset -ex\n\necho \"Rolling back flannel-etcd upgrade on master\"\n\nsource $SNAP/actions/common/utils.sh\nBACKUP_DIR=\"$SNAP_DATA/var/tmp/upgrades/002-switch-to-flannel-etcd\"\n\n${SNAP}/microk8s-stop.wrapper\n\necho \"Restarting etcd\"\nset_service_not_expected_to_start etcd\nif [ -e \"$BACKUP_DIR/args/etcd\" ]; then\n cp \"$BACKUP_DIR\"/args/etcd \"$SNAP_DATA/args/\"\nfi\n\necho \"Restarting kube-apiserver\"\nif [ -e \"$BACKUP_DIR/args/kube-apiserver\" ]; then\n cp \"$BACKUP_DIR\"/args/kube-apiserver \"$SNAP_DATA/args/\"\nfi\n\nset_service_not_expected_to_start flanneld\nif [ -e \"$BACKUP_DIR/args/cni-network\" ]; then\n find \"$SNAP_DATA\"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \\;\n cp -rf \"$BACKUP_DIR\"/args/cni-network/* \"$SNAP_DATA/args/cni-network/\"\nfi\n\ngroup=$(get_microk8s_group)\nchmod -R ug+rwX \"${SNAP_DATA}/args/\"\nchmod -R o-rwX \"${SNAP_DATA}/args/\"\nif getent group ${group} >/dev/null 2>&1\nthen\n chgrp ${group} -R ${SNAP_DATA}/args/ || true\nfi\n\n${SNAP}/microk8s-start.wrapper || true\n${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30\n\nif [ -e \"$SNAP_DATA/args/cni-network/cni.yaml\" ]; then\n KUBECTL=\"$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config\"\n $KUBECTL apply -f \"$SNAP_DATA/args/cni-network/cni.yaml\"\nfi\n\necho \"Flannel-etcd rolled back\"\n" }, { "path": "upgrade-scripts/002-switch-to-flannel-etcd/rollback-node.sh", "content": "#!/bin/bash\nset -ex\n\necho \"This will transition only the master node\"\nexit 1\n" } ]